From 924722e2d2f5bc62e6806582b34312d77f0f3e5f Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Sun, 25 Jun 2023 14:47:21 +0800 Subject: [PATCH 01/54] init --- actors/miner/src/deadlines.rs | 20 ++++ actors/miner/src/lib.rs | 178 ++++++++++++++++++++++++++++++++++ actors/miner/src/types.rs | 7 ++ 3 files changed, 205 insertions(+) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 3e625f908..f4b8a25c8 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -128,6 +128,26 @@ pub fn deadline_available_for_compaction( ) } +// the distance between from_deadline and to_deadline clockwise in deadline unit. +fn deadline_distance(policy: &Policy, from_deadline: u64, to_deadline: u64) -> u64 { + if to_deadline > from_deadline { + to_deadline - from_deadline + } else { + policy.wpost_period_deadlines - from_deadline + to_deadline + } +} + +// only allow moving to a nearer deadline from current one +pub fn deadline_available_for_move( + policy: &Policy, + from_deadline: u64, + to_deadline: u64, + current_deadline: u64, +) -> bool { + deadline_distance(policy, current_deadline, to_deadline) + < deadline_distance(policy, current_deadline, from_deadline) +} + // Determine current period start and deadline index directly from current epoch and // the offset implied by the proving period. This works correctly even for the state // of a miner actor without an active deadline cron diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 6411b722f..0a1a6cd01 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -128,6 +128,7 @@ pub enum Method { ChangeBeneficiary = 30, GetBeneficiary = 31, ExtendSectorExpiration2 = 32, + MovePartitions = 33, // Method numbers derived from FRC-0042 standards ChangeWorkerAddressExported = frc42_dispatch::method_hash!("ChangeWorkerAddress"), ChangePeerIDExported = frc42_dispatch::method_hash!("ChangePeerID"), @@ -2995,6 +2996,182 @@ impl Actor { Ok(()) } + fn move_partitions(rt: &impl Runtime, params: MovePartitionsParams) -> Result<(), ActorError> { + { + let policy = rt.policy(); + if params.from_deadline == params.to_deadline { + return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); + } + if params.from_deadline >= policy.wpost_period_deadlines + || params.to_deadline >= policy.wpost_period_deadlines + { + return Err(actor_error!( + illegal_argument, + "invalid deadline {}", + if params.from_deadline >= policy.wpost_period_deadlines { + params.from_deadline + } else { + params.to_deadline + } + )); + } + + let partitions = params.partitions.validate().map_err(|e| { + actor_error!(illegal_argument, "failed to parse partitions bitfield: {}", e) + })?; + + rt.transaction(|state: &mut State, rt| { + let info = get_miner_info(rt.store(), state)?; + + rt.validate_immediate_caller_is( + info.control_addresses.iter().chain(&[info.worker, info.owner]), + )?; + + if !deadline_available_for_compaction( + policy, + state.current_proving_period_start(policy, rt.curr_epoch()), + params.from_deadline, + rt.curr_epoch(), + ) { + return Err(actor_error!( + forbidden, + "cannot move from deadline {} during its challenge window, \ + or the prior challenge window, + or before {} epochs have passed since its last challenge window ended", + params.from_deadline, + policy.wpost_dispute_window + )); + } + + if !deadline_available_for_compaction( + policy, + state.current_proving_period_start(policy, rt.curr_epoch()), + params.to_deadline, + rt.curr_epoch(), + ) { + return Err(actor_error!( + forbidden, + "cannot move to deadline {} during its challenge window, \ + or the prior challenge window, + or before {} epochs have passed since its last challenge window ended", + params.to_deadline, + policy.wpost_dispute_window + )); + } + + let current_deadline = state.deadline_info(policy, rt.curr_epoch()); + if !deadline_available_for_move( + policy, + params.from_deadline, + params.to_deadline, + current_deadline.index, + ) { + return Err(actor_error!( + forbidden, + "cannot move to a deadline which is further from current deadline" + )); + } + + let store = rt.store(); + + let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); + let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); + let mut deadlines = + state.load_deadlines(store).map_err(|e| e.wrap("failed to load deadlines"))?; + + let mut from_deadline = + deadlines.load_deadline(policy, store, params.from_deadline).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load deadline {}", params.from_deadline), + ) + })?; + let mut to_deadline = + deadlines.load_deadline(policy, store, params.to_deadline).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load deadline {}", params.to_deadline), + ) + })?; + + let (live, dead, removed_power) = from_deadline + .remove_partitions(store, partitions, from_quant) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!( + "failed to remove partitions from deadline {}", + params.from_deadline + ), + ) + })?; + + state.delete_sectors(store, &dead).map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors") + })?; + + let sectors = state.load_sector_infos(store, &live).map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors") + })?; + let proven = true; + let added_power = to_deadline + .add_sectors( + store, + info.window_post_partition_sectors, + proven, + §ors, + info.sector_size, + to_quant, + ) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to add back moved sectors", + ) + })?; + + if removed_power != added_power { + return Err(actor_error!( + illegal_state, + "power changed when compacting partitions: was {:?}, is now {:?}", + removed_power, + added_power + )); + } + + deadlines + .update_deadline(policy, store, params.from_deadline, &from_deadline) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to update deadline {}", params.from_deadline), + ) + })?; + deadlines + .update_deadline(policy, store, params.to_deadline, &from_deadline) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to update deadline {}", params.to_deadline), + ) + })?; + + state.save_deadlines(store, deadlines).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!( + "failed to save deadline when move_partitions from {} to {}", + params.from_deadline, params.to_deadline + ), + ) + })?; + + Ok(()) + })?; + } + + Ok(()) + } /// Compacts sector number allocations to reduce the size of the allocated sector /// number bitfield. /// @@ -5113,6 +5290,7 @@ impl ActorCode for Actor { ConfirmSectorProofsValid => confirm_sector_proofs_valid, ChangeMultiaddrs|ChangeMultiaddrsExported => change_multiaddresses, CompactPartitions => compact_partitions, + MovePartitions => move_partitions, CompactSectorNumbers => compact_sector_numbers, ConfirmChangeWorkerAddress|ConfirmChangeWorkerAddressExported => confirm_change_worker_address, RepayDebt|RepayDebtExported => repay_debt, diff --git a/actors/miner/src/types.rs b/actors/miner/src/types.rs index b7d0a63be..8d4aa006f 100644 --- a/actors/miner/src/types.rs +++ b/actors/miner/src/types.rs @@ -243,6 +243,13 @@ pub struct CompactPartitionsParams { pub partitions: BitField, } +#[derive(Serialize_tuple, Deserialize_tuple)] +pub struct MovePartitionsParams { + pub from_deadline: u64, + pub to_deadline: u64, + pub partitions: BitField, +} + #[derive(Serialize_tuple, Deserialize_tuple)] pub struct CompactSectorNumbersParams { pub mask_sector_numbers: BitField, From da95dbbd766f9838ab81e3dcd1e26c31f5cc9676 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Sun, 25 Jun 2023 17:15:26 +0800 Subject: [PATCH 02/54] avoid recompute current_deadline --- actors/miner/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 0a1a6cd01..0b9571dac 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3027,9 +3027,10 @@ impl Actor { info.control_addresses.iter().chain(&[info.worker, info.owner]), )?; + let current_deadline = state.deadline_info(policy, rt.curr_epoch()); if !deadline_available_for_compaction( policy, - state.current_proving_period_start(policy, rt.curr_epoch()), + current_deadline.period_start, params.from_deadline, rt.curr_epoch(), ) { @@ -3045,7 +3046,7 @@ impl Actor { if !deadline_available_for_compaction( policy, - state.current_proving_period_start(policy, rt.curr_epoch()), + current_deadline.period_start, params.to_deadline, rt.curr_epoch(), ) { @@ -3059,7 +3060,6 @@ impl Actor { )); } - let current_deadline = state.deadline_info(policy, rt.curr_epoch()); if !deadline_available_for_move( policy, params.from_deadline, From e5027094e70fd2f765c274e6ba92a66f75f8d13b Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Mon, 26 Jun 2023 20:29:24 +0800 Subject: [PATCH 03/54] treat empty bitfield as all --- actors/miner/src/lib.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 0b9571dac..7fed685fa 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2996,7 +2996,10 @@ impl Actor { Ok(()) } - fn move_partitions(rt: &impl Runtime, params: MovePartitionsParams) -> Result<(), ActorError> { + fn move_partitions( + rt: &impl Runtime, + mut params: MovePartitionsParams, + ) -> Result<(), ActorError> { { let policy = rt.policy(); if params.from_deadline == params.to_deadline { @@ -3016,9 +3019,7 @@ impl Actor { )); } - let partitions = params.partitions.validate().map_err(|e| { - actor_error!(illegal_argument, "failed to parse partitions bitfield: {}", e) - })?; + let partitions = &mut params.partitions; rt.transaction(|state: &mut State, rt| { let info = get_miner_info(rt.store(), state)?; @@ -3094,6 +3095,22 @@ impl Actor { ) })?; + if partitions.len() == 0 { + let partitions_amt = from_deadline.partitions_amt(rt.store()).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!( + "failed to load partitions for deadline {}", + params.from_deadline + ), + ) + })?; + + for partition_idx in 0..partitions_amt.count() { + partitions.set(partition_idx); + } + } + let (live, dead, removed_power) = from_deadline .remove_partitions(store, partitions, from_quant) .map_err(|e| { From eaf5370505b23009d3484e23a9fb0fa7672c7aa0 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Mon, 26 Jun 2023 20:35:28 +0800 Subject: [PATCH 04/54] rm useless quote --- actors/miner/src/lib.rs | 308 ++++++++++++++++++++-------------------- 1 file changed, 151 insertions(+), 157 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 7fed685fa..fc35f0aa0 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3000,192 +3000,186 @@ impl Actor { rt: &impl Runtime, mut params: MovePartitionsParams, ) -> Result<(), ActorError> { + let policy = rt.policy(); + if params.from_deadline == params.to_deadline { + return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); + } + if params.from_deadline >= policy.wpost_period_deadlines + || params.to_deadline >= policy.wpost_period_deadlines { - let policy = rt.policy(); - if params.from_deadline == params.to_deadline { - return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); - } - if params.from_deadline >= policy.wpost_period_deadlines - || params.to_deadline >= policy.wpost_period_deadlines - { - return Err(actor_error!( - illegal_argument, - "invalid deadline {}", - if params.from_deadline >= policy.wpost_period_deadlines { - params.from_deadline - } else { - params.to_deadline - } - )); - } + return Err(actor_error!( + illegal_argument, + "invalid deadline {}", + if params.from_deadline >= policy.wpost_period_deadlines { + params.from_deadline + } else { + params.to_deadline + } + )); + } - let partitions = &mut params.partitions; + let partitions = &mut params.partitions; - rt.transaction(|state: &mut State, rt| { - let info = get_miner_info(rt.store(), state)?; + rt.transaction(|state: &mut State, rt| { + let info = get_miner_info(rt.store(), state)?; - rt.validate_immediate_caller_is( - info.control_addresses.iter().chain(&[info.worker, info.owner]), - )?; + rt.validate_immediate_caller_is( + info.control_addresses.iter().chain(&[info.worker, info.owner]), + )?; - let current_deadline = state.deadline_info(policy, rt.curr_epoch()); - if !deadline_available_for_compaction( - policy, - current_deadline.period_start, - params.from_deadline, - rt.curr_epoch(), - ) { - return Err(actor_error!( - forbidden, - "cannot move from deadline {} during its challenge window, \ + let current_deadline = state.deadline_info(policy, rt.curr_epoch()); + if !deadline_available_for_compaction( + policy, + current_deadline.period_start, + params.from_deadline, + rt.curr_epoch(), + ) { + return Err(actor_error!( + forbidden, + "cannot move from deadline {} during its challenge window, \ or the prior challenge window, or before {} epochs have passed since its last challenge window ended", - params.from_deadline, - policy.wpost_dispute_window - )); - } + params.from_deadline, + policy.wpost_dispute_window + )); + } - if !deadline_available_for_compaction( - policy, - current_deadline.period_start, - params.to_deadline, - rt.curr_epoch(), - ) { - return Err(actor_error!( - forbidden, - "cannot move to deadline {} during its challenge window, \ + if !deadline_available_for_compaction( + policy, + current_deadline.period_start, + params.to_deadline, + rt.curr_epoch(), + ) { + return Err(actor_error!( + forbidden, + "cannot move to deadline {} during its challenge window, \ or the prior challenge window, or before {} epochs have passed since its last challenge window ended", - params.to_deadline, - policy.wpost_dispute_window - )); - } - - if !deadline_available_for_move( - policy, - params.from_deadline, params.to_deadline, - current_deadline.index, - ) { - return Err(actor_error!( - forbidden, - "cannot move to a deadline which is further from current deadline" - )); - } - - let store = rt.store(); - - let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); - let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); - let mut deadlines = - state.load_deadlines(store).map_err(|e| e.wrap("failed to load deadlines"))?; - - let mut from_deadline = - deadlines.load_deadline(policy, store, params.from_deadline).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.from_deadline), - ) - })?; - let mut to_deadline = - deadlines.load_deadline(policy, store, params.to_deadline).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.to_deadline), - ) - })?; + policy.wpost_dispute_window + )); + } - if partitions.len() == 0 { - let partitions_amt = from_deadline.partitions_amt(rt.store()).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - format!( - "failed to load partitions for deadline {}", - params.from_deadline - ), - ) - })?; + if !deadline_available_for_move( + policy, + params.from_deadline, + params.to_deadline, + current_deadline.index, + ) { + return Err(actor_error!( + forbidden, + "cannot move to a deadline which is further from current deadline" + )); + } - for partition_idx in 0..partitions_amt.count() { - partitions.set(partition_idx); - } - } + let store = rt.store(); - let (live, dead, removed_power) = from_deadline - .remove_partitions(store, partitions, from_quant) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - format!( - "failed to remove partitions from deadline {}", - params.from_deadline - ), - ) - })?; + let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); + let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); + let mut deadlines = + state.load_deadlines(store).map_err(|e| e.wrap("failed to load deadlines"))?; - state.delete_sectors(store, &dead).map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors") + let mut from_deadline = + deadlines.load_deadline(policy, store, params.from_deadline).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load deadline {}", params.from_deadline), + ) })?; - - let sectors = state.load_sector_infos(store, &live).map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors") + let mut to_deadline = + deadlines.load_deadline(policy, store, params.to_deadline).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load deadline {}", params.to_deadline), + ) })?; - let proven = true; - let added_power = to_deadline - .add_sectors( - store, - info.window_post_partition_sectors, - proven, - §ors, - info.sector_size, - to_quant, + + if partitions.len() == 0 { + let partitions_amt = from_deadline.partitions_amt(rt.store()).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load partitions for deadline {}", params.from_deadline), ) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "failed to add back moved sectors", - ) - })?; + })?; - if removed_power != added_power { - return Err(actor_error!( - illegal_state, - "power changed when compacting partitions: was {:?}, is now {:?}", - removed_power, - added_power - )); + for partition_idx in 0..partitions_amt.count() { + partitions.set(partition_idx); } + } - deadlines - .update_deadline(policy, store, params.from_deadline, &from_deadline) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - format!("failed to update deadline {}", params.from_deadline), - ) - })?; - deadlines - .update_deadline(policy, store, params.to_deadline, &from_deadline) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - format!("failed to update deadline {}", params.to_deadline), - ) - })?; - - state.save_deadlines(store, deadlines).map_err(|e| { + let (live, dead, removed_power) = + from_deadline.remove_partitions(store, partitions, from_quant).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, format!( - "failed to save deadline when move_partitions from {} to {}", - params.from_deadline, params.to_deadline + "failed to remove partitions from deadline {}", + params.from_deadline ), ) })?; - Ok(()) + state.delete_sectors(store, &dead).map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors") })?; - } + + let sectors = state.load_sector_infos(store, &live).map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors") + })?; + let proven = true; + let added_power = to_deadline + .add_sectors( + store, + info.window_post_partition_sectors, + proven, + §ors, + info.sector_size, + to_quant, + ) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to add back moved sectors", + ) + })?; + + if removed_power != added_power { + return Err(actor_error!( + illegal_state, + "power changed when compacting partitions: was {:?}, is now {:?}", + removed_power, + added_power + )); + } + + deadlines + .update_deadline(policy, store, params.from_deadline, &from_deadline) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to update deadline {}", params.from_deadline), + ) + })?; + deadlines.update_deadline(policy, store, params.to_deadline, &from_deadline).map_err( + |e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to update deadline {}", params.to_deadline), + ) + }, + )?; + + state.save_deadlines(store, deadlines).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + format!( + "failed to save deadline when move_partitions from {} to {}", + params.from_deadline, params.to_deadline + ), + ) + })?; + + Ok(()) + })?; Ok(()) } From 2fa969354fa6d4a8b201ef41b517774ecd083273 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 27 Jun 2023 17:44:41 +0800 Subject: [PATCH 05/54] add verify --- actors/miner/src/lib.rs | 101 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index fc35f0aa0..f44c78aed 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3027,7 +3027,106 @@ impl Actor { info.control_addresses.iter().chain(&[info.worker, info.owner]), )?; + let store = rt.store(); let current_deadline = state.deadline_info(policy, rt.curr_epoch()); + { + // Load the target deadline + let deadlines_current = state + .load_deadlines(rt.store()) + .map_err(|e| e.wrap("failed to load deadlines"))?; + + let dl_current = deadlines_current + .load_deadline(policy, rt.store(), params.from_deadline) + .map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline") + })?; + + let proofs_snapshot = + dl_current.optimistic_proofs_snapshot_amt(store).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to load proofs snapshot", + ) + })?; + + let partitions_snapshot = + dl_current.partitions_snapshot_amt(store).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to load partitions snapshot", + ) + })?; + + // Load sectors for the dispute. + let sectors = + Sectors::load(rt.store(), &dl_current.sectors_snapshot).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to load sectors array", + ) + })?; + + proofs_snapshot + .for_each(|_, window_proof| { + let mut all_sectors = + Vec::::with_capacity(window_proof.partitions.len() as usize); + let mut all_ignored = + Vec::::with_capacity(window_proof.partitions.len() as usize); + + for partition_idx in window_proof.partitions.iter() { + let partition = partitions_snapshot + .get(partition_idx) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to load partitions snapshot for proof", + ) + })? + .ok_or_else(|| { + actor_error!( + illegal_state, + "failed to load partitions snapshot for proof" + ) + })?; + all_sectors.push(partition.sectors.clone()); + all_ignored.push(partition.terminated.clone()); + all_ignored.push(partition.faults.clone()); + } + + // Load sector infos for proof, substituting a known-good sector for known-faulty sectors. + // Note: this is slightly sub-optimal, loading info for the recovering sectors again after they were already + // loaded above. + let sector_infos = sectors + .load_for_proof( + &BitField::union(&all_sectors), + &BitField::union(&all_ignored), + ) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to load sectors for post verification", + ) + })?; + if !verify_windowed_post( + rt, + current_deadline.challenge, + §or_infos, + window_proof.proofs.clone(), + ) + .map_err(|e| e.wrap("window post failed"))? + { + return Err(actor_error!( + illegal_argument, + "invalid post was submitted" + ) + .into()); + } + Ok(()) + }) + .map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "while removing partitions") + })?; + } if !deadline_available_for_compaction( policy, current_deadline.period_start, @@ -3072,8 +3171,6 @@ impl Actor { )); } - let store = rt.store(); - let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); let mut deadlines = From 872542055607c0b209c4a257fa9471df5a883ad4 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 09:40:01 +0800 Subject: [PATCH 06/54] combine option1 & option2 --- actors/miner/src/lib.rs | 61 +++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index f44c78aed..303a0b4c7 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3023,19 +3023,38 @@ impl Actor { rt.transaction(|state: &mut State, rt| { let info = get_miner_info(rt.store(), state)?; - rt.validate_immediate_caller_is( - info.control_addresses.iter().chain(&[info.worker, info.owner]), - )?; + rt.validate_immediate_caller_is(&[info.worker, info.owner])?; let store = rt.store(); let current_deadline = state.deadline_info(policy, rt.curr_epoch()); - { - // Load the target deadline - let deadlines_current = state - .load_deadlines(rt.store()) - .map_err(|e| e.wrap("failed to load deadlines"))?; + let mut deadlines = + state.load_deadlines(store).map_err(|e| e.wrap("failed to load deadlines"))?; + + // only try to do synchronous Window Post verification if from_deadline doesn't satisfy deadline_available_for_compaction + if !deadline_available_for_compaction( + policy, + current_deadline.period_start, + params.from_deadline, + rt.curr_epoch(), + ) { + // the heavy path: try to do synchronous Window Post verification + + // current deadline must be in the dispute window to satisfy the condition of synchronous Window POST verification + if !deadline_available_for_optimistic_post_dispute( + policy, + current_deadline.period_start, + params.from_deadline, + rt.curr_epoch(), + ) { + return Err(actor_error!( + forbidden, + "cannot move from deadline {} when !deadline_available_for_compaction && !deadline_available_for_optimistic_post_dispute", + params.from_deadline + )); + } + - let dl_current = deadlines_current + let dl_current = deadlines .load_deadline(policy, rt.store(), params.from_deadline) .map_err(|e| { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline") @@ -3090,7 +3109,12 @@ impl Actor { })?; all_sectors.push(partition.sectors.clone()); all_ignored.push(partition.terminated.clone()); - all_ignored.push(partition.faults.clone()); + // fail early since remove_partitions will fail when there're faults anyway. + if !partition.faults.is_empty() { + return Err(anyhow::anyhow!("unable to do synchronous Window POST verification while there're faults in from deadline {}", + params.from_deadline + )); + } } // Load sector infos for proof, substituting a known-good sector for known-faulty sectors. @@ -3127,21 +3151,6 @@ impl Actor { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "while removing partitions") })?; } - if !deadline_available_for_compaction( - policy, - current_deadline.period_start, - params.from_deadline, - rt.curr_epoch(), - ) { - return Err(actor_error!( - forbidden, - "cannot move from deadline {} during its challenge window, \ - or the prior challenge window, - or before {} epochs have passed since its last challenge window ended", - params.from_deadline, - policy.wpost_dispute_window - )); - } if !deadline_available_for_compaction( policy, @@ -3173,8 +3182,6 @@ impl Actor { let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); - let mut deadlines = - state.load_deadlines(store).map_err(|e| e.wrap("failed to load deadlines"))?; let mut from_deadline = deadlines.load_deadline(policy, store, params.from_deadline).map_err(|e| { From d534373674d42b7040a083ade2c3b98dd541458d Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 10:09:22 +0800 Subject: [PATCH 07/54] fix --- actors/miner/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 303a0b4c7..04801f9fa 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3263,7 +3263,7 @@ impl Actor { format!("failed to update deadline {}", params.from_deadline), ) })?; - deadlines.update_deadline(policy, store, params.to_deadline, &from_deadline).map_err( + deadlines.update_deadline(policy, store, params.to_deadline, &to_deadline).map_err( |e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, From 36caece9066ee5c7c0c82c205aa875413798665a Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 10:33:20 +0800 Subject: [PATCH 08/54] fix --- actors/miner/src/deadlines.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index f4b8a25c8..cd58f52b5 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -130,7 +130,7 @@ pub fn deadline_available_for_compaction( // the distance between from_deadline and to_deadline clockwise in deadline unit. fn deadline_distance(policy: &Policy, from_deadline: u64, to_deadline: u64) -> u64 { - if to_deadline > from_deadline { + if to_deadline >= from_deadline { to_deadline - from_deadline } else { policy.wpost_period_deadlines - from_deadline + to_deadline From badb69167f96909d7ca8ad37b7b0c5c9e5f95c44 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 11:20:11 +0800 Subject: [PATCH 09/54] nit --- actors/miner/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 04801f9fa..490434fd7 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3000,10 +3000,11 @@ impl Actor { rt: &impl Runtime, mut params: MovePartitionsParams, ) -> Result<(), ActorError> { - let policy = rt.policy(); + if params.from_deadline == params.to_deadline { return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); } + let policy = rt.policy(); if params.from_deadline >= policy.wpost_period_deadlines || params.to_deadline >= policy.wpost_period_deadlines { From fd39aac668c00df70fc31ab3fc07484418c3b972 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 11:24:12 +0800 Subject: [PATCH 10/54] mod error --- actors/miner/src/lib.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 490434fd7..3c105caee 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3010,12 +3010,9 @@ impl Actor { { return Err(actor_error!( illegal_argument, - "invalid deadline {}", - if params.from_deadline >= policy.wpost_period_deadlines { - params.from_deadline - } else { - params.to_deadline - } + "invalid param, from_deadline: {}, to_deadline: {}", + params.from_deadline, + params.to_deadline )); } From daab781a2a811b763c11bf6649a1c8a223ffca94 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 13:30:55 +0800 Subject: [PATCH 11/54] nit --- actors/miner/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 3c105caee..61e90fcd5 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3000,7 +3000,6 @@ impl Actor { rt: &impl Runtime, mut params: MovePartitionsParams, ) -> Result<(), ActorError> { - if params.from_deadline == params.to_deadline { return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); } @@ -3016,8 +3015,6 @@ impl Actor { )); } - let partitions = &mut params.partitions; - rt.transaction(|state: &mut State, rt| { let info = get_miner_info(rt.store(), state)?; @@ -3196,6 +3193,7 @@ impl Actor { ) })?; + let partitions = &mut params.partitions; if partitions.len() == 0 { let partitions_amt = from_deadline.partitions_amt(rt.store()).map_err(|e| { e.downcast_default( From 0271369ab0debc7760f3fc62c55bd45b94254abf Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 14:31:53 +0800 Subject: [PATCH 12/54] fmt --- actors/miner/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 61e90fcd5..cace30e10 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3033,7 +3033,6 @@ impl Actor { rt.curr_epoch(), ) { // the heavy path: try to do synchronous Window Post verification - // current deadline must be in the dispute window to satisfy the condition of synchronous Window POST verification if !deadline_available_for_optimistic_post_dispute( policy, From c046cc64009868ba1153cb03a96d6b39c9ec0d72 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 28 Jun 2023 15:46:50 +0800 Subject: [PATCH 13/54] fix ci --- actors/miner/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index cace30e10..111a9a52f 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3193,7 +3193,7 @@ impl Actor { })?; let partitions = &mut params.partitions; - if partitions.len() == 0 { + if partitions.is_empty() { let partitions_amt = from_deadline.partitions_amt(rt.store()).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, From 3eac91ee882dbef5a588f532e8218f40936536e5 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Mon, 3 Jul 2023 13:17:25 +0800 Subject: [PATCH 14/54] fix bug --- actors/miner/src/lib.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 111a9a52f..74645c24e 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3048,14 +3048,14 @@ impl Actor { } - let dl_current = deadlines + let dl_from = deadlines .load_deadline(policy, rt.store(), params.from_deadline) .map_err(|e| { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline") })?; let proofs_snapshot = - dl_current.optimistic_proofs_snapshot_amt(store).map_err(|e| { + dl_from.optimistic_proofs_snapshot_amt(store).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, "failed to load proofs snapshot", @@ -3063,7 +3063,7 @@ impl Actor { })?; let partitions_snapshot = - dl_current.partitions_snapshot_amt(store).map_err(|e| { + dl_from.partitions_snapshot_amt(store).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, "failed to load partitions snapshot", @@ -3072,7 +3072,7 @@ impl Actor { // Load sectors for the dispute. let sectors = - Sectors::load(rt.store(), &dl_current.sectors_snapshot).map_err(|e| { + Sectors::load(rt.store(), &dl_from.sectors_snapshot).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, "failed to load sectors array", @@ -3125,9 +3125,18 @@ impl Actor { "failed to load sectors for post verification", ) })?; + + // Find the proving period start for the deadline in question. + let mut pp_start = current_deadline.period_start; + if current_deadline.index < params.from_deadline { + pp_start -= policy.wpost_proving_period + } + let target_deadline = + new_deadline_info(policy, pp_start, params.from_deadline, rt.curr_epoch()); + if !verify_windowed_post( rt, - current_deadline.challenge, + target_deadline.challenge, §or_infos, window_proof.proofs.clone(), ) From 181dc2eb1773ac63144ad258dc3df5baf91f23e2 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 4 Jul 2023 10:25:01 +0800 Subject: [PATCH 15/54] add test --- actors/miner/tests/move_partitions_test.rs | 337 +++++++++++++++++++++ actors/miner/tests/util.rs | 33 +- 2 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 actors/miner/tests/move_partitions_test.rs diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs new file mode 100644 index 000000000..6500ba4f8 --- /dev/null +++ b/actors/miner/tests/move_partitions_test.rs @@ -0,0 +1,337 @@ +use fil_actor_miner::{ + deadline_available_for_compaction, deadline_available_for_optimistic_post_dispute, + new_deadline_info, DeadlineInfo, +}; +use fil_actors_runtime::{ + runtime::{DomainSeparationTag, RuntimePolicy}, + test_utils::{expect_abort_contains_message, MockRuntime}, +}; +use fvm_ipld_bitfield::BitField; +use fvm_ipld_encoding::RawBytes; +use fvm_shared::randomness::Randomness; +use fvm_shared::{clock::ChainEpoch, error::ExitCode}; + +mod util; +use util::*; +const PERIOD_OFFSET: ChainEpoch = 100; + +fn setup() -> (ActorHarness, MockRuntime) { + let h = ActorHarness::new(PERIOD_OFFSET); + let rt = h.new_runtime(); + h.construct_and_verify(&rt); + rt.balance.replace(BIG_BALANCE.clone()); + + (h, rt) +} + +// returns the nearest epoch such that synchronous post verification is required +fn nearest_unsafe_epoch(rt: &MockRuntime, h: &ActorHarness, from_deadline_id: u64) -> i64 { + let current_ddl = h.current_deadline(rt); + + for i in *rt.epoch.borrow().. { + if !deadline_available_for_compaction( + &rt.policy, + current_ddl.period_start, + from_deadline_id, + i, + ) && deadline_available_for_optimistic_post_dispute( + &rt.policy, + current_ddl.period_start, + from_deadline_id, + i, + ) { + return i; + } + } + + panic!("impossible path"); +} + +// returns the nearest epoch such that no synchronous post verification is necessary +fn nearest_safe_epoch(rt: &MockRuntime, h: &ActorHarness, from_deadline_id: u64) -> i64 { + let current_ddl = h.current_deadline(rt); + + for i in *rt.epoch.borrow().. { + if deadline_available_for_compaction( + &rt.policy, + current_ddl.period_start, + from_deadline_id, + i, + ) { + return i; + } + } + + panic!("impossible path"); +} + +// returns the farthest deadline from current that satisfies deadline_available_for_move +fn farthest_possible_to_deadline( + rt: &MockRuntime, + from_deadline_id: u64, + current_deadline: DeadlineInfo, +) -> u64 { + assert!(from_deadline_id != current_deadline.index, "can't move nearer when the gap is 0"); + + if current_deadline.index < from_deadline_id { + // the deadline distance can only be nearer + for i in (current_deadline.index..(from_deadline_id)).rev() { + if deadline_available_for_compaction( + &rt.policy, + current_deadline.period_start, + i, + *rt.epoch.borrow(), + ) { + return i; + } + } + } else { + for i in (0..(from_deadline_id)).rev() { + if deadline_available_for_compaction( + &rt.policy, + current_deadline.period_start, + i, + *rt.epoch.borrow(), + ) { + return i; + } + } + + for i in (current_deadline.index..rt.policy.wpost_period_deadlines).rev() { + if deadline_available_for_compaction( + &rt.policy, + current_deadline.period_start, + i, + *rt.epoch.borrow(), + ) { + return i; + } + } + } + + panic!("no candidate to_deadline"); +} + +#[test] +fn fail_to_move_partitions_with_faults_from_safe_epoch() { + let (mut h, rt) = setup(); + rt.set_epoch(200); + + // create 2 sectors in partition 0 + let sectors_info = h.commit_and_prove_sectors( + &rt, + 2, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + // fault sector 1 + h.declare_faults(&rt, §ors_info[0..1]); + + let partition_id = 0; + let from_deadline_id = 0; + + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, from_deadline_id)); + + let to_deadline_id = + farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + from_deadline_id, + to_deadline_id, + bitfield_from_slice(&[partition_id]), + || {}, + ); + expect_abort_contains_message( + ExitCode::USR_ILLEGAL_ARGUMENT, + "cannot remove partition 0: has faults", + result, + ); + + h.check_state(&rt); +} + +#[test] +fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { + let (mut h, rt) = setup(); + rt.set_epoch(200); + + // create 2 sectors in partition 0 + let sectors_info = h.commit_and_prove_sectors( + &rt, + 2, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + // fault sector 1 + h.declare_faults(&rt, §ors_info[0..1]); + + let partition_id = 0; + let from_deadline_id = 0; + + h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, from_deadline_id)); + + let to_deadline_id = + farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + from_deadline_id, + to_deadline_id, + bitfield_from_slice(&[partition_id]), + || { + let current_deadline = h.current_deadline(&rt); + + let from_deadline = new_deadline_info( + rt.policy(), + if current_deadline.index < from_deadline_id { + current_deadline.period_start - rt.policy().wpost_proving_period + } else { + current_deadline.period_start + }, + from_deadline_id, + *rt.epoch.borrow(), + ); + + let from_ddl = h.get_deadline(&rt, from_deadline_id); + + let entropy = RawBytes::serialize(h.receiver).unwrap(); + rt.expect_get_randomness_from_beacon( + DomainSeparationTag::WindowedPoStChallengeSeed, + from_deadline.challenge, + entropy.to_vec(), + TEST_RANDOMNESS_ARRAY_FROM_ONE, + ); + + let post = h.get_submitted_proof(&rt, &from_ddl, 0); + + let all_ignored = BitField::new(); + let vi = h.make_window_post_verify_info( + §ors_info, + &all_ignored, + sectors_info[1].clone(), + Randomness(TEST_RANDOMNESS_ARRAY_FROM_ONE.into()), + post.proofs, + ); + rt.expect_verify_post(vi, ExitCode::OK); + }, + ); + expect_abort_contains_message( + ExitCode::USR_ILLEGAL_ARGUMENT, + "cannot remove partition 0: has faults", + result, + ); + + h.check_state(&rt); +} + +#[test] +fn ok_to_move_partitions_from_safe_epoch() { + let (mut h, rt) = setup(); + rt.set_epoch(200); + + // create 2 sectors in partition 0 + let sectors_info = h.commit_and_prove_sectors( + &rt, + 2, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + let from_deadline_id = 0; + + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, from_deadline_id)); + + let partition_id = 0; + let to_deadline_id = + farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + from_deadline_id, + to_deadline_id, + bitfield_from_slice(&[partition_id]), + || {}, + ); + assert!(result.is_ok()); + + h.check_state(&rt); +} + +#[test] +fn ok_to_move_partitions_from_unsafe_epoch() { + let (mut h, rt) = setup(); + rt.set_epoch(200); + + // create 2 sectors in partition 0 + let sectors_info = h.commit_and_prove_sectors( + &rt, + 2, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + let from_deadline_id = 0; + + h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, from_deadline_id)); + + let partition_id = 0; + let to_deadline_id = + farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + from_deadline_id, + to_deadline_id, + bitfield_from_slice(&[partition_id]), + || { + let current_deadline = h.current_deadline(&rt); + + let from_deadline = new_deadline_info( + rt.policy(), + if current_deadline.index < from_deadline_id { + current_deadline.period_start - rt.policy().wpost_proving_period + } else { + current_deadline.period_start + }, + from_deadline_id, + *rt.epoch.borrow(), + ); + + let from_ddl = h.get_deadline(&rt, from_deadline_id); + + let entropy = RawBytes::serialize(h.receiver).unwrap(); + rt.expect_get_randomness_from_beacon( + DomainSeparationTag::WindowedPoStChallengeSeed, + from_deadline.challenge, + entropy.to_vec(), + TEST_RANDOMNESS_ARRAY_FROM_ONE, + ); + + let post = h.get_submitted_proof(&rt, &from_ddl, 0); + + let all_ignored = BitField::new(); + let vi = h.make_window_post_verify_info( + §ors_info, + &all_ignored, + sectors_info[1].clone(), + Randomness(TEST_RANDOMNESS_ARRAY_FROM_ONE.into()), + post.proofs, + ); + rt.expect_verify_post(vi, ExitCode::OK); + }, + ); + assert!(result.is_ok()); + + h.check_state(&rt); +} diff --git a/actors/miner/tests/util.rs b/actors/miner/tests/util.rs index cc44bf1db..2ec1d66cc 100644 --- a/actors/miner/tests/util.rs +++ b/actors/miner/tests/util.rs @@ -25,7 +25,7 @@ use fil_actor_miner::{ ExpirationQueue, ExpirationSet, ExtendSectorExpiration2Params, ExtendSectorExpirationParams, FaultDeclaration, GetAvailableBalanceReturn, GetBeneficiaryReturn, GetControlAddressesReturn, GetMultiaddrsReturn, GetPeerIDReturn, Method, MinerConstructorParams as ConstructorParams, - MinerInfo, Partition, PendingBeneficiaryChange, PoStPartition, PowerPair, + MinerInfo, MovePartitionsParams, Partition, PendingBeneficiaryChange, PoStPartition, PowerPair, PreCommitSectorBatchParams, PreCommitSectorBatchParams2, PreCommitSectorParams, ProveCommitSectorParams, RecoveryDeclaration, ReportConsensusFaultParams, SectorOnChainInfo, SectorPreCommitInfo, SectorPreCommitOnChainInfo, Sectors, State, SubmitWindowedPoStParams, @@ -1470,7 +1470,7 @@ impl ActorHarness { ) } - fn make_window_post_verify_info( + pub fn make_window_post_verify_info( &self, infos: &[SectorOnChainInfo], all_ignored: &BitField, @@ -1630,7 +1630,12 @@ impl ActorHarness { rt.verify(); } - fn get_submitted_proof(&self, rt: &MockRuntime, deadline: &Deadline, idx: u64) -> WindowedPoSt { + pub fn get_submitted_proof( + &self, + rt: &MockRuntime, + deadline: &Deadline, + idx: u64, + ) -> WindowedPoSt { amt_get::(rt, &deadline.optimistic_post_submissions_snapshot, idx) } @@ -2590,6 +2595,28 @@ impl ActorHarness { Ok(()) } + pub fn move_partitions( + &self, + rt: &MockRuntime, + from_deadline: u64, + to_deadline: u64, + partitions: BitField, + mut f: impl FnMut(), + ) -> Result<(), ActorError> { + f(); + + let params = MovePartitionsParams { from_deadline, to_deadline, partitions }; + + rt.expect_validate_caller_addr(vec![self.worker, self.owner]); + rt.set_caller(*ACCOUNT_ACTOR_CODE_ID, self.worker); + + rt.call::( + Method::MovePartitions as u64, + IpldBlock::serialize_cbor(¶ms).unwrap(), + )?; + rt.verify(); + Ok(()) + } pub fn get_info(&self, rt: &MockRuntime) -> MinerInfo { let state: State = rt.get_state(); state.get_info(rt.store()).unwrap() From 8eb224374e324333456e55e7bce9f09fb64305b8 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 5 Jul 2023 14:07:50 +0800 Subject: [PATCH 16/54] add more test --- test_vm/tests/move_partitions_test.rs | 215 ++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 test_vm/tests/move_partitions_test.rs diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs new file mode 100644 index 000000000..38b4a419f --- /dev/null +++ b/test_vm/tests/move_partitions_test.rs @@ -0,0 +1,215 @@ +use fvm_ipld_bitfield::BitField; +use fvm_ipld_blockstore::{Blockstore, MemoryBlockstore}; +use fvm_ipld_encoding::ipld_block::IpldBlock; +use fvm_shared::address::Address; +use fvm_shared::bigint::Zero; +use fvm_shared::econ::TokenAmount; +use fvm_shared::error::ExitCode; +use fvm_shared::sector::{RegisteredSealProof, SectorNumber}; + +use fil_actor_cron::Method as CronMethod; +use fil_actor_market::Method as MarketMethod; +use fil_actor_miner::{ + power_for_sector, DeadlineInfo, Method as MinerMethod, MovePartitionsParams, + ProveCommitSectorParams, State as MinerState, +}; + +use fil_actor_power::{Method as PowerMethod, State as PowerState}; +use fil_actors_runtime::runtime::Policy; +use fil_actors_runtime::{ + CRON_ACTOR_ADDR, STORAGE_MARKET_ACTOR_ADDR, STORAGE_POWER_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, +}; +use test_vm::expects::Expect; +use test_vm::trace::ExpectInvocation; +use test_vm::util::{ + advance_by_deadline_to_epoch, advance_to_proving_deadline, apply_ok, assert_invariants, + create_accounts, create_miner, cron_tick, get_network_stats, get_state, miner_balance, + precommit_sectors, submit_windowed_post, +}; +use test_vm::{TestVM, VM}; + +#[test] +fn move_partitions_success() { + let store = MemoryBlockstore::new(); + let (v, miner, sector) = setup(&store); + + submit_post_succeeds_test(&v, miner.clone(), sector); + + let prove_time = v.epoch() + Policy::default().wpost_dispute_window; + advance_by_deadline_to_epoch(&v, &miner.miner_id, prove_time); + + let move_params = + MovePartitionsParams { from_deadline: 0, to_deadline: 47, partitions: BitField::new() }; + let prove_params_ser = IpldBlock::serialize_cbor(&move_params).unwrap(); + apply_ok( + &v, + &miner.worker, + &miner._miner_robust, + &TokenAmount::zero(), + MinerMethod::MovePartitions as u64, + Some(move_params), + ); + ExpectInvocation { + from: miner.worker, + to: miner.miner_id, + method: MinerMethod::MovePartitions as u64, + params: Some(prove_params_ser), + subinvocs: Some(vec![]), + ..Default::default() + } + .matches(v.take_invocations().last().unwrap()); + + cron_tick(&v); + v.set_epoch(v.epoch() + 1); + assert_invariants(&v); +} + +fn submit_post_succeeds_test( + v: &dyn VM, + miner_info: MinerInfo, + sector_info: SectorInfo, +) { + // submit post + let st: MinerState = get_state(v, &miner_info.miner_id).unwrap(); + let sector = st.get_sector(*v.blockstore(), sector_info.number).unwrap().unwrap(); + let sector_power = power_for_sector(miner_info.seal_proof.sector_size().unwrap(), §or); + submit_windowed_post( + v, + &miner_info.worker, + &miner_info.miner_id, + sector_info.deadline_info, + sector_info.partition_index, + Some(sector_power.clone()), + ); + let balances = miner_balance(v, &miner_info.miner_id); + assert!(balances.initial_pledge.is_positive()); + let p_st: PowerState = get_state(v, &STORAGE_POWER_ACTOR_ADDR).unwrap(); + assert_eq!(sector_power.raw, p_st.total_bytes_committed); + + assert_invariants(v); +} + +struct SectorInfo { + number: SectorNumber, + deadline_info: DeadlineInfo, + partition_index: u64, +} + +#[derive(Clone)] +struct MinerInfo { + seal_proof: RegisteredSealProof, + _owner: Address, + worker: Address, + miner_id: Address, + _miner_robust: Address, +} + +fn setup(store: &'_ MemoryBlockstore) -> (TestVM, MinerInfo, SectorInfo) { + let v = TestVM::::new_with_singletons(store); + let addrs = create_accounts(&v, 1, &TokenAmount::from_whole(10_000)); + let seal_proof = RegisteredSealProof::StackedDRG32GiBV1P1; + let (owner, worker) = (addrs[0], addrs[0]); + let (id_addr, robust_addr) = create_miner( + &v, + &owner, + &worker, + seal_proof.registered_window_post_proof().unwrap(), + &TokenAmount::from_whole(10_000), + ); + v.set_epoch(200); + + // precommit and advance to prove commit time + let sector_number: SectorNumber = 100; + precommit_sectors(&v, 1, 1, &worker, &id_addr, seal_proof, sector_number, true, None); + + let balances = miner_balance(&v, &id_addr); + assert!(balances.pre_commit_deposit.is_positive()); + + let prove_time = v.epoch() + Policy::default().pre_commit_challenge_delay + 1; + advance_by_deadline_to_epoch(&v, &id_addr, prove_time); + + // prove commit, cron, advance to post time + let prove_params = ProveCommitSectorParams { sector_number, proof: vec![] }; + let prove_params_ser = IpldBlock::serialize_cbor(&prove_params).unwrap(); + apply_ok( + &v, + &worker, + &robust_addr, + &TokenAmount::zero(), + MinerMethod::ProveCommitSector as u64, + Some(prove_params), + ); + ExpectInvocation { + from: worker, + to: id_addr, + method: MinerMethod::ProveCommitSector as u64, + params: Some(prove_params_ser), + subinvocs: Some(vec![Expect::power_submit_porep(id_addr)]), + ..Default::default() + } + .matches(v.take_invocations().last().unwrap()); + let res = v + .execute_message( + &SYSTEM_ACTOR_ADDR, + &CRON_ACTOR_ADDR, + &TokenAmount::zero(), + CronMethod::EpochTick as u64, + None, + ) + .unwrap(); + assert_eq!(ExitCode::OK, res.code); + ExpectInvocation { + to: CRON_ACTOR_ADDR, + method: CronMethod::EpochTick as u64, + subinvocs: Some(vec![ + ExpectInvocation { + from: CRON_ACTOR_ADDR, + to: STORAGE_POWER_ACTOR_ADDR, + method: PowerMethod::OnEpochTickEnd as u64, + subinvocs: Some(vec![ + Expect::reward_this_epoch(STORAGE_POWER_ACTOR_ADDR), + ExpectInvocation { + from: STORAGE_POWER_ACTOR_ADDR, + to: id_addr, + method: MinerMethod::ConfirmSectorProofsValid as u64, + subinvocs: Some(vec![Expect::power_update_pledge(id_addr, None)]), + ..Default::default() + }, + Expect::reward_update_kpi(), + ]), + ..Default::default() + }, + ExpectInvocation { + from: CRON_ACTOR_ADDR, + to: STORAGE_MARKET_ACTOR_ADDR, + method: MarketMethod::CronTick as u64, + ..Default::default() + }, + ]), + ..Default::default() + } + .matches(v.take_invocations().last().unwrap()); + // pcd is released ip is added + let balances = miner_balance(&v, &id_addr); + assert!(balances.initial_pledge.is_positive()); + assert!(balances.pre_commit_deposit.is_zero()); + + // power unproven so network stats are the same + + let network_stats = get_network_stats(&v); + assert!(network_stats.total_bytes_committed.is_zero()); + assert!(network_stats.total_pledge_collateral.is_positive()); + + let (deadline_info, partition_index) = advance_to_proving_deadline(&v, &id_addr, sector_number); + ( + v, + MinerInfo { + seal_proof, + worker, + _owner: owner, + miner_id: id_addr, + _miner_robust: robust_addr, + }, + SectorInfo { number: sector_number, deadline_info, partition_index }, + ) +} From d1fdb0fcc7f19ce08d2b0f14d70ae1d0317b1e2f Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 18 Jul 2023 22:13:32 +0800 Subject: [PATCH 17/54] partial fix for review --- actors/miner/src/lib.rs | 6 ++--- actors/miner/tests/move_partitions_test.rs | 26 ++++++---------------- actors/miner/tests/util.rs | 2 +- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 275e7a5fd..2d1518cb1 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3018,7 +3018,7 @@ impl Actor { rt.transaction(|state: &mut State, rt| { let info = get_miner_info(rt.store(), state)?; - rt.validate_immediate_caller_is(&[info.worker, info.owner])?; + rt.validate_immediate_caller_is(info.control_addresses.iter().chain(&[info.worker, info.owner]))?; let store = rt.store(); let current_deadline = state.deadline_info(policy, rt.curr_epoch()); @@ -3112,8 +3112,6 @@ impl Actor { } // Load sector infos for proof, substituting a known-good sector for known-faulty sectors. - // Note: this is slightly sub-optimal, loading info for the recovering sectors again after they were already - // loaded above. let sector_infos = sectors .load_for_proof( &BitField::union(&all_sectors), @@ -3155,7 +3153,7 @@ impl Actor { })?; } - if !deadline_available_for_compaction( + if !deadline_is_mutable( policy, current_deadline.period_start, params.to_deadline, diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index 6500ba4f8..f1347f739 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -1,6 +1,6 @@ use fil_actor_miner::{ deadline_available_for_compaction, deadline_available_for_optimistic_post_dispute, - new_deadline_info, DeadlineInfo, + deadline_is_mutable, new_deadline_info, DeadlineInfo, }; use fil_actors_runtime::{ runtime::{DomainSeparationTag, RuntimePolicy}, @@ -76,34 +76,22 @@ fn farthest_possible_to_deadline( if current_deadline.index < from_deadline_id { // the deadline distance can only be nearer for i in (current_deadline.index..(from_deadline_id)).rev() { - if deadline_available_for_compaction( - &rt.policy, - current_deadline.period_start, - i, - *rt.epoch.borrow(), - ) { + if deadline_is_mutable(&rt.policy, current_deadline.period_start, i, *rt.epoch.borrow()) + { return i; } } } else { for i in (0..(from_deadline_id)).rev() { - if deadline_available_for_compaction( - &rt.policy, - current_deadline.period_start, - i, - *rt.epoch.borrow(), - ) { + if deadline_is_mutable(&rt.policy, current_deadline.period_start, i, *rt.epoch.borrow()) + { return i; } } for i in (current_deadline.index..rt.policy.wpost_period_deadlines).rev() { - if deadline_available_for_compaction( - &rt.policy, - current_deadline.period_start, - i, - *rt.epoch.borrow(), - ) { + if deadline_is_mutable(&rt.policy, current_deadline.period_start, i, *rt.epoch.borrow()) + { return i; } } diff --git a/actors/miner/tests/util.rs b/actors/miner/tests/util.rs index 2d20d815a..0c08a3108 100644 --- a/actors/miner/tests/util.rs +++ b/actors/miner/tests/util.rs @@ -2607,7 +2607,7 @@ impl ActorHarness { let params = MovePartitionsParams { from_deadline, to_deadline, partitions }; - rt.expect_validate_caller_addr(vec![self.worker, self.owner]); + rt.expect_validate_caller_addr(self.caller_addrs()); rt.set_caller(*ACCOUNT_ACTOR_CODE_ID, self.worker); rt.call::( From 0353a28e55415e3c29259e24a2301a8b67548cf8 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 09:13:50 +0800 Subject: [PATCH 18/54] adjust test --- test_vm/tests/move_partitions_test.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs index 38b4a419f..2d75a850c 100644 --- a/test_vm/tests/move_partitions_test.rs +++ b/test_vm/tests/move_partitions_test.rs @@ -1,5 +1,5 @@ use fvm_ipld_bitfield::BitField; -use fvm_ipld_blockstore::{Blockstore, MemoryBlockstore}; +use fvm_ipld_blockstore::MemoryBlockstore; use fvm_ipld_encoding::ipld_block::IpldBlock; use fvm_shared::address::Address; use fvm_shared::bigint::Zero; @@ -24,7 +24,7 @@ use test_vm::trace::ExpectInvocation; use test_vm::util::{ advance_by_deadline_to_epoch, advance_to_proving_deadline, apply_ok, assert_invariants, create_accounts, create_miner, cron_tick, get_network_stats, get_state, miner_balance, - precommit_sectors, submit_windowed_post, + precommit_sectors, submit_windowed_post, DynBlockstore, }; use test_vm::{TestVM, VM}; @@ -64,14 +64,11 @@ fn move_partitions_success() { assert_invariants(&v); } -fn submit_post_succeeds_test( - v: &dyn VM, - miner_info: MinerInfo, - sector_info: SectorInfo, -) { +fn submit_post_succeeds_test(v: &dyn VM, miner_info: MinerInfo, sector_info: SectorInfo) { // submit post let st: MinerState = get_state(v, &miner_info.miner_id).unwrap(); - let sector = st.get_sector(*v.blockstore(), sector_info.number).unwrap().unwrap(); + let sector = + st.get_sector(&DynBlockstore::wrap(v.blockstore()), sector_info.number).unwrap().unwrap(); let sector_power = power_for_sector(miner_info.seal_proof.sector_size().unwrap(), §or); submit_windowed_post( v, From bc9e6a1c76a3257381d6cdd3863d588f86a0c12c Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 10:32:03 +0800 Subject: [PATCH 19/54] use .context_code --- actors/miner/src/lib.rs | 117 ++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 78 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 2d1518cb1..6094a2990 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3023,7 +3023,7 @@ impl Actor { let store = rt.store(); let current_deadline = state.deadline_info(policy, rt.curr_epoch()); let mut deadlines = - state.load_deadlines(store).map_err(|e| e.wrap("failed to load deadlines"))?; + state.load_deadlines(store).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadlines")?; // only try to do synchronous Window Post verification if from_deadline doesn't satisfy deadline_available_for_compaction if !deadline_available_for_compaction( @@ -3050,34 +3050,26 @@ impl Actor { let dl_from = deadlines .load_deadline(policy, rt.store(), params.from_deadline) - .map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline") - })?; + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline")?; let proofs_snapshot = - dl_from.optimistic_proofs_snapshot_amt(store).map_err(|e| { - e.downcast_default( + dl_from.optimistic_proofs_snapshot_amt(store).context_code( ExitCode::USR_ILLEGAL_STATE, "failed to load proofs snapshot", - ) - })?; + )?; let partitions_snapshot = - dl_from.partitions_snapshot_amt(store).map_err(|e| { - e.downcast_default( + dl_from.partitions_snapshot_amt(store).context_code( ExitCode::USR_ILLEGAL_STATE, "failed to load partitions snapshot", - ) - })?; + )?; // Load sectors for the dispute. let sectors = - Sectors::load(rt.store(), &dl_from.sectors_snapshot).map_err(|e| { - e.downcast_default( + Sectors::load(rt.store(), &dl_from.sectors_snapshot).context_code( ExitCode::USR_ILLEGAL_STATE, "failed to load sectors array", - ) - })?; + )?; proofs_snapshot .for_each(|_, window_proof| { @@ -3089,12 +3081,10 @@ impl Actor { for partition_idx in window_proof.partitions.iter() { let partition = partitions_snapshot .get(partition_idx) - .map_err(|e| { - e.downcast_default( + .context_code( ExitCode::USR_ILLEGAL_STATE, - "failed to load partitions snapshot for proof", - ) - })? + "failed to load partitions snapshot for proof", + )? .ok_or_else(|| { actor_error!( illegal_state, @@ -3105,7 +3095,7 @@ impl Actor { all_ignored.push(partition.terminated.clone()); // fail early since remove_partitions will fail when there're faults anyway. if !partition.faults.is_empty() { - return Err(anyhow::anyhow!("unable to do synchronous Window POST verification while there're faults in from deadline {}", + return actor_error!(forbidden,format!("unable to do synchronous Window POST verification while there're faults in from deadline {}", params.from_deadline )); } @@ -3117,12 +3107,10 @@ impl Actor { &BitField::union(&all_sectors), &BitField::union(&all_ignored), ) - .map_err(|e| { - e.downcast_default( + .context_code( ExitCode::USR_ILLEGAL_STATE, "failed to load sectors for post verification", - ) - })?; + )?; // Find the proving period start for the deadline in question. let mut pp_start = current_deadline.period_start; @@ -3138,7 +3126,7 @@ impl Actor { §or_infos, window_proof.proofs.clone(), ) - .map_err(|e| e.wrap("window post failed"))? + .context_code(ExitCode::USR_ILLEGAL_STATE, "window post failed")? { return Err(actor_error!( illegal_argument, @@ -3148,9 +3136,7 @@ impl Actor { } Ok(()) }) - .map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "while removing partitions") - })?; + .context_code(ExitCode::USR_ILLEGAL_STATE, "while removing partitions")?; } if !deadline_is_mutable( @@ -3185,28 +3171,22 @@ impl Actor { let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); let mut from_deadline = - deadlines.load_deadline(policy, store, params.from_deadline).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + deadlines.load_deadline(policy, store, params.from_deadline) + .context_code(ExitCode::USR_ILLEGAL_STATE, format!("failed to load deadline {}", params.from_deadline), - ) - })?; + )?; let mut to_deadline = - deadlines.load_deadline(policy, store, params.to_deadline).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + deadlines.load_deadline(policy, store, params.to_deadline) + .context_code(ExitCode::USR_ILLEGAL_STATE, format!("failed to load deadline {}", params.to_deadline), - ) - })?; + )?; let partitions = &mut params.partitions; if partitions.is_empty() { - let partitions_amt = from_deadline.partitions_amt(rt.store()).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + let partitions_amt = from_deadline.partitions_amt(rt.store()) + .context_code(ExitCode::USR_ILLEGAL_STATE, format!("failed to load partitions for deadline {}", params.from_deadline), - ) - })?; + )?; for partition_idx in 0..partitions_amt.count() { partitions.set(partition_idx); @@ -3214,23 +3194,17 @@ impl Actor { } let (live, dead, removed_power) = - from_deadline.remove_partitions(store, partitions, from_quant).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + from_deadline.remove_partitions(store, partitions, from_quant) + .context_code(ExitCode::USR_ILLEGAL_STATE, format!( "failed to remove partitions from deadline {}", params.from_deadline ), - ) - })?; + )?; - state.delete_sectors(store, &dead).map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors") - })?; + state.delete_sectors(store, &dead).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors")?; - let sectors = state.load_sector_infos(store, &live).map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors") - })?; + let sectors = state.load_sector_infos(store, &live).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors")?; let proven = true; let added_power = to_deadline .add_sectors( @@ -3241,12 +3215,9 @@ impl Actor { info.sector_size, to_quant, ) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to add back moved sectors", - ) - })?; + )?; if removed_power != added_power { return Err(actor_error!( @@ -3259,30 +3230,20 @@ impl Actor { deadlines .update_deadline(policy, store, params.from_deadline, &from_deadline) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + .context_code(ExitCode::USR_ILLEGAL_STATE, format!("failed to update deadline {}", params.from_deadline), - ) - })?; - deadlines.update_deadline(policy, store, params.to_deadline, &to_deadline).map_err( - |e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + )?; + deadlines.update_deadline(policy, store, params.to_deadline, &to_deadline) + .context_code(ExitCode::USR_ILLEGAL_STATE, format!("failed to update deadline {}", params.to_deadline), - ) - }, - )?; + )?; - state.save_deadlines(store, deadlines).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, + state.save_deadlines(store, deadlines).context_code(ExitCode::USR_ILLEGAL_STATE, format!( "failed to save deadline when move_partitions from {} to {}", params.from_deadline, params.to_deadline ), - ) - })?; + )?; Ok(()) })?; From c1d88a29b144779693b2026ed6508cf25ee50c13 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 10:38:17 +0800 Subject: [PATCH 20/54] fix for test --- actors/miner/src/lib.rs | 2 +- actors/miner/tests/move_partitions_test.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 6094a2990..1d8c18382 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3095,7 +3095,7 @@ impl Actor { all_ignored.push(partition.terminated.clone()); // fail early since remove_partitions will fail when there're faults anyway. if !partition.faults.is_empty() { - return actor_error!(forbidden,format!("unable to do synchronous Window POST verification while there're faults in from deadline {}", + return Err(anyhow::anyhow!("unable to do synchronous Window POST verification while there're faults in from deadline {}", params.from_deadline )); } diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index f1347f739..30705adb7 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -211,7 +211,7 @@ fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { }, ); expect_abort_contains_message( - ExitCode::USR_ILLEGAL_ARGUMENT, + ExitCode::USR_ILLEGAL_STATE, "cannot remove partition 0: has faults", result, ); From 37d7d038d4e50fc56138c93eda0254ffa2122f63 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 10:53:19 +0800 Subject: [PATCH 21/54] disallow empty partitions --- actors/miner/src/lib.rs | 9 +-------- actors/miner/tests/move_partitions_test.rs | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 1d8c18382..e9adc0357 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3183,14 +3183,7 @@ impl Actor { let partitions = &mut params.partitions; if partitions.is_empty() { - let partitions_amt = from_deadline.partitions_amt(rt.store()) - .context_code(ExitCode::USR_ILLEGAL_STATE, - format!("failed to load partitions for deadline {}", params.from_deadline), - )?; - - for partition_idx in 0..partitions_amt.count() { - partitions.set(partition_idx); - } + return Err(actor_error!(illegal_argument,"empty partitions not allowed")); } let (live, dead, removed_power) = diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index 30705adb7..1f5472de8 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -134,7 +134,7 @@ fn fail_to_move_partitions_with_faults_from_safe_epoch() { || {}, ); expect_abort_contains_message( - ExitCode::USR_ILLEGAL_ARGUMENT, + ExitCode::USR_ILLEGAL_STATE, "cannot remove partition 0: has faults", result, ); From 89bcb749947eabd2afe2102551fc3f9b94c81bdf Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 13:38:52 +0800 Subject: [PATCH 22/54] refactor deadline_available_for_move --- actors/miner/src/deadlines.rs | 31 ++++++++++++-- actors/miner/src/lib.rs | 59 ++++++--------------------- test_vm/tests/move_partitions_test.rs | 12 +++--- 3 files changed, 47 insertions(+), 55 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 3346d8000..1d01e63ad 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -142,10 +142,33 @@ pub fn deadline_available_for_move( policy: &Policy, from_deadline: u64, to_deadline: u64, - current_deadline: u64, -) -> bool { - deadline_distance(policy, current_deadline, to_deadline) - < deadline_distance(policy, current_deadline, from_deadline) + current_deadline: &DeadlineInfo, +) -> Result<(), String> { + if !deadline_is_mutable( + policy, + current_deadline.period_start, + from_deadline, + current_deadline.current_epoch, + ) { + return Err(format!("cannot move from a deadline when it's not mutable")); + } + + if !deadline_is_mutable( + policy, + current_deadline.period_start, + to_deadline, + current_deadline.current_epoch, + ) { + return Err(format!("cannot move to a deadline when it's not mutable")); + } + + if deadline_distance(policy, current_deadline.index, to_deadline) + >= deadline_distance(policy, current_deadline.index, from_deadline) + { + return Err("can only move to a deadline which is nearer from current deadline".into()); + } + + Ok(()) } // Determine current period start and deadline index directly from current epoch and diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index e9adc0357..1ee3bf0df 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3025,29 +3025,23 @@ impl Actor { let mut deadlines = state.load_deadlines(store).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadlines")?; - // only try to do synchronous Window Post verification if from_deadline doesn't satisfy deadline_available_for_compaction - if !deadline_available_for_compaction( + deadline_available_for_move( + policy, + params.from_deadline, + params.to_deadline, + ¤t_deadline, + ).context_code( + ExitCode::USR_ILLEGAL_ARGUMENT, + "conditions not satisfied for deadline_available_for_move" + )?; + + // only try to do synchronous Window Post verification if deadline_available_for_optimistic_post_dispute + if deadline_available_for_optimistic_post_dispute( policy, current_deadline.period_start, params.from_deadline, rt.curr_epoch(), ) { - // the heavy path: try to do synchronous Window Post verification - // current deadline must be in the dispute window to satisfy the condition of synchronous Window POST verification - if !deadline_available_for_optimistic_post_dispute( - policy, - current_deadline.period_start, - params.from_deadline, - rt.curr_epoch(), - ) { - return Err(actor_error!( - forbidden, - "cannot move from deadline {} when !deadline_available_for_compaction && !deadline_available_for_optimistic_post_dispute", - params.from_deadline - )); - } - - let dl_from = deadlines .load_deadline(policy, rt.store(), params.from_deadline) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline")?; @@ -3139,33 +3133,6 @@ impl Actor { .context_code(ExitCode::USR_ILLEGAL_STATE, "while removing partitions")?; } - if !deadline_is_mutable( - policy, - current_deadline.period_start, - params.to_deadline, - rt.curr_epoch(), - ) { - return Err(actor_error!( - forbidden, - "cannot move to deadline {} during its challenge window, \ - or the prior challenge window, - or before {} epochs have passed since its last challenge window ended", - params.to_deadline, - policy.wpost_dispute_window - )); - } - - if !deadline_available_for_move( - policy, - params.from_deadline, - params.to_deadline, - current_deadline.index, - ) { - return Err(actor_error!( - forbidden, - "cannot move to a deadline which is further from current deadline" - )); - } let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); @@ -3183,7 +3150,7 @@ impl Actor { let partitions = &mut params.partitions; if partitions.is_empty() { - return Err(actor_error!(illegal_argument,"empty partitions not allowed")); + return Err(actor_error!(illegal_argument, "empty partitions not allowed")); } let (live, dead, removed_power) = diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs index 2d75a850c..4f6028a2f 100644 --- a/test_vm/tests/move_partitions_test.rs +++ b/test_vm/tests/move_partitions_test.rs @@ -1,4 +1,3 @@ -use fvm_ipld_bitfield::BitField; use fvm_ipld_blockstore::MemoryBlockstore; use fvm_ipld_encoding::ipld_block::IpldBlock; use fvm_shared::address::Address; @@ -23,8 +22,8 @@ use test_vm::expects::Expect; use test_vm::trace::ExpectInvocation; use test_vm::util::{ advance_by_deadline_to_epoch, advance_to_proving_deadline, apply_ok, assert_invariants, - create_accounts, create_miner, cron_tick, get_network_stats, get_state, miner_balance, - precommit_sectors, submit_windowed_post, DynBlockstore, + create_accounts, create_miner, cron_tick, get_network_stats, get_state, make_bitfield, + miner_balance, precommit_sectors, submit_windowed_post, DynBlockstore, }; use test_vm::{TestVM, VM}; @@ -38,8 +37,11 @@ fn move_partitions_success() { let prove_time = v.epoch() + Policy::default().wpost_dispute_window; advance_by_deadline_to_epoch(&v, &miner.miner_id, prove_time); - let move_params = - MovePartitionsParams { from_deadline: 0, to_deadline: 47, partitions: BitField::new() }; + let move_params = MovePartitionsParams { + from_deadline: 0, + to_deadline: 47, + partitions: make_bitfield(&[0u64]), + }; let prove_params_ser = IpldBlock::serialize_cbor(&move_params).unwrap(); apply_ok( &v, From 79ab15d9c481270013f675e9b92f67a41d05abfe Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 13:45:21 +0800 Subject: [PATCH 23/54] fix for clippy --- actors/miner/src/deadlines.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 1d01e63ad..b1e1ffe9a 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -150,7 +150,7 @@ pub fn deadline_available_for_move( from_deadline, current_deadline.current_epoch, ) { - return Err(format!("cannot move from a deadline when it's not mutable")); + return Err("cannot move from a deadline when it's not mutable".to_string()); } if !deadline_is_mutable( @@ -159,13 +159,13 @@ pub fn deadline_available_for_move( to_deadline, current_deadline.current_epoch, ) { - return Err(format!("cannot move to a deadline when it's not mutable")); + return Err("cannot move to a deadline when it's not mutable".to_string()); } if deadline_distance(policy, current_deadline.index, to_deadline) >= deadline_distance(policy, current_deadline.index, from_deadline) { - return Err("can only move to a deadline which is nearer from current deadline".into()); + return Err("can only move to a deadline which is nearer from current deadline".to_string()); } Ok(()) From 39d644719715e6ff1dcc8043f9de25e96fa5cf28 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 19 Jul 2023 13:54:01 +0800 Subject: [PATCH 24/54] minor opt --- actors/miner/src/lib.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 1ee3bf0df..3695be3f3 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2996,10 +2996,7 @@ impl Actor { Ok(()) } - fn move_partitions( - rt: &impl Runtime, - mut params: MovePartitionsParams, - ) -> Result<(), ActorError> { + fn move_partitions(rt: &impl Runtime, params: MovePartitionsParams) -> Result<(), ActorError> { if params.from_deadline == params.to_deadline { return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); } @@ -3014,6 +3011,9 @@ impl Actor { params.to_deadline )); } + if params.partitions.is_empty() { + return Err(actor_error!(illegal_argument, "empty partitions not allowed")); + } rt.transaction(|state: &mut State, rt| { let info = get_miner_info(rt.store(), state)?; @@ -3148,13 +3148,9 @@ impl Actor { format!("failed to load deadline {}", params.to_deadline), )?; - let partitions = &mut params.partitions; - if partitions.is_empty() { - return Err(actor_error!(illegal_argument, "empty partitions not allowed")); - } let (live, dead, removed_power) = - from_deadline.remove_partitions(store, partitions, from_quant) + from_deadline.remove_partitions(store, ¶ms.partitions, from_quant) .context_code(ExitCode::USR_ILLEGAL_STATE, format!( "failed to remove partitions from deadline {}", From 0bd0611a5fdf5284be3ae987cc79865b6c8e2124 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Thu, 20 Jul 2023 13:24:54 +0800 Subject: [PATCH 25/54] only verify_windowed_post once --- actors/miner/src/deadlines.rs | 15 +++ actors/miner/src/lib.rs | 213 +++++++++++++++++----------------- 2 files changed, 124 insertions(+), 104 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index b1e1ffe9a..9970957db 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -171,6 +171,21 @@ pub fn deadline_available_for_move( Ok(()) } +// returns the nearest deadline info with index `target_deadline` that has already occured from the point of view of the current deadline(including the current deadline). +pub fn nearest_occured_deadline_info( + policy: &Policy, + current_deadline: &DeadlineInfo, + target_deadline: u64, +) -> DeadlineInfo { + // Find the proving period start for the deadline in question. + let mut pp_start = current_deadline.period_start; + if current_deadline.index < target_deadline { + pp_start -= policy.wpost_proving_period + } + + new_deadline_info(policy, pp_start, target_deadline, current_deadline.current_epoch) +} + // Determine current period start and deadline index directly from current epoch and // the offset implied by the proving period. This works correctly even for the state // of a miner actor without an active deadline cron diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 3695be3f3..f340c5c84 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -1536,12 +1536,8 @@ impl Actor { // --- check proof --- // Find the proving period start for the deadline in question. - let mut pp_start = dl_info.period_start; - if dl_info.index < params.deadline { - pp_start -= policy.wpost_proving_period - } let target_deadline = - new_deadline_info(policy, pp_start, params.deadline, current_epoch); + nearest_occured_deadline_info(policy, &dl_info, params.deadline); // Load the target deadline let mut deadlines_current = st .load_deadlines(rt.store()) @@ -3018,66 +3014,80 @@ impl Actor { rt.transaction(|state: &mut State, rt| { let info = get_miner_info(rt.store(), state)?; - rt.validate_immediate_caller_is(info.control_addresses.iter().chain(&[info.worker, info.owner]))?; + rt.validate_immediate_caller_is( + info.control_addresses.iter().chain(&[info.worker, info.owner]), + )?; let store = rt.store(); let current_deadline = state.deadline_info(policy, rt.curr_epoch()); - let mut deadlines = - state.load_deadlines(store).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadlines")?; + let mut deadlines = state + .load_deadlines(store) + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadlines")?; deadline_available_for_move( policy, params.from_deadline, params.to_deadline, ¤t_deadline, - ).context_code( + ) + .context_code( ExitCode::USR_ILLEGAL_ARGUMENT, - "conditions not satisfied for deadline_available_for_move" - )?; + "conditions not satisfied for deadline_available_for_move", + )?; - // only try to do synchronous Window Post verification if deadline_available_for_optimistic_post_dispute + let mut from_deadline = + deadlines.load_deadline(policy, store, params.from_deadline).context_code( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load deadline {}", params.from_deadline), + )?; + // only try to do synchronous Window Post verification if the from deadline is in dispute window + // note that as window post is batched, the best we can do is to verify only those that contains at least one partition being moved. + // besides, after verification, the corresponding PostProof is deleted, leaving other PostProof intact. + // thus it's necessary that for those moved partitions, there's an empty partition left in place to keep the partitions from re-indexing. if deadline_available_for_optimistic_post_dispute( policy, current_deadline.period_start, params.from_deadline, rt.curr_epoch(), ) { - let dl_from = deadlines - .load_deadline(policy, rt.store(), params.from_deadline) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadline")?; - - let proofs_snapshot = - dl_from.optimistic_proofs_snapshot_amt(store).context_code( - ExitCode::USR_ILLEGAL_STATE, - "failed to load proofs snapshot", - )?; + let mut proofs_snapshot = from_deadline + .optimistic_proofs_snapshot_amt(store) + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load proofs snapshot")?; let partitions_snapshot = - dl_from.partitions_snapshot_amt(store).context_code( - ExitCode::USR_ILLEGAL_STATE, - "failed to load partitions snapshot", - )?; + from_deadline.partitions_snapshot_amt(store).context_code( + ExitCode::USR_ILLEGAL_STATE, + "failed to load partitions snapshot", + )?; // Load sectors for the dispute. - let sectors = - Sectors::load(rt.store(), &dl_from.sectors_snapshot).context_code( - ExitCode::USR_ILLEGAL_STATE, - "failed to load sectors array", - )?; - + let sectors = Sectors::load(rt.store(), &from_deadline.sectors_snapshot) + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load sectors array")?; + + let mut verified_index = vec![]; + let mut all_window_proofs = + Vec::::with_capacity(params.partitions.len() as usize); + let mut all_sector_infos = + Vec::::with_capacity(params.partitions.len() as usize); proofs_snapshot - .for_each(|_, window_proof| { + .for_each(|index, window_proof| { + if !params.partitions.contains_any(&window_proof.partitions) { + return Ok(()); + } + verified_index.push(index); + all_window_proofs.extend(window_proof.proofs.clone()); + let mut all_sectors = - Vec::::with_capacity(window_proof.partitions.len() as usize); + Vec::::with_capacity(params.partitions.len() as usize); let mut all_ignored = - Vec::::with_capacity(window_proof.partitions.len() as usize); + Vec::::with_capacity(params.partitions.len() as usize); for partition_idx in window_proof.partitions.iter() { let partition = partitions_snapshot .get(partition_idx) .context_code( - ExitCode::USR_ILLEGAL_STATE, - "failed to load partitions snapshot for proof", + ExitCode::USR_ILLEGAL_STATE, + "failed to load partitions snapshot for proof", )? .ok_or_else(|| { actor_error!( @@ -3086,13 +3096,9 @@ impl Actor { ) })?; all_sectors.push(partition.sectors.clone()); + all_ignored.push(partition.faults.clone()); all_ignored.push(partition.terminated.clone()); - // fail early since remove_partitions will fail when there're faults anyway. - if !partition.faults.is_empty() { - return Err(anyhow::anyhow!("unable to do synchronous Window POST verification while there're faults in from deadline {}", - params.from_deadline - )); - } + all_ignored.push(partition.unproven.clone()); } // Load sector infos for proof, substituting a known-good sector for known-faulty sectors. @@ -3102,65 +3108,62 @@ impl Actor { &BitField::union(&all_ignored), ) .context_code( - ExitCode::USR_ILLEGAL_STATE, - "failed to load sectors for post verification", - )?; + ExitCode::USR_ILLEGAL_STATE, + "failed to load sectors for post verification", + )?; + all_sector_infos.extend(sector_infos); - // Find the proving period start for the deadline in question. - let mut pp_start = current_deadline.period_start; - if current_deadline.index < params.from_deadline { - pp_start -= policy.wpost_proving_period - } - let target_deadline = - new_deadline_info(policy, pp_start, params.from_deadline, rt.curr_epoch()); - - if !verify_windowed_post( - rt, - target_deadline.challenge, - §or_infos, - window_proof.proofs.clone(), - ) - .context_code(ExitCode::USR_ILLEGAL_STATE, "window post failed")? - { - return Err(actor_error!( - illegal_argument, - "invalid post was submitted" - ) - .into()); - } Ok(()) }) .context_code(ExitCode::USR_ILLEGAL_STATE, "while removing partitions")?; - } + // Find the proving period start for the deadline in question. + let target_deadline = + nearest_occured_deadline_info(policy, ¤t_deadline, params.from_deadline); + + if !verify_windowed_post( + rt, + target_deadline.challenge, + &all_sector_infos, + all_window_proofs, + ) + .context_code(ExitCode::USR_ILLEGAL_STATE, "window post failed")? + { + return Err(actor_error!(illegal_argument, "invalid post was submitted")); + } + + proofs_snapshot.batch_delete(verified_index, true).context_code( + ExitCode::USR_ILLEGAL_STATE, + "while batch deletes verified post index", + )?; + from_deadline.optimistic_post_submissions_snapshot = proofs_snapshot + .flush() + .context_code(ExitCode::USR_ILLEGAL_STATE, "while flushing proofs_snapshot")?; + } let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); - let mut from_deadline = - deadlines.load_deadline(policy, store, params.from_deadline) - .context_code(ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.from_deadline), - )?; let mut to_deadline = - deadlines.load_deadline(policy, store, params.to_deadline) - .context_code(ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.to_deadline), - )?; - + deadlines.load_deadline(policy, store, params.to_deadline).context_code( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to load deadline {}", params.to_deadline), + )?; - let (live, dead, removed_power) = - from_deadline.remove_partitions(store, ¶ms.partitions, from_quant) - .context_code(ExitCode::USR_ILLEGAL_STATE, - format!( - "failed to remove partitions from deadline {}", - params.from_deadline - ), - )?; + let (live, dead, removed_power) = from_deadline + .remove_partitions(store, ¶ms.partitions, from_quant) + .context_code( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to remove partitions from deadline {}", params.from_deadline), + )?; - state.delete_sectors(store, &dead).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors")?; + state + .delete_sectors(store, &dead) + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors")?; - let sectors = state.load_sector_infos(store, &live).context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors")?; + let sectors = state + .load_sector_infos(store, &live) + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors")?; let proven = true; let added_power = to_deadline .add_sectors( @@ -3171,9 +3174,7 @@ impl Actor { info.sector_size, to_quant, ) - .context_code(ExitCode::USR_ILLEGAL_STATE, - "failed to add back moved sectors", - )?; + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to add back moved sectors")?; if removed_power != added_power { return Err(actor_error!( @@ -3186,20 +3187,24 @@ impl Actor { deadlines .update_deadline(policy, store, params.from_deadline, &from_deadline) - .context_code(ExitCode::USR_ILLEGAL_STATE, - format!("failed to update deadline {}", params.from_deadline), - )?; - deadlines.update_deadline(policy, store, params.to_deadline, &to_deadline) - .context_code(ExitCode::USR_ILLEGAL_STATE, - format!("failed to update deadline {}", params.to_deadline), - )?; - - state.save_deadlines(store, deadlines).context_code(ExitCode::USR_ILLEGAL_STATE, - format!( - "failed to save deadline when move_partitions from {} to {}", - params.from_deadline, params.to_deadline - ), + .context_code( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to update deadline {}", params.from_deadline), )?; + deadlines + .update_deadline(policy, store, params.to_deadline, &to_deadline) + .context_code( + ExitCode::USR_ILLEGAL_STATE, + format!("failed to update deadline {}", params.to_deadline), + )?; + + state.save_deadlines(store, deadlines).context_code( + ExitCode::USR_ILLEGAL_STATE, + format!( + "failed to save deadline when move_partitions from {} to {}", + params.from_deadline, params.to_deadline + ), + )?; Ok(()) })?; From c01b54c5302af3fdc539bc23bdfff72114e1cd07 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Mon, 24 Jul 2023 09:17:10 +0800 Subject: [PATCH 26/54] mod error msg --- actors/miner/src/deadlines.rs | 15 ++++++++++++--- actors/miner/src/lib.rs | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 9970957db..7c5cea955 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -150,7 +150,10 @@ pub fn deadline_available_for_move( from_deadline, current_deadline.current_epoch, ) { - return Err("cannot move from a deadline when it's not mutable".to_string()); + return Err(format!( + "cannot move from a deadline {}, immutable at epoch {}", + from_deadline, current_deadline.current_epoch + )); } if !deadline_is_mutable( @@ -159,13 +162,19 @@ pub fn deadline_available_for_move( to_deadline, current_deadline.current_epoch, ) { - return Err("cannot move to a deadline when it's not mutable".to_string()); + return Err(format!( + "cannot move to a deadline {}, immutable at epoch {}", + to_deadline, current_deadline.current_epoch + )); } if deadline_distance(policy, current_deadline.index, to_deadline) >= deadline_distance(policy, current_deadline.index, from_deadline) { - return Err("can only move to a deadline which is nearer from current deadline".to_string()); + return Err(format!( + "can only move to a deadline which is nearer from current deadline {}, to_deadline {} is not nearer than from_deadline {}", + current_deadline.index, to_deadline, from_deadline + )); } Ok(()) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index f340c5c84..65b318e58 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3031,7 +3031,7 @@ impl Actor { ¤t_deadline, ) .context_code( - ExitCode::USR_ILLEGAL_ARGUMENT, + ExitCode::USR_FORBIDDEN, "conditions not satisfied for deadline_available_for_move", )?; From ccdb45f9d368c4506847f43f8175bb6f2d8618d9 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 25 Jul 2023 13:52:44 +0800 Subject: [PATCH 27/54] 1. verify_window_post batch by batch 2. move partitions intact, the `expirations_epochs` and `early_terminated` may have a gap of 24H --- actors/miner/src/deadline_state.rs | 127 ++++++++++++++++++++- actors/miner/src/lib.rs | 107 +++++++---------- actors/miner/src/partition_state.rs | 4 +- actors/miner/src/testing.rs | 2 +- actors/miner/tests/move_partitions_test.rs | 4 +- 5 files changed, 171 insertions(+), 73 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 2525af756..4a884a133 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -23,7 +23,11 @@ use super::{ BitFieldQueue, ExpirationSet, Partition, PartitionSectorMap, PoStPartition, PowerPair, SectorOnChainInfo, Sectors, TerminationResult, }; -use crate::SECTORS_AMT_BITWIDTH; + +use crate::{ + PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, PARTITION_EXPIRATION_AMT_BITWIDTH, + SECTORS_AMT_BITWIDTH, +}; // Bitwidth of AMTs determined empirically from mutation patterns and projections of mainnet data. // Usually a small array @@ -99,6 +103,127 @@ impl Deadlines { self.due[deadline_idx as usize] = store.put_cbor(deadline, Code::Blake2b256)?; Ok(()) } + + pub fn move_partitions( + &mut self, + store: &BS, + from_deadline: &mut Deadline, + to_deadline: &mut Deadline, + to_quant: QuantSpec, + partitions: &BitField, + ) -> anyhow::Result<()> { + let mut from_partitions = from_deadline.partitions_amt(store)?; + let mut to_partitions = to_deadline.partitions_amt(store)?; + + // even though we're moving partitions intact, we still need to update from/to `Deadline` accordingly. + + let first_to_partition_idx = to_partitions.count(); + for (i, from_partition_idx) in partitions.iter().enumerate() { + let mut moving_partition = from_partitions + .get(from_partition_idx)? + .ok_or_else(|| actor_error!(not_found, "no partition {}", from_partition_idx))? + .clone(); + if !moving_partition.faults.is_empty() || !moving_partition.unproven.is_empty() { + return Err(actor_error!(forbidden, "partition with faults or unproven sectors are not allowed to move, partition_idx {}", from_partition_idx))?; + } + + let to_partition_idx = first_to_partition_idx + i as u64; + + let from_expirations_epochs: Array = + Array::load(&moving_partition.expirations_epochs, store)?; + + let from_early_terminations: Array = + Array::load(&moving_partition.early_terminated, store)?; + + let mut to_expirations_epochs = Array::::new_with_bit_width( + store, + PARTITION_EXPIRATION_AMT_BITWIDTH, + ); + let mut to_early_terminations = Array::::new_with_bit_width( + store, + PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, + ); + + // ajust epoch info of expirations_epochs and early_terminated within `Partition` + from_expirations_epochs.for_each(|from_epoch, expire_set| { + to_expirations_epochs.set( + to_quant.quantize_up(from_epoch as ChainEpoch) as u64, + expire_set.clone(), + )?; + return Ok(()); + })?; + from_early_terminations.for_each(|from_epoch, bitfield| { + to_early_terminations + .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; + return Ok(()); + })?; + moving_partition.expirations_epochs = to_expirations_epochs.flush()?; + moving_partition.early_terminated = to_early_terminations.flush()?; + + let all_sectors = moving_partition.sectors.len(); + let live_sectors = moving_partition.live_sectors().len(); + let early_terminations = from_deadline.early_terminations.get(from_partition_idx); + if early_terminations != (from_early_terminations.count() > 0) { + return Err(actor_error!(illegal_state, "Deadline.early_terminations doesn't agree with Partition.early_terminated for partition {}", from_partition_idx))?; + } + + // start updating from/to `Deadline` here + + from_deadline.total_sectors -= all_sectors; + from_deadline.live_sectors -= live_sectors; + from_deadline.faulty_power -= &moving_partition.faulty_power; + + to_deadline.total_sectors += all_sectors; + to_deadline.live_sectors += live_sectors; + to_deadline.faulty_power += &moving_partition.faulty_power; + + // update early_terminations BitField of `Deadline` + if early_terminations { + from_deadline.early_terminations.unset(from_partition_idx); + to_deadline.early_terminations.set(to_partition_idx); + } + + from_partitions.set(from_partition_idx, Partition::new(store)?)?; + to_partitions.set(to_partition_idx, moving_partition)?; + } + + // update expirations_epochs Cid of Deadline. + { + let mut epochs_to_remove = Vec::::new(); + let mut from_expirations_epochs: Array = + Array::load(&from_deadline.expirations_epochs, store)?; + let mut to_expirations_epochs: Array = + Array::load(&to_deadline.expirations_epochs, store)?; + from_expirations_epochs.for_each_mut(|from_epoch, from_bitfield| { + let to_epoch = to_quant.quantize_up(from_epoch as ChainEpoch); + let mut to_bitfield = + to_expirations_epochs.get(to_epoch as u64)?.cloned().unwrap_or_default(); + for (i, partition_id) in partitions.iter().enumerate() { + if from_bitfield.get(partition_id) { + from_bitfield.unset(partition_id); + to_bitfield.set(first_to_partition_idx + i as u64); + } + } + to_expirations_epochs.set(to_epoch as u64, to_bitfield)?; + + if from_bitfield.is_empty() { + epochs_to_remove.push(from_epoch); + } + + Ok(()) + })?; + if !epochs_to_remove.is_empty() { + from_expirations_epochs.batch_delete(epochs_to_remove, true)?; + } + from_deadline.expirations_epochs = from_expirations_epochs.flush()?; + to_deadline.expirations_epochs = to_expirations_epochs.flush()?; + } + + from_deadline.partitions = from_partitions.flush()?; + to_deadline.partitions = to_partitions.flush()?; + + Ok(()) + } } /// Deadline holds the state for all sectors due at a specific deadline. diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 65b318e58..e131dba70 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3050,7 +3050,7 @@ impl Actor { params.from_deadline, rt.curr_epoch(), ) { - let mut proofs_snapshot = from_deadline + let proofs_snapshot = from_deadline .optimistic_proofs_snapshot_amt(store) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load proofs snapshot")?; @@ -3060,22 +3060,19 @@ impl Actor { "failed to load partitions snapshot", )?; + // Find the proving period start for the deadline in question. + let target_deadline = + nearest_occured_deadline_info(policy, ¤t_deadline, params.from_deadline); + // Load sectors for the dispute. let sectors = Sectors::load(rt.store(), &from_deadline.sectors_snapshot) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load sectors array")?; - let mut verified_index = vec![]; - let mut all_window_proofs = - Vec::::with_capacity(params.partitions.len() as usize); - let mut all_sector_infos = - Vec::::with_capacity(params.partitions.len() as usize); proofs_snapshot - .for_each(|index, window_proof| { + .for_each(|_, window_proof| { if !params.partitions.contains_any(&window_proof.partitions) { return Ok(()); } - verified_index.push(index); - all_window_proofs.extend(window_proof.proofs.clone()); let mut all_sectors = Vec::::with_capacity(params.partitions.len() as usize); @@ -3095,10 +3092,22 @@ impl Actor { "failed to load partitions snapshot for proof" ) })?; + if !partition.faults.is_empty() { + return Err(actor_error!( + illegal_argument, + "cannot move partition {}: has faults", + partition_idx + ))?; + } + if !partition.unproven.is_empty() { + return Err(actor_error!( + illegal_argument, + "cannot move partition {}: has unproven", + partition_idx + ))?; + } all_sectors.push(partition.sectors.clone()); - all_ignored.push(partition.faults.clone()); all_ignored.push(partition.terminated.clone()); - all_ignored.push(partition.unproven.clone()); } // Load sector infos for proof, substituting a known-good sector for known-faulty sectors. @@ -3111,37 +3120,26 @@ impl Actor { ExitCode::USR_ILLEGAL_STATE, "failed to load sectors for post verification", )?; - all_sector_infos.extend(sector_infos); + + if !verify_windowed_post( + rt, + target_deadline.challenge, + §or_infos, + window_proof.proofs.clone(), + ) + .context_code(ExitCode::USR_ILLEGAL_STATE, "window post failed")? + { + return Err(actor_error!( + illegal_argument, + "invalid post was submitted" + ))?; + } Ok(()) }) .context_code(ExitCode::USR_ILLEGAL_STATE, "while removing partitions")?; - - // Find the proving period start for the deadline in question. - let target_deadline = - nearest_occured_deadline_info(policy, ¤t_deadline, params.from_deadline); - - if !verify_windowed_post( - rt, - target_deadline.challenge, - &all_sector_infos, - all_window_proofs, - ) - .context_code(ExitCode::USR_ILLEGAL_STATE, "window post failed")? - { - return Err(actor_error!(illegal_argument, "invalid post was submitted")); - } - - proofs_snapshot.batch_delete(verified_index, true).context_code( - ExitCode::USR_ILLEGAL_STATE, - "while batch deletes verified post index", - )?; - from_deadline.optimistic_post_submissions_snapshot = proofs_snapshot - .flush() - .context_code(ExitCode::USR_ILLEGAL_STATE, "while flushing proofs_snapshot")?; } - let from_quant = state.quant_spec_for_deadline(policy, params.from_deadline); let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); let mut to_deadline = @@ -3150,40 +3148,15 @@ impl Actor { format!("failed to load deadline {}", params.to_deadline), )?; - let (live, dead, removed_power) = from_deadline - .remove_partitions(store, ¶ms.partitions, from_quant) - .context_code( - ExitCode::USR_ILLEGAL_STATE, - format!("failed to remove partitions from deadline {}", params.from_deadline), - )?; - - state - .delete_sectors(store, &dead) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to delete dead sectors")?; - - let sectors = state - .load_sector_infos(store, &live) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load moved sectors")?; - let proven = true; - let added_power = to_deadline - .add_sectors( + deadlines + .move_partitions( store, - info.window_post_partition_sectors, - proven, - §ors, - info.sector_size, + &mut from_deadline, + &mut to_deadline, to_quant, + ¶ms.partitions, ) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to add back moved sectors")?; - - if removed_power != added_power { - return Err(actor_error!( - illegal_state, - "power changed when compacting partitions: was {:?}, is now {:?}", - removed_power, - added_power - )); - } + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to move partitions")?; deadlines .update_deadline(policy, store, params.from_deadline, &from_deadline) diff --git a/actors/miner/src/partition_state.rs b/actors/miner/src/partition_state.rs index 9ff3c3e72..d3ae5092c 100644 --- a/actors/miner/src/partition_state.rs +++ b/actors/miner/src/partition_state.rs @@ -24,8 +24,8 @@ use super::{ }; // Bitwidth of AMTs determined empirically from mutation patterns and projections of mainnet data. -const PARTITION_EXPIRATION_AMT_BITWIDTH: u32 = 4; -const PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH: u32 = 3; +pub const PARTITION_EXPIRATION_AMT_BITWIDTH: u32 = 4; +pub const PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH: u32 = 3; #[derive(Serialize_tuple, Deserialize_tuple, Clone)] pub struct Partition { diff --git a/actors/miner/src/testing.rs b/actors/miner/src/testing.rs index 79a0ece39..8d7d26a81 100644 --- a/actors/miner/src/testing.rs +++ b/actors/miner/src/testing.rs @@ -641,7 +641,7 @@ impl ExpirationQueueStateSummary { // check expiring sectors are still alive if let Some(sector) = live_sectors.get(§or_number) { let target = quant.quantize_up(sector.expiration); - acc.require(epoch == target, format!("invalid expiration {epoch} for sector {sector_number}, expected {target}")); + acc.require(epoch == target || target+quant.unit == epoch, format!("invalid expiration {epoch} for sector {sector_number}, expected {target}")); on_time_sectors_pledge += sector.initial_pledge.clone(); } else { acc.add(format!("on time expiration sector {sector_number} isn't live")); diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index 1f5472de8..af9a5bfb1 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -135,7 +135,7 @@ fn fail_to_move_partitions_with_faults_from_safe_epoch() { ); expect_abort_contains_message( ExitCode::USR_ILLEGAL_STATE, - "cannot remove partition 0: has faults", + "partition with faults or unproven sectors are not allowed to move", result, ); @@ -212,7 +212,7 @@ fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { ); expect_abort_contains_message( ExitCode::USR_ILLEGAL_STATE, - "cannot remove partition 0: has faults", + "partition with faults or unproven sectors are not allowed to move", result, ); From b775fa81f8a7fd25268e0e661e06d874e9a956c1 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 25 Jul 2023 14:02:15 +0800 Subject: [PATCH 28/54] fix ci --- actors/miner/src/deadline_state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 4a884a133..a61b08089 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -150,12 +150,12 @@ impl Deadlines { to_quant.quantize_up(from_epoch as ChainEpoch) as u64, expire_set.clone(), )?; - return Ok(()); + Ok(()) })?; from_early_terminations.for_each(|from_epoch, bitfield| { to_early_terminations .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; - return Ok(()); + Ok(()) })?; moving_partition.expirations_epochs = to_expirations_epochs.flush()?; moving_partition.early_terminated = to_early_terminations.flush()?; From a5290cc15f4a0eaa234ea7a0d4865e7174835886 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 25 Jul 2023 14:07:08 +0800 Subject: [PATCH 29/54] mod check for epoch --- actors/miner/src/testing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/miner/src/testing.rs b/actors/miner/src/testing.rs index 8d7d26a81..4cfa77920 100644 --- a/actors/miner/src/testing.rs +++ b/actors/miner/src/testing.rs @@ -641,7 +641,7 @@ impl ExpirationQueueStateSummary { // check expiring sectors are still alive if let Some(sector) = live_sectors.get(§or_number) { let target = quant.quantize_up(sector.expiration); - acc.require(epoch == target || target+quant.unit == epoch, format!("invalid expiration {epoch} for sector {sector_number}, expected {target}")); + acc.require(epoch >= target && (epoch-target)%quant.unit ==0 , format!("invalid expiration {epoch} for sector {sector_number}, expected {target}")); on_time_sectors_pledge += sector.initial_pledge.clone(); } else { acc.add(format!("on time expiration sector {sector_number} isn't live")); From bf135580bd4ac7eeac4064c2d897250ca4be50b8 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 2 Aug 2023 13:07:05 +0800 Subject: [PATCH 30/54] partial review fix --- actors/miner/src/deadline_state.rs | 1 - actors/miner/src/lib.rs | 21 ++++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index a61b08089..3ba8c4847 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -105,7 +105,6 @@ impl Deadlines { } pub fn move_partitions( - &mut self, store: &BS, from_deadline: &mut Deadline, to_deadline: &mut Deadline, diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index e131dba70..55a47ddc7 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3094,14 +3094,14 @@ impl Actor { })?; if !partition.faults.is_empty() { return Err(actor_error!( - illegal_argument, + forbidden, "cannot move partition {}: has faults", partition_idx ))?; } if !partition.unproven.is_empty() { return Err(actor_error!( - illegal_argument, + forbidden, "cannot move partition {}: has unproven", partition_idx ))?; @@ -3148,15 +3148,14 @@ impl Actor { format!("failed to load deadline {}", params.to_deadline), )?; - deadlines - .move_partitions( - store, - &mut from_deadline, - &mut to_deadline, - to_quant, - ¶ms.partitions, - ) - .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to move partitions")?; + Deadlines::move_partitions( + store, + &mut from_deadline, + &mut to_deadline, + to_quant, + ¶ms.partitions, + ) + .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to move partitions")?; deadlines .update_deadline(policy, store, params.from_deadline, &from_deadline) From 70fd5fe10901774c9b7f1b602668b2e69e24b767 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 2 Aug 2023 13:53:38 +0800 Subject: [PATCH 31/54] adjust test --- test_vm/tests/move_partitions_test.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs index 4f6028a2f..cbd4b8841 100644 --- a/test_vm/tests/move_partitions_test.rs +++ b/test_vm/tests/move_partitions_test.rs @@ -14,18 +14,20 @@ use fil_actor_miner::{ }; use fil_actor_power::{Method as PowerMethod, State as PowerState}; +use fil_actors_integration_tests::expects::Expect; +use fil_actors_integration_tests::util::{ + advance_by_deadline_to_epoch, advance_to_proving_deadline, assert_invariants, create_accounts, + create_miner, cron_tick, get_network_stats, make_bitfield, miner_balance, precommit_sectors, + submit_windowed_post, +}; use fil_actors_runtime::runtime::Policy; use fil_actors_runtime::{ CRON_ACTOR_ADDR, STORAGE_MARKET_ACTOR_ADDR, STORAGE_POWER_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, }; -use test_vm::expects::Expect; -use test_vm::trace::ExpectInvocation; -use test_vm::util::{ - advance_by_deadline_to_epoch, advance_to_proving_deadline, apply_ok, assert_invariants, - create_accounts, create_miner, cron_tick, get_network_stats, get_state, make_bitfield, - miner_balance, precommit_sectors, submit_windowed_post, DynBlockstore, -}; -use test_vm::{TestVM, VM}; +use test_vm::TestVM; +use vm_api::trace::ExpectInvocation; +use vm_api::util::{apply_ok, get_state, DynBlockstore}; +use vm_api::VM; #[test] fn move_partitions_success() { @@ -63,7 +65,7 @@ fn move_partitions_success() { cron_tick(&v); v.set_epoch(v.epoch() + 1); - assert_invariants(&v); + assert_invariants(&v, &Policy::default()); } fn submit_post_succeeds_test(v: &dyn VM, miner_info: MinerInfo, sector_info: SectorInfo) { @@ -85,7 +87,7 @@ fn submit_post_succeeds_test(v: &dyn VM, miner_info: MinerInfo, sector_info: Sec let p_st: PowerState = get_state(v, &STORAGE_POWER_ACTOR_ADDR).unwrap(); assert_eq!(sector_power.raw, p_st.total_bytes_committed); - assert_invariants(v); + assert_invariants(v, &Policy::default()); } struct SectorInfo { From 511e1708990b11733ee2c42775fa4b5c37938486 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 2 Aug 2023 14:12:04 +0800 Subject: [PATCH 32/54] refactor with Partition::adjust_for_move --- actors/miner/src/deadline_state.rs | 39 ++------------------------ actors/miner/src/partition_state.rs | 43 +++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 3ba8c4847..e90f1e029 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -24,10 +24,7 @@ use super::{ SectorOnChainInfo, Sectors, TerminationResult, }; -use crate::{ - PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, PARTITION_EXPIRATION_AMT_BITWIDTH, - SECTORS_AMT_BITWIDTH, -}; +use crate::SECTORS_AMT_BITWIDTH; // Bitwidth of AMTs determined empirically from mutation patterns and projections of mainnet data. // Usually a small array @@ -128,43 +125,11 @@ impl Deadlines { let to_partition_idx = first_to_partition_idx + i as u64; - let from_expirations_epochs: Array = - Array::load(&moving_partition.expirations_epochs, store)?; - - let from_early_terminations: Array = - Array::load(&moving_partition.early_terminated, store)?; - - let mut to_expirations_epochs = Array::::new_with_bit_width( - store, - PARTITION_EXPIRATION_AMT_BITWIDTH, - ); - let mut to_early_terminations = Array::::new_with_bit_width( - store, - PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, - ); - - // ajust epoch info of expirations_epochs and early_terminated within `Partition` - from_expirations_epochs.for_each(|from_epoch, expire_set| { - to_expirations_epochs.set( - to_quant.quantize_up(from_epoch as ChainEpoch) as u64, - expire_set.clone(), - )?; - Ok(()) - })?; - from_early_terminations.for_each(|from_epoch, bitfield| { - to_early_terminations - .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; - Ok(()) - })?; - moving_partition.expirations_epochs = to_expirations_epochs.flush()?; - moving_partition.early_terminated = to_early_terminations.flush()?; + moving_partition.adjust_for_move(store, &to_quant)?; let all_sectors = moving_partition.sectors.len(); let live_sectors = moving_partition.live_sectors().len(); let early_terminations = from_deadline.early_terminations.get(from_partition_idx); - if early_terminations != (from_early_terminations.count() > 0) { - return Err(actor_error!(illegal_state, "Deadline.early_terminations doesn't agree with Partition.early_terminated for partition {}", from_partition_idx))?; - } // start updating from/to `Deadline` here diff --git a/actors/miner/src/partition_state.rs b/actors/miner/src/partition_state.rs index d3ae5092c..ffc96e2ab 100644 --- a/actors/miner/src/partition_state.rs +++ b/actors/miner/src/partition_state.rs @@ -4,7 +4,7 @@ use std::convert::TryInto; use std::ops::{self, Neg}; -use anyhow::{anyhow, Context}; +use anyhow::{anyhow, Context, Ok}; use cid::Cid; use fil_actors_runtime::runtime::Policy; use fil_actors_runtime::{actor_error, ActorDowncast, Array}; @@ -24,8 +24,8 @@ use super::{ }; // Bitwidth of AMTs determined empirically from mutation patterns and projections of mainnet data. -pub const PARTITION_EXPIRATION_AMT_BITWIDTH: u32 = 4; -pub const PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH: u32 = 3; +const PARTITION_EXPIRATION_AMT_BITWIDTH: u32 = 4; +const PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH: u32 = 3; #[derive(Serialize_tuple, Deserialize_tuple, Clone)] pub struct Partition { @@ -822,6 +822,43 @@ impl Partition { self.validate_bf_state()?; Ok(()) } + + // ajust epoch info of expirations_epochs and early_terminated within `Partition` + pub fn adjust_for_move( + &mut self, + store: &BS, + to_quant: &QuantSpec, + ) -> anyhow::Result<()> { + let from_expirations_epochs: Array = + Array::load(&self.expirations_epochs, store)?; + + let from_early_terminations: Array = + Array::load(&self.early_terminated, store)?; + + let mut to_expirations_epochs = Array::::new_with_bit_width( + store, + PARTITION_EXPIRATION_AMT_BITWIDTH, + ); + let mut to_early_terminations = Array::::new_with_bit_width( + store, + PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, + ); + + from_expirations_epochs.for_each(|from_epoch, expire_set| { + to_expirations_epochs + .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, expire_set.clone())?; + Ok(()) + })?; + from_early_terminations.for_each(|from_epoch, bitfield| { + to_early_terminations + .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; + Ok(()) + })?; + self.expirations_epochs = to_expirations_epochs.flush()?; + self.early_terminated = to_early_terminations.flush()?; + + Ok(()) + } } #[derive(Serialize_tuple, Deserialize_tuple, Eq, PartialEq, Clone, Debug, Default)] From d16077cfb5a0522c4e06c54c285365570525f0ce Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 09:49:24 +0800 Subject: [PATCH 33/54] share the language with FIP --- actors/miner/src/deadlines.rs | 20 ++++----- actors/miner/src/lib.rs | 58 +++++++++++++-------------- actors/miner/src/types.rs | 4 +- actors/miner/tests/util.rs | 6 +-- test_vm/tests/move_partitions_test.rs | 4 +- 5 files changed, 46 insertions(+), 46 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 7c5cea955..9f28a2bf5 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -140,40 +140,40 @@ fn deadline_distance(policy: &Policy, from_deadline: u64, to_deadline: u64) -> u // only allow moving to a nearer deadline from current one pub fn deadline_available_for_move( policy: &Policy, - from_deadline: u64, - to_deadline: u64, + orig_deadline: u64, + dest_deadline: u64, current_deadline: &DeadlineInfo, ) -> Result<(), String> { if !deadline_is_mutable( policy, current_deadline.period_start, - from_deadline, + orig_deadline, current_deadline.current_epoch, ) { return Err(format!( "cannot move from a deadline {}, immutable at epoch {}", - from_deadline, current_deadline.current_epoch + orig_deadline, current_deadline.current_epoch )); } if !deadline_is_mutable( policy, current_deadline.period_start, - to_deadline, + dest_deadline, current_deadline.current_epoch, ) { return Err(format!( "cannot move to a deadline {}, immutable at epoch {}", - to_deadline, current_deadline.current_epoch + dest_deadline, current_deadline.current_epoch )); } - if deadline_distance(policy, current_deadline.index, to_deadline) - >= deadline_distance(policy, current_deadline.index, from_deadline) + if deadline_distance(policy, current_deadline.index, dest_deadline) + >= deadline_distance(policy, current_deadline.index, orig_deadline) { return Err(format!( - "can only move to a deadline which is nearer from current deadline {}, to_deadline {} is not nearer than from_deadline {}", - current_deadline.index, to_deadline, from_deadline + "can only move to a deadline which is nearer from current deadline {}, dest_deadline {} is not nearer than orig_deadline {}", + current_deadline.index, dest_deadline, orig_deadline )); } diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index c4bd38fb6..c1ae1969a 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2993,18 +2993,18 @@ impl Actor { } fn move_partitions(rt: &impl Runtime, params: MovePartitionsParams) -> Result<(), ActorError> { - if params.from_deadline == params.to_deadline { - return Err(actor_error!(illegal_argument, "from_deadline == to_deadline")); + if params.orig_deadline == params.dest_deadline { + return Err(actor_error!(illegal_argument, "orig_deadline == dest_deadline")); } let policy = rt.policy(); - if params.from_deadline >= policy.wpost_period_deadlines - || params.to_deadline >= policy.wpost_period_deadlines + if params.orig_deadline >= policy.wpost_period_deadlines + || params.dest_deadline >= policy.wpost_period_deadlines { return Err(actor_error!( illegal_argument, - "invalid param, from_deadline: {}, to_deadline: {}", - params.from_deadline, - params.to_deadline + "invalid param, orig_deadline: {}, dest_deadline: {}", + params.orig_deadline, + params.dest_deadline )); } if params.partitions.is_empty() { @@ -3026,8 +3026,8 @@ impl Actor { deadline_available_for_move( policy, - params.from_deadline, - params.to_deadline, + params.orig_deadline, + params.dest_deadline, ¤t_deadline, ) .context_code( @@ -3035,10 +3035,10 @@ impl Actor { "conditions not satisfied for deadline_available_for_move", )?; - let mut from_deadline = - deadlines.load_deadline(policy, store, params.from_deadline).context_code( + let mut orig_deadline = + deadlines.load_deadline(policy, store, params.orig_deadline).context_code( ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.from_deadline), + format!("failed to load deadline {}", params.orig_deadline), )?; // only try to do synchronous Window Post verification if the from deadline is in dispute window // note that as window post is batched, the best we can do is to verify only those that contains at least one partition being moved. @@ -3047,25 +3047,25 @@ impl Actor { if deadline_available_for_optimistic_post_dispute( policy, current_deadline.period_start, - params.from_deadline, + params.orig_deadline, rt.curr_epoch(), ) { - let proofs_snapshot = from_deadline + let proofs_snapshot = orig_deadline .optimistic_proofs_snapshot_amt(store) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load proofs snapshot")?; let partitions_snapshot = - from_deadline.partitions_snapshot_amt(store).context_code( + orig_deadline.partitions_snapshot_amt(store).context_code( ExitCode::USR_ILLEGAL_STATE, "failed to load partitions snapshot", )?; // Find the proving period start for the deadline in question. let target_deadline = - nearest_occured_deadline_info(policy, ¤t_deadline, params.from_deadline); + nearest_occured_deadline_info(policy, ¤t_deadline, params.orig_deadline); // Load sectors for the dispute. - let sectors = Sectors::load(rt.store(), &from_deadline.sectors_snapshot) + let sectors = Sectors::load(rt.store(), &orig_deadline.sectors_snapshot) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load sectors array")?; proofs_snapshot @@ -3140,41 +3140,41 @@ impl Actor { .context_code(ExitCode::USR_ILLEGAL_STATE, "while removing partitions")?; } - let to_quant = state.quant_spec_for_deadline(policy, params.to_deadline); + let dest_quant = state.quant_spec_for_deadline(policy, params.dest_deadline); - let mut to_deadline = - deadlines.load_deadline(policy, store, params.to_deadline).context_code( + let mut dest_deadline = + deadlines.load_deadline(policy, store, params.dest_deadline).context_code( ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.to_deadline), + format!("failed to load deadline {}", params.dest_deadline), )?; Deadlines::move_partitions( store, - &mut from_deadline, - &mut to_deadline, - to_quant, + &mut orig_deadline, + &mut dest_deadline, + dest_quant, ¶ms.partitions, ) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to move partitions")?; deadlines - .update_deadline(policy, store, params.from_deadline, &from_deadline) + .update_deadline(policy, store, params.orig_deadline, &orig_deadline) .context_code( ExitCode::USR_ILLEGAL_STATE, - format!("failed to update deadline {}", params.from_deadline), + format!("failed to update deadline {}", params.orig_deadline), )?; deadlines - .update_deadline(policy, store, params.to_deadline, &to_deadline) + .update_deadline(policy, store, params.dest_deadline, &dest_deadline) .context_code( ExitCode::USR_ILLEGAL_STATE, - format!("failed to update deadline {}", params.to_deadline), + format!("failed to update deadline {}", params.dest_deadline), )?; state.save_deadlines(store, deadlines).context_code( ExitCode::USR_ILLEGAL_STATE, format!( "failed to save deadline when move_partitions from {} to {}", - params.from_deadline, params.to_deadline + params.orig_deadline, params.dest_deadline ), )?; diff --git a/actors/miner/src/types.rs b/actors/miner/src/types.rs index 8d4aa006f..bc38e5833 100644 --- a/actors/miner/src/types.rs +++ b/actors/miner/src/types.rs @@ -245,8 +245,8 @@ pub struct CompactPartitionsParams { #[derive(Serialize_tuple, Deserialize_tuple)] pub struct MovePartitionsParams { - pub from_deadline: u64, - pub to_deadline: u64, + pub orig_deadline: u64, + pub dest_deadline: u64, pub partitions: BitField, } diff --git a/actors/miner/tests/util.rs b/actors/miner/tests/util.rs index c63b8d8a3..8bfc59ec1 100644 --- a/actors/miner/tests/util.rs +++ b/actors/miner/tests/util.rs @@ -2602,14 +2602,14 @@ impl ActorHarness { pub fn move_partitions( &self, rt: &MockRuntime, - from_deadline: u64, - to_deadline: u64, + orig_deadline: u64, + dest_deadline: u64, partitions: BitField, mut f: impl FnMut(), ) -> Result<(), ActorError> { f(); - let params = MovePartitionsParams { from_deadline, to_deadline, partitions }; + let params = MovePartitionsParams { orig_deadline, dest_deadline, partitions }; rt.expect_validate_caller_addr(self.caller_addrs()); rt.set_caller(*ACCOUNT_ACTOR_CODE_ID, self.worker); diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs index cbd4b8841..62fd5e72f 100644 --- a/test_vm/tests/move_partitions_test.rs +++ b/test_vm/tests/move_partitions_test.rs @@ -40,8 +40,8 @@ fn move_partitions_success() { advance_by_deadline_to_epoch(&v, &miner.miner_id, prove_time); let move_params = MovePartitionsParams { - from_deadline: 0, - to_deadline: 47, + orig_deadline: 0, + dest_deadline: 47, partitions: make_bitfield(&[0u64]), }; let prove_params_ser = IpldBlock::serialize_cbor(&move_params).unwrap(); From 9ed53b84536839cad8473c958c10d47c145769fb Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 09:55:17 +0800 Subject: [PATCH 34/54] deadline_available_for_move => ensure_deadline_available_for_move --- actors/miner/src/deadlines.rs | 2 +- actors/miner/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 9f28a2bf5..788444c08 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -138,7 +138,7 @@ fn deadline_distance(policy: &Policy, from_deadline: u64, to_deadline: u64) -> u } // only allow moving to a nearer deadline from current one -pub fn deadline_available_for_move( +pub fn ensure_deadline_available_for_move( policy: &Policy, orig_deadline: u64, dest_deadline: u64, diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index c1ae1969a..48f7a0497 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -3024,7 +3024,7 @@ impl Actor { .load_deadlines(store) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load deadlines")?; - deadline_available_for_move( + ensure_deadline_available_for_move( policy, params.orig_deadline, params.dest_deadline, From 15d5ac1dc3eb4c1f75f9074c158c4bd2f94f5baf Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 11:04:13 +0800 Subject: [PATCH 35/54] add some doc comment --- actors/miner/src/deadlines.rs | 4 ++-- actors/miner/src/lib.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/deadlines.rs b/actors/miner/src/deadlines.rs index 788444c08..d5b18c096 100644 --- a/actors/miner/src/deadlines.rs +++ b/actors/miner/src/deadlines.rs @@ -128,7 +128,7 @@ pub fn deadline_available_for_compaction( ) } -// the distance between from_deadline and to_deadline clockwise in deadline unit. +/// the distance between from_deadline and to_deadline clockwise in deadline unit. fn deadline_distance(policy: &Policy, from_deadline: u64, to_deadline: u64) -> u64 { if to_deadline >= from_deadline { to_deadline - from_deadline @@ -137,7 +137,7 @@ fn deadline_distance(policy: &Policy, from_deadline: u64, to_deadline: u64) -> u } } -// only allow moving to a nearer deadline from current one +/// only allow moving to a nearer deadline from current one pub fn ensure_deadline_available_for_move( policy: &Policy, orig_deadline: u64, diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 48f7a0497..ebe805fb1 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2992,6 +2992,7 @@ impl Actor { Ok(()) } + /// FYI: https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0070.md fn move_partitions(rt: &impl Runtime, params: MovePartitionsParams) -> Result<(), ActorError> { if params.orig_deadline == params.dest_deadline { return Err(actor_error!(illegal_argument, "orig_deadline == dest_deadline")); From 5744223f6513d6f0711e8c5864ca057fb48f49fb Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 11:18:49 +0800 Subject: [PATCH 36/54] more renaming --- actors/miner/src/deadline_state.rs | 84 +++++++++++++++--------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index e90f1e029..e98aa34ad 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -103,88 +103,88 @@ impl Deadlines { pub fn move_partitions( store: &BS, - from_deadline: &mut Deadline, - to_deadline: &mut Deadline, - to_quant: QuantSpec, + orig_deadline: &mut Deadline, + dest_deadline: &mut Deadline, + dest_quant: QuantSpec, partitions: &BitField, ) -> anyhow::Result<()> { - let mut from_partitions = from_deadline.partitions_amt(store)?; - let mut to_partitions = to_deadline.partitions_amt(store)?; + let mut orig_partitions = orig_deadline.partitions_amt(store)?; + let mut dest_partitions = dest_deadline.partitions_amt(store)?; // even though we're moving partitions intact, we still need to update from/to `Deadline` accordingly. - let first_to_partition_idx = to_partitions.count(); - for (i, from_partition_idx) in partitions.iter().enumerate() { - let mut moving_partition = from_partitions - .get(from_partition_idx)? - .ok_or_else(|| actor_error!(not_found, "no partition {}", from_partition_idx))? + let first_dest_partition_idx = dest_partitions.count(); + for (i, orig_partition_idx) in partitions.iter().enumerate() { + let mut moving_partition = orig_partitions + .get(orig_partition_idx)? + .ok_or_else(|| actor_error!(not_found, "no partition {}", orig_partition_idx))? .clone(); if !moving_partition.faults.is_empty() || !moving_partition.unproven.is_empty() { - return Err(actor_error!(forbidden, "partition with faults or unproven sectors are not allowed to move, partition_idx {}", from_partition_idx))?; + return Err(actor_error!(forbidden, "partition with faults or unproven sectors are not allowed to move, partition_idx {}", orig_partition_idx))?; } - let to_partition_idx = first_to_partition_idx + i as u64; + let dest_partition_idx = first_dest_partition_idx + i as u64; - moving_partition.adjust_for_move(store, &to_quant)?; + moving_partition.adjust_for_move(store, &dest_quant)?; let all_sectors = moving_partition.sectors.len(); let live_sectors = moving_partition.live_sectors().len(); - let early_terminations = from_deadline.early_terminations.get(from_partition_idx); + let early_terminations = orig_deadline.early_terminations.get(orig_partition_idx); // start updating from/to `Deadline` here - from_deadline.total_sectors -= all_sectors; - from_deadline.live_sectors -= live_sectors; - from_deadline.faulty_power -= &moving_partition.faulty_power; + orig_deadline.total_sectors -= all_sectors; + orig_deadline.live_sectors -= live_sectors; + orig_deadline.faulty_power -= &moving_partition.faulty_power; - to_deadline.total_sectors += all_sectors; - to_deadline.live_sectors += live_sectors; - to_deadline.faulty_power += &moving_partition.faulty_power; + dest_deadline.total_sectors += all_sectors; + dest_deadline.live_sectors += live_sectors; + dest_deadline.faulty_power += &moving_partition.faulty_power; // update early_terminations BitField of `Deadline` if early_terminations { - from_deadline.early_terminations.unset(from_partition_idx); - to_deadline.early_terminations.set(to_partition_idx); + orig_deadline.early_terminations.unset(orig_partition_idx); + dest_deadline.early_terminations.set(dest_partition_idx); } - from_partitions.set(from_partition_idx, Partition::new(store)?)?; - to_partitions.set(to_partition_idx, moving_partition)?; + orig_partitions.set(orig_partition_idx, Partition::new(store)?)?; + dest_partitions.set(dest_partition_idx, moving_partition)?; } // update expirations_epochs Cid of Deadline. { let mut epochs_to_remove = Vec::::new(); - let mut from_expirations_epochs: Array = - Array::load(&from_deadline.expirations_epochs, store)?; - let mut to_expirations_epochs: Array = - Array::load(&to_deadline.expirations_epochs, store)?; - from_expirations_epochs.for_each_mut(|from_epoch, from_bitfield| { - let to_epoch = to_quant.quantize_up(from_epoch as ChainEpoch); + let mut orig_expirations_epochs: Array = + Array::load(&orig_deadline.expirations_epochs, store)?; + let mut dest_expirations_epochs: Array = + Array::load(&dest_deadline.expirations_epochs, store)?; + orig_expirations_epochs.for_each_mut(|orig_epoch, orig_bitfield| { + let dest_epoch = dest_quant.quantize_up(orig_epoch as ChainEpoch); let mut to_bitfield = - to_expirations_epochs.get(to_epoch as u64)?.cloned().unwrap_or_default(); + dest_expirations_epochs.get(dest_epoch as u64)?.cloned().unwrap_or_default(); for (i, partition_id) in partitions.iter().enumerate() { - if from_bitfield.get(partition_id) { - from_bitfield.unset(partition_id); - to_bitfield.set(first_to_partition_idx + i as u64); + if orig_bitfield.get(partition_id) { + orig_bitfield.unset(partition_id); + to_bitfield.set(first_dest_partition_idx + i as u64); } } - to_expirations_epochs.set(to_epoch as u64, to_bitfield)?; + dest_expirations_epochs.set(dest_epoch as u64, to_bitfield)?; - if from_bitfield.is_empty() { - epochs_to_remove.push(from_epoch); + if orig_bitfield.is_empty() { + epochs_to_remove.push(orig_epoch); } Ok(()) })?; if !epochs_to_remove.is_empty() { - from_expirations_epochs.batch_delete(epochs_to_remove, true)?; + orig_expirations_epochs.batch_delete(epochs_to_remove, true)?; } - from_deadline.expirations_epochs = from_expirations_epochs.flush()?; - to_deadline.expirations_epochs = to_expirations_epochs.flush()?; + orig_deadline.expirations_epochs = orig_expirations_epochs.flush()?; + dest_deadline.expirations_epochs = dest_expirations_epochs.flush()?; } - from_deadline.partitions = from_partitions.flush()?; - to_deadline.partitions = to_partitions.flush()?; + orig_deadline.partitions = orig_partitions.flush()?; + dest_deadline.partitions = dest_partitions.flush()?; Ok(()) } From e8f0a221ee598ece1d62cdf550dde75a2e1fac0f Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 11:22:24 +0800 Subject: [PATCH 37/54] more renaming --- test_vm/tests/move_partitions_test.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs index 62fd5e72f..763c3bbae 100644 --- a/test_vm/tests/move_partitions_test.rs +++ b/test_vm/tests/move_partitions_test.rs @@ -48,7 +48,7 @@ fn move_partitions_success() { apply_ok( &v, &miner.worker, - &miner._miner_robust, + &miner.miner_robust, &TokenAmount::zero(), MinerMethod::MovePartitions as u64, Some(move_params), @@ -99,10 +99,9 @@ struct SectorInfo { #[derive(Clone)] struct MinerInfo { seal_proof: RegisteredSealProof, - _owner: Address, worker: Address, miner_id: Address, - _miner_robust: Address, + miner_robust: Address, } fn setup(store: &'_ MemoryBlockstore) -> (TestVM, MinerInfo, SectorInfo) { @@ -204,13 +203,7 @@ fn setup(store: &'_ MemoryBlockstore) -> (TestVM, MinerInfo, S let (deadline_info, partition_index) = advance_to_proving_deadline(&v, &id_addr, sector_number); ( v, - MinerInfo { - seal_proof, - worker, - _owner: owner, - miner_id: id_addr, - _miner_robust: robust_addr, - }, + MinerInfo { seal_proof, worker, miner_id: id_addr, miner_robust: robust_addr }, SectorInfo { number: sector_number, deadline_info, partition_index }, ) } From 5e400d21706b58251ca37c9fab9d5dfac97747fd Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 11:50:57 +0800 Subject: [PATCH 38/54] rename + merge master --- actors/miner/src/lib.rs | 4 ++-- actors/miner/src/partition_state.rs | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 2d5f9b9aa..6e9585344 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2737,7 +2737,7 @@ impl Actor { )?; let mut orig_deadline = - deadlines.load_deadline(policy, store, params.orig_deadline).context_code( + deadlines.load_deadline(store, params.orig_deadline).context_code( ExitCode::USR_ILLEGAL_STATE, format!("failed to load deadline {}", params.orig_deadline), )?; @@ -2844,7 +2844,7 @@ impl Actor { let dest_quant = state.quant_spec_for_deadline(policy, params.dest_deadline); let mut dest_deadline = - deadlines.load_deadline(policy, store, params.dest_deadline).context_code( + deadlines.load_deadline(store, params.dest_deadline).context_code( ExitCode::USR_ILLEGAL_STATE, format!("failed to load deadline {}", params.dest_deadline), )?; diff --git a/actors/miner/src/partition_state.rs b/actors/miner/src/partition_state.rs index ffc96e2ab..2e78cc887 100644 --- a/actors/miner/src/partition_state.rs +++ b/actors/miner/src/partition_state.rs @@ -827,35 +827,35 @@ impl Partition { pub fn adjust_for_move( &mut self, store: &BS, - to_quant: &QuantSpec, + dest_quant: &QuantSpec, ) -> anyhow::Result<()> { - let from_expirations_epochs: Array = + let orig_expirations_epochs: Array = Array::load(&self.expirations_epochs, store)?; - let from_early_terminations: Array = + let orig_early_terminations: Array = Array::load(&self.early_terminated, store)?; - let mut to_expirations_epochs = Array::::new_with_bit_width( + let mut dest_expirations_epochs = Array::::new_with_bit_width( store, PARTITION_EXPIRATION_AMT_BITWIDTH, ); - let mut to_early_terminations = Array::::new_with_bit_width( + let mut dest_early_terminations = Array::::new_with_bit_width( store, PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, ); - from_expirations_epochs.for_each(|from_epoch, expire_set| { - to_expirations_epochs - .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, expire_set.clone())?; + orig_expirations_epochs.for_each(|from_epoch, expire_set| { + dest_expirations_epochs + .set(dest_quant.quantize_up(from_epoch as ChainEpoch) as u64, expire_set.clone())?; Ok(()) })?; - from_early_terminations.for_each(|from_epoch, bitfield| { - to_early_terminations - .set(to_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; + orig_early_terminations.for_each(|from_epoch, bitfield| { + dest_early_terminations + .set(dest_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; Ok(()) })?; - self.expirations_epochs = to_expirations_epochs.flush()?; - self.early_terminated = to_early_terminations.flush()?; + self.expirations_epochs = dest_expirations_epochs.flush()?; + self.early_terminated = dest_early_terminations.flush()?; Ok(()) } From c1159c5591271c7161da4b09b2e709cef6507cb5 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 11:56:46 +0800 Subject: [PATCH 39/54] mod wording --- actors/miner/tests/move_partitions_test.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index af9a5bfb1..13c0f7f14 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -71,7 +71,10 @@ fn farthest_possible_to_deadline( from_deadline_id: u64, current_deadline: DeadlineInfo, ) -> u64 { - assert!(from_deadline_id != current_deadline.index, "can't move nearer when the gap is 0"); + assert_ne!( + from_deadline_id, current_deadline.index, + "can't move nearer when the deadline_distance is 0" + ); if current_deadline.index < from_deadline_id { // the deadline distance can only be nearer From cb5e3c450778990af65c1b35678334411d8dae0d Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 30 Aug 2023 13:44:55 +0800 Subject: [PATCH 40/54] fix test --- test_vm/tests/move_partitions_test.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test_vm/tests/move_partitions_test.rs b/test_vm/tests/move_partitions_test.rs index 763c3bbae..724561715 100644 --- a/test_vm/tests/move_partitions_test.rs +++ b/test_vm/tests/move_partitions_test.rs @@ -22,7 +22,8 @@ use fil_actors_integration_tests::util::{ }; use fil_actors_runtime::runtime::Policy; use fil_actors_runtime::{ - CRON_ACTOR_ADDR, STORAGE_MARKET_ACTOR_ADDR, STORAGE_POWER_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, + CRON_ACTOR_ADDR, CRON_ACTOR_ID, STORAGE_MARKET_ACTOR_ADDR, STORAGE_POWER_ACTOR_ADDR, + STORAGE_POWER_ACTOR_ID, SYSTEM_ACTOR_ADDR, }; use test_vm::TestVM; use vm_api::trace::ExpectInvocation; @@ -54,7 +55,7 @@ fn move_partitions_success() { Some(move_params), ); ExpectInvocation { - from: miner.worker, + from: miner.worker.id().unwrap(), to: miner.miner_id, method: MinerMethod::MovePartitions as u64, params: Some(prove_params_ser), @@ -140,11 +141,11 @@ fn setup(store: &'_ MemoryBlockstore) -> (TestVM, MinerInfo, S Some(prove_params), ); ExpectInvocation { - from: worker, + from: worker.id().unwrap(), to: id_addr, method: MinerMethod::ProveCommitSector as u64, params: Some(prove_params_ser), - subinvocs: Some(vec![Expect::power_submit_porep(id_addr)]), + subinvocs: Some(vec![Expect::power_submit_porep(id_addr.id().unwrap())]), ..Default::default() } .matches(v.take_invocations().last().unwrap()); @@ -163,16 +164,19 @@ fn setup(store: &'_ MemoryBlockstore) -> (TestVM, MinerInfo, S method: CronMethod::EpochTick as u64, subinvocs: Some(vec![ ExpectInvocation { - from: CRON_ACTOR_ADDR, + from: CRON_ACTOR_ID, to: STORAGE_POWER_ACTOR_ADDR, method: PowerMethod::OnEpochTickEnd as u64, subinvocs: Some(vec![ - Expect::reward_this_epoch(STORAGE_POWER_ACTOR_ADDR), + Expect::reward_this_epoch(STORAGE_POWER_ACTOR_ID), ExpectInvocation { - from: STORAGE_POWER_ACTOR_ADDR, + from: STORAGE_POWER_ACTOR_ID, to: id_addr, method: MinerMethod::ConfirmSectorProofsValid as u64, - subinvocs: Some(vec![Expect::power_update_pledge(id_addr, None)]), + subinvocs: Some(vec![Expect::power_update_pledge( + id_addr.id().unwrap(), + None, + )]), ..Default::default() }, Expect::reward_update_kpi(), @@ -180,7 +184,7 @@ fn setup(store: &'_ MemoryBlockstore) -> (TestVM, MinerInfo, S ..Default::default() }, ExpectInvocation { - from: CRON_ACTOR_ADDR, + from: CRON_ACTOR_ID, to: STORAGE_MARKET_ACTOR_ADDR, method: MarketMethod::CronTick as u64, ..Default::default() From faa4873e517d43e464338c094aaf7b66df256110 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Thu, 31 Aug 2023 09:17:01 +0800 Subject: [PATCH 41/54] renaming in test --- actors/miner/tests/move_partitions_test.rs | 76 +++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index 13c0f7f14..ce31b4635 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -25,19 +25,19 @@ fn setup() -> (ActorHarness, MockRuntime) { } // returns the nearest epoch such that synchronous post verification is required -fn nearest_unsafe_epoch(rt: &MockRuntime, h: &ActorHarness, from_deadline_id: u64) -> i64 { +fn nearest_unsafe_epoch(rt: &MockRuntime, h: &ActorHarness, orig_deadline_id: u64) -> i64 { let current_ddl = h.current_deadline(rt); for i in *rt.epoch.borrow().. { if !deadline_available_for_compaction( &rt.policy, current_ddl.period_start, - from_deadline_id, + orig_deadline_id, i, ) && deadline_available_for_optimistic_post_dispute( &rt.policy, current_ddl.period_start, - from_deadline_id, + orig_deadline_id, i, ) { return i; @@ -48,14 +48,14 @@ fn nearest_unsafe_epoch(rt: &MockRuntime, h: &ActorHarness, from_deadline_id: u6 } // returns the nearest epoch such that no synchronous post verification is necessary -fn nearest_safe_epoch(rt: &MockRuntime, h: &ActorHarness, from_deadline_id: u64) -> i64 { +fn nearest_safe_epoch(rt: &MockRuntime, h: &ActorHarness, orig_deadline_id: u64) -> i64 { let current_ddl = h.current_deadline(rt); for i in *rt.epoch.borrow().. { if deadline_available_for_compaction( &rt.policy, current_ddl.period_start, - from_deadline_id, + orig_deadline_id, i, ) { return i; @@ -68,24 +68,24 @@ fn nearest_safe_epoch(rt: &MockRuntime, h: &ActorHarness, from_deadline_id: u64) // returns the farthest deadline from current that satisfies deadline_available_for_move fn farthest_possible_to_deadline( rt: &MockRuntime, - from_deadline_id: u64, + orig_deadline_id: u64, current_deadline: DeadlineInfo, ) -> u64 { assert_ne!( - from_deadline_id, current_deadline.index, + orig_deadline_id, current_deadline.index, "can't move nearer when the deadline_distance is 0" ); - if current_deadline.index < from_deadline_id { + if current_deadline.index < orig_deadline_id { // the deadline distance can only be nearer - for i in (current_deadline.index..(from_deadline_id)).rev() { + for i in (current_deadline.index..(orig_deadline_id)).rev() { if deadline_is_mutable(&rt.policy, current_deadline.period_start, i, *rt.epoch.borrow()) { return i; } } } else { - for i in (0..(from_deadline_id)).rev() { + for i in (0..(orig_deadline_id)).rev() { if deadline_is_mutable(&rt.policy, current_deadline.period_start, i, *rt.epoch.borrow()) { return i; @@ -122,16 +122,16 @@ fn fail_to_move_partitions_with_faults_from_safe_epoch() { h.declare_faults(&rt, §ors_info[0..1]); let partition_id = 0; - let from_deadline_id = 0; + let orig_deadline_id = 0; - h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, from_deadline_id)); + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); let to_deadline_id = - farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); let result = h.move_partitions( &rt, - from_deadline_id, + orig_deadline_id, to_deadline_id, bitfield_from_slice(&[partition_id]), || {}, @@ -164,33 +164,33 @@ fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { h.declare_faults(&rt, §ors_info[0..1]); let partition_id = 0; - let from_deadline_id = 0; + let orig_deadline_id = 0; - h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, from_deadline_id)); + h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, orig_deadline_id)); - let to_deadline_id = - farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); let result = h.move_partitions( &rt, - from_deadline_id, - to_deadline_id, + orig_deadline_id, + dest_deadline_id, bitfield_from_slice(&[partition_id]), || { let current_deadline = h.current_deadline(&rt); let from_deadline = new_deadline_info( rt.policy(), - if current_deadline.index < from_deadline_id { + if current_deadline.index < orig_deadline_id { current_deadline.period_start - rt.policy().wpost_proving_period } else { current_deadline.period_start }, - from_deadline_id, + orig_deadline_id, *rt.epoch.borrow(), ); - let from_ddl = h.get_deadline(&rt, from_deadline_id); + let from_ddl = h.get_deadline(&rt, orig_deadline_id); let entropy = RawBytes::serialize(h.receiver).unwrap(); rt.expect_get_randomness_from_beacon( @@ -237,18 +237,18 @@ fn ok_to_move_partitions_from_safe_epoch() { ); h.advance_and_submit_posts(&rt, §ors_info); - let from_deadline_id = 0; + let orig_deadline_id = 0; - h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, from_deadline_id)); + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); let partition_id = 0; - let to_deadline_id = - farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); let result = h.move_partitions( &rt, - from_deadline_id, - to_deadline_id, + orig_deadline_id, + dest_deadline_id, bitfield_from_slice(&[partition_id]), || {}, ); @@ -272,34 +272,34 @@ fn ok_to_move_partitions_from_unsafe_epoch() { ); h.advance_and_submit_posts(&rt, §ors_info); - let from_deadline_id = 0; + let orig_deadline_id = 0; - h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, from_deadline_id)); + h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, orig_deadline_id)); let partition_id = 0; - let to_deadline_id = - farthest_possible_to_deadline(&rt, from_deadline_id, h.current_deadline(&rt)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); let result = h.move_partitions( &rt, - from_deadline_id, - to_deadline_id, + orig_deadline_id, + dest_deadline_id, bitfield_from_slice(&[partition_id]), || { let current_deadline = h.current_deadline(&rt); let from_deadline = new_deadline_info( rt.policy(), - if current_deadline.index < from_deadline_id { + if current_deadline.index < orig_deadline_id { current_deadline.period_start - rt.policy().wpost_proving_period } else { current_deadline.period_start }, - from_deadline_id, + orig_deadline_id, *rt.epoch.borrow(), ); - let from_ddl = h.get_deadline(&rt, from_deadline_id); + let from_ddl = h.get_deadline(&rt, orig_deadline_id); let entropy = RawBytes::serialize(h.receiver).unwrap(); rt.expect_get_randomness_from_beacon( From 7bc1156d09531d15f033bbd3d65f2edf67e1f707 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 19 Sep 2023 14:52:47 +0800 Subject: [PATCH 42/54] apply alex's idea of not re-quantizing at all. --- actors/miner/src/deadline_state.rs | 13 ++++++------- actors/miner/src/lib.rs | 3 --- actors/miner/src/partition_state.rs | 12 +++--------- actors/miner/src/testing.rs | 6 +----- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 537d54021..1aa6e87c3 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -108,13 +108,12 @@ impl Deadlines { store: &BS, orig_deadline: &mut Deadline, dest_deadline: &mut Deadline, - dest_quant: QuantSpec, partitions: &BitField, ) -> anyhow::Result<()> { let mut orig_partitions = orig_deadline.partitions_amt(store)?; let mut dest_partitions = dest_deadline.partitions_amt(store)?; - // even though we're moving partitions intact, we still need to update from/to `Deadline` accordingly. + // even though we're moving partitions intact, we still need to update orig/dest `Deadline` accordingly. let first_dest_partition_idx = dest_partitions.count(); for (i, orig_partition_idx) in partitions.iter().enumerate() { @@ -128,13 +127,13 @@ impl Deadlines { let dest_partition_idx = first_dest_partition_idx + i as u64; - moving_partition.adjust_for_move(store, &dest_quant)?; + moving_partition.adjust_for_move(store)?; let all_sectors = moving_partition.sectors.len(); let live_sectors = moving_partition.live_sectors().len(); let early_terminations = orig_deadline.early_terminations.get(orig_partition_idx); - // start updating from/to `Deadline` here + // start updating orig/dest `Deadline` here orig_deadline.total_sectors -= all_sectors; orig_deadline.live_sectors -= live_sectors; @@ -162,16 +161,16 @@ impl Deadlines { let mut dest_expirations_epochs: Array = Array::load(&dest_deadline.expirations_epochs, store)?; orig_expirations_epochs.for_each_mut(|orig_epoch, orig_bitfield| { - let dest_epoch = dest_quant.quantize_up(orig_epoch as ChainEpoch); + let dest_epoch = orig_epoch; let mut to_bitfield = - dest_expirations_epochs.get(dest_epoch as u64)?.cloned().unwrap_or_default(); + dest_expirations_epochs.get(dest_epoch)?.cloned().unwrap_or_default(); for (i, partition_id) in partitions.iter().enumerate() { if orig_bitfield.get(partition_id) { orig_bitfield.unset(partition_id); to_bitfield.set(first_dest_partition_idx + i as u64); } } - dest_expirations_epochs.set(dest_epoch as u64, to_bitfield)?; + dest_expirations_epochs.set(dest_epoch, to_bitfield)?; if orig_bitfield.is_empty() { epochs_to_remove.push(orig_epoch); diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 6e9585344..f7da30c1d 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2841,8 +2841,6 @@ impl Actor { .context_code(ExitCode::USR_ILLEGAL_STATE, "while removing partitions")?; } - let dest_quant = state.quant_spec_for_deadline(policy, params.dest_deadline); - let mut dest_deadline = deadlines.load_deadline(store, params.dest_deadline).context_code( ExitCode::USR_ILLEGAL_STATE, @@ -2853,7 +2851,6 @@ impl Actor { store, &mut orig_deadline, &mut dest_deadline, - dest_quant, ¶ms.partitions, ) .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to move partitions")?; diff --git a/actors/miner/src/partition_state.rs b/actors/miner/src/partition_state.rs index 2e78cc887..ea4b2e112 100644 --- a/actors/miner/src/partition_state.rs +++ b/actors/miner/src/partition_state.rs @@ -824,11 +824,7 @@ impl Partition { } // ajust epoch info of expirations_epochs and early_terminated within `Partition` - pub fn adjust_for_move( - &mut self, - store: &BS, - dest_quant: &QuantSpec, - ) -> anyhow::Result<()> { + pub fn adjust_for_move(&mut self, store: &BS) -> anyhow::Result<()> { let orig_expirations_epochs: Array = Array::load(&self.expirations_epochs, store)?; @@ -845,13 +841,11 @@ impl Partition { ); orig_expirations_epochs.for_each(|from_epoch, expire_set| { - dest_expirations_epochs - .set(dest_quant.quantize_up(from_epoch as ChainEpoch) as u64, expire_set.clone())?; + dest_expirations_epochs.set(from_epoch, expire_set.clone())?; Ok(()) })?; orig_early_terminations.for_each(|from_epoch, bitfield| { - dest_early_terminations - .set(dest_quant.quantize_up(from_epoch as ChainEpoch) as u64, bitfield.clone())?; + dest_early_terminations.set(from_epoch, bitfield.clone())?; Ok(()) })?; self.expirations_epochs = dest_expirations_epochs.flush()?; diff --git a/actors/miner/src/testing.rs b/actors/miner/src/testing.rs index e2b3e322c..19b07c960 100644 --- a/actors/miner/src/testing.rs +++ b/actors/miner/src/testing.rs @@ -629,9 +629,6 @@ impl ExpirationQueueStateSummary { let ret = expiration_queue.amt.for_each(|epoch, expiration_set| { let epoch = epoch as i64; let acc = acc.with_prefix(format!("expiration epoch {epoch}: ")); - let quant_up = quant.quantize_up(epoch); - acc.require(quant_up == epoch, format!("expiration queue key {epoch} is not quantized, expected {quant_up}")); - expiration_epochs.push(epoch); let mut on_time_sectors_pledge = TokenAmount::zero(); @@ -643,8 +640,7 @@ impl ExpirationQueueStateSummary { // check expiring sectors are still alive if let Some(sector) = live_sectors.get(§or_number) { - let target = quant.quantize_up(sector.expiration); - acc.require(epoch >= target && (epoch-target)%quant.unit ==0 , format!("invalid expiration {epoch} for sector {sector_number}, expected {target}")); + acc.require(epoch >= sector.expiration , format!("invalid expiration {epoch} for sector {sector_number}")); on_time_sectors_pledge += sector.initial_pledge.clone(); } else { acc.add(format!("on time expiration sector {sector_number} isn't live")); From f7b7bd7af6d1981c56db415175492163a5c099dc Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Sat, 23 Sep 2023 01:55:36 +0800 Subject: [PATCH 43/54] 1. forbid moving when there're early terminations 2. remove `adjust_for_move` --- actors/miner/src/deadline_state.rs | 14 ++++--------- actors/miner/src/partition_state.rs | 31 ----------------------------- 2 files changed, 4 insertions(+), 41 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 1aa6e87c3..554a14c4b 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -117,7 +117,7 @@ impl Deadlines { let first_dest_partition_idx = dest_partitions.count(); for (i, orig_partition_idx) in partitions.iter().enumerate() { - let mut moving_partition = orig_partitions + let moving_partition = orig_partitions .get(orig_partition_idx)? .ok_or_else(|| actor_error!(not_found, "no partition {}", orig_partition_idx))? .clone(); @@ -127,11 +127,11 @@ impl Deadlines { let dest_partition_idx = first_dest_partition_idx + i as u64; - moving_partition.adjust_for_move(store)?; - let all_sectors = moving_partition.sectors.len(); let live_sectors = moving_partition.live_sectors().len(); - let early_terminations = orig_deadline.early_terminations.get(orig_partition_idx); + if orig_deadline.early_terminations.get(orig_partition_idx) { + return Err(actor_error!(forbidden, "partition with early terminated sectors are not allowed to move, partition_idx {}", orig_partition_idx))?; + } // start updating orig/dest `Deadline` here @@ -143,12 +143,6 @@ impl Deadlines { dest_deadline.live_sectors += live_sectors; dest_deadline.faulty_power += &moving_partition.faulty_power; - // update early_terminations BitField of `Deadline` - if early_terminations { - orig_deadline.early_terminations.unset(orig_partition_idx); - dest_deadline.early_terminations.set(dest_partition_idx); - } - orig_partitions.set(orig_partition_idx, Partition::new(store)?)?; dest_partitions.set(dest_partition_idx, moving_partition)?; } diff --git a/actors/miner/src/partition_state.rs b/actors/miner/src/partition_state.rs index ea4b2e112..0b5d2ef30 100644 --- a/actors/miner/src/partition_state.rs +++ b/actors/miner/src/partition_state.rs @@ -822,37 +822,6 @@ impl Partition { self.validate_bf_state()?; Ok(()) } - - // ajust epoch info of expirations_epochs and early_terminated within `Partition` - pub fn adjust_for_move(&mut self, store: &BS) -> anyhow::Result<()> { - let orig_expirations_epochs: Array = - Array::load(&self.expirations_epochs, store)?; - - let orig_early_terminations: Array = - Array::load(&self.early_terminated, store)?; - - let mut dest_expirations_epochs = Array::::new_with_bit_width( - store, - PARTITION_EXPIRATION_AMT_BITWIDTH, - ); - let mut dest_early_terminations = Array::::new_with_bit_width( - store, - PARTITION_EARLY_TERMINATION_ARRAY_AMT_BITWIDTH, - ); - - orig_expirations_epochs.for_each(|from_epoch, expire_set| { - dest_expirations_epochs.set(from_epoch, expire_set.clone())?; - Ok(()) - })?; - orig_early_terminations.for_each(|from_epoch, bitfield| { - dest_early_terminations.set(from_epoch, bitfield.clone())?; - Ok(()) - })?; - self.expirations_epochs = dest_expirations_epochs.flush()?; - self.early_terminated = dest_early_terminations.flush()?; - - Ok(()) - } } #[derive(Serialize_tuple, Deserialize_tuple, Eq, PartialEq, Clone, Debug, Default)] From 3b25fdcf2c075833d008e2583e42220e1571db07 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Sat, 23 Sep 2023 02:49:45 +0800 Subject: [PATCH 44/54] rm anyhow::Ok --- actors/miner/src/partition_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actors/miner/src/partition_state.rs b/actors/miner/src/partition_state.rs index 0b5d2ef30..9ff3c3e72 100644 --- a/actors/miner/src/partition_state.rs +++ b/actors/miner/src/partition_state.rs @@ -4,7 +4,7 @@ use std::convert::TryInto; use std::ops::{self, Neg}; -use anyhow::{anyhow, Context, Ok}; +use anyhow::{anyhow, Context}; use cid::Cid; use fil_actors_runtime::runtime::Policy; use fil_actors_runtime::{actor_error, ActorDowncast, Array}; From 9c8517ab295dc2d5db5764654e59fce803efb201 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Sat, 23 Sep 2023 21:54:57 +0800 Subject: [PATCH 45/54] minor optimization by observing that partition `faulty_power` should be zero when `faults` is empty --- actors/miner/src/deadline_state.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 554a14c4b..8dc04075d 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -124,6 +124,13 @@ impl Deadlines { if !moving_partition.faults.is_empty() || !moving_partition.unproven.is_empty() { return Err(actor_error!(forbidden, "partition with faults or unproven sectors are not allowed to move, partition_idx {}", orig_partition_idx))?; } + if !moving_partition.faulty_power.is_zero() { + return Err(actor_error!( + illegal_state, + "partition faulty_power should be zero when faults is empty, partition_idx {}", + orig_partition_idx + ))?; + } let dest_partition_idx = first_dest_partition_idx + i as u64; @@ -137,11 +144,9 @@ impl Deadlines { orig_deadline.total_sectors -= all_sectors; orig_deadline.live_sectors -= live_sectors; - orig_deadline.faulty_power -= &moving_partition.faulty_power; dest_deadline.total_sectors += all_sectors; dest_deadline.live_sectors += live_sectors; - dest_deadline.faulty_power += &moving_partition.faulty_power; orig_partitions.set(orig_partition_idx, Partition::new(store)?)?; dest_partitions.set(dest_partition_idx, moving_partition)?; From 3c6a374bb15dd09766e4b4d7e329c84321c37294 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Mon, 25 Sep 2023 14:31:14 +0800 Subject: [PATCH 46/54] adjust find_sectors_by_expiration for not re-quantizing --- actors/miner/src/expiration_queue.rs | 91 ++++++++++++++++++---------- 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index d6777c9e9..e2eb6d461 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -6,6 +6,7 @@ use std::convert::TryInto; use anyhow::{anyhow, Context}; use cid::Cid; +use fil_actors_runtime::network::EPOCHS_IN_DAY; use fil_actors_runtime::runtime::Policy; use fil_actors_runtime::{ActorDowncast, Array}; use fvm_ipld_amt::{Error as AmtError, ValueMut}; @@ -776,33 +777,53 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { sector_size: SectorSize, sectors: &[SectorOnChainInfo], ) -> anyhow::Result> { - let mut declared_expirations = BTreeMap::::new(); + if sectors.is_empty() { + return Ok(Vec::new()); + } + let mut sectors_by_number = BTreeMap::::new(); let mut all_remaining = BTreeSet::::new(); for sector in sectors { - let q_expiration = self.quant.quantize_up(sector.expiration); - declared_expirations.insert(q_expiration, true); all_remaining.insert(sector.sector_number); sectors_by_number.insert(sector.sector_number, sector); } - let mut expiration_groups = - Vec::::with_capacity(declared_expirations.len()); + let mut expiration_groups = Vec::::with_capacity(sectors.len()); + + for sector in sectors { + // scan [sector.expiration, sector.expiration+EPOCHS_IN_DAY] for active sectors. + // since a sector may have been moved from another deadline, the possible range for an active sector is [sector.expiration, sector.expiration+EPOCHS_IN_DAY] + let end_epoch = (sector.expiration + EPOCHS_IN_DAY) as u64; + self.amt.for_each_while_ranged(Some(sector.expiration as u64), None, |epoch, es| { + if epoch > end_epoch || !all_remaining.contains(§or.sector_number) { + // no need to scan for this sector any more + return Ok(false); + } - for (&expiration, _) in declared_expirations.iter() { - let es = self.may_get(expiration)?; + if !es.on_time_sectors.get(sector.sector_number) { + // continue scan for this sector + return Ok(true); + } - let group = group_expiration_set( - sector_size, - §ors_by_number, - &mut all_remaining, - es, - expiration, - ); - if !group.sector_epoch_set.sectors.is_empty() { + let group = group_expiration_set( + sector_size, + §ors_by_number, + &mut all_remaining, + es, + epoch as ChainEpoch, + ); + if group.sector_epoch_set.sectors.is_empty() { + // by here it's guaranteed this is true: + // all_remaining.contains(§or.sector_number) && es.on_time_sectors.get(sector.sector_number) + // so group.sector_epoch_set.sectors should not be empty + return Err(anyhow!("should never happen")); + } expiration_groups.push(group); - } + + // no need to scan for this sector any more + Ok(false) + })?; } // If sectors remain, traverse next in epoch order. Remaining sectors should be @@ -810,12 +831,6 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { if !all_remaining.is_empty() { self.amt.for_each_while(|epoch, es| { let epoch = epoch as ChainEpoch; - // If this set's epoch is one of our declared epochs, we've already processed it - // in the loop above, so skip processing here. Sectors rescheduled to this epoch - // would have been included in the earlier processing. - if declared_expirations.contains_key(&epoch) { - return Ok(true); - } // Sector should not be found in EarlyExpirations which holds faults. An implicit assumption // of grouping is that it only returns sectors with active power. ExpirationQueue should not @@ -826,7 +841,7 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { sector_size, §ors_by_number, &mut all_remaining, - es.clone(), + es, epoch, ); @@ -911,7 +926,7 @@ fn group_expiration_set( sector_size: SectorSize, sectors: &BTreeMap, include_set: &mut BTreeSet, - es: ExpirationSet, + es: &ExpirationSet, expiration: ChainEpoch, ) -> SectorExpirationSet { let mut sector_numbers = Vec::new(); @@ -927,14 +942,26 @@ fn group_expiration_set( } } - SectorExpirationSet { - sector_epoch_set: SectorEpochSet { - epoch: expiration, - sectors: sector_numbers, - power: total_power, - pledge: total_pledge, - }, - expiration_set: es, + if sector_numbers.is_empty() { + SectorExpirationSet { + sector_epoch_set: SectorEpochSet { + epoch: expiration, + sectors: sector_numbers, + power: total_power, + pledge: total_pledge, + }, + expiration_set: ExpirationSet::default(), + } + } else { + SectorExpirationSet { + sector_epoch_set: SectorEpochSet { + epoch: expiration, + sectors: sector_numbers, + power: total_power, + pledge: total_pledge, + }, + expiration_set: es.clone(), // lazy clone + } } } From 271e3c0605fdbe1f00dd8974ef1d9bbccaa9dfd5 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 26 Sep 2023 13:21:39 +0800 Subject: [PATCH 47/54] add test --- actors/miner/src/expiration_queue.rs | 2 +- actors/miner/tests/move_partitions_test.rs | 331 ++++++++++++++++++++- actors/miner/tests/util.rs | 29 +- 3 files changed, 338 insertions(+), 24 deletions(-) diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index e2eb6d461..a141949f6 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -653,7 +653,7 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { faulty_power: &PowerPair, pledge: &TokenAmount, ) -> anyhow::Result<()> { - let epoch = self.quant.quantize_up(raw_epoch); + let epoch = raw_epoch; let mut expiration_set = self .amt .get(epoch.try_into()?) diff --git a/actors/miner/tests/move_partitions_test.rs b/actors/miner/tests/move_partitions_test.rs index ce31b4635..04e75b71d 100644 --- a/actors/miner/tests/move_partitions_test.rs +++ b/actors/miner/tests/move_partitions_test.rs @@ -1,18 +1,26 @@ use fil_actor_miner::{ deadline_available_for_compaction, deadline_available_for_optimistic_post_dispute, - deadline_is_mutable, new_deadline_info, DeadlineInfo, + deadline_is_mutable, expected_reward_for_power, new_deadline_info, + pledge_penalty_for_termination, qa_power_for_sector, DeadlineInfo, SectorOnChainInfo, State, + INITIAL_PLEDGE_PROJECTION_PERIOD, }; + +use fil_actors_runtime::network::EPOCHS_IN_DAY; use fil_actors_runtime::{ + runtime::Runtime, runtime::{DomainSeparationTag, RuntimePolicy}, test_utils::{expect_abort_contains_message, MockRuntime}, }; use fvm_ipld_bitfield::BitField; use fvm_ipld_encoding::RawBytes; +use fvm_shared::econ::TokenAmount; use fvm_shared::randomness::Randomness; use fvm_shared::{clock::ChainEpoch, error::ExitCode}; +use num_traits::Zero; mod util; use util::*; + const PERIOD_OFFSET: ChainEpoch = 100; fn setup() -> (ActorHarness, MockRuntime) { @@ -106,7 +114,6 @@ fn farthest_possible_to_deadline( #[test] fn fail_to_move_partitions_with_faults_from_safe_epoch() { let (mut h, rt) = setup(); - rt.set_epoch(200); // create 2 sectors in partition 0 let sectors_info = h.commit_and_prove_sectors( @@ -118,12 +125,13 @@ fn fail_to_move_partitions_with_faults_from_safe_epoch() { ); h.advance_and_submit_posts(&rt, §ors_info); + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + // fault sector 1 h.declare_faults(&rt, §ors_info[0..1]); - let partition_id = 0; - let orig_deadline_id = 0; - h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); let to_deadline_id = @@ -148,7 +156,6 @@ fn fail_to_move_partitions_with_faults_from_safe_epoch() { #[test] fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { let (mut h, rt) = setup(); - rt.set_epoch(200); // create 2 sectors in partition 0 let sectors_info = h.commit_and_prove_sectors( @@ -160,12 +167,13 @@ fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { ); h.advance_and_submit_posts(&rt, §ors_info); + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + // fault sector 1 h.declare_faults(&rt, §ors_info[0..1]); - let partition_id = 0; - let orig_deadline_id = 0; - h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, orig_deadline_id)); let dest_deadline_id = @@ -225,7 +233,6 @@ fn fail_to_move_partitions_with_faults_from_unsafe_epoch() { #[test] fn ok_to_move_partitions_from_safe_epoch() { let (mut h, rt) = setup(); - rt.set_epoch(200); // create 2 sectors in partition 0 let sectors_info = h.commit_and_prove_sectors( @@ -237,11 +244,12 @@ fn ok_to_move_partitions_from_safe_epoch() { ); h.advance_and_submit_posts(&rt, §ors_info); - let orig_deadline_id = 0; + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); - let partition_id = 0; let dest_deadline_id = farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); @@ -260,7 +268,6 @@ fn ok_to_move_partitions_from_safe_epoch() { #[test] fn ok_to_move_partitions_from_unsafe_epoch() { let (mut h, rt) = setup(); - rt.set_epoch(200); // create 2 sectors in partition 0 let sectors_info = h.commit_and_prove_sectors( @@ -272,11 +279,12 @@ fn ok_to_move_partitions_from_unsafe_epoch() { ); h.advance_and_submit_posts(&rt, §ors_info); - let orig_deadline_id = 0; + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); h.advance_to_epoch_with_cron(&rt, nearest_unsafe_epoch(&rt, &h, orig_deadline_id)); - let partition_id = 0; let dest_deadline_id = farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); @@ -326,3 +334,296 @@ fn ok_to_move_partitions_from_unsafe_epoch() { h.check_state(&rt); } + +#[test] +fn fault_and_recover_after_move() { + let (mut h, rt) = setup(); + + let sectors_info = h.commit_and_prove_sectors( + &rt, + 2, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + orig_deadline_id, + dest_deadline_id, + bitfield_from_slice(&[partition_id]), + || {}, + ); + assert!(result.is_ok()); + + let st = h.get_state(&rt); + let (dl_idx, p_idx) = st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + assert!(dl_idx == dest_deadline_id); + + h.check_state(&rt); + + // fault and recover + + h.declare_faults(&rt, §ors_info); + + h.declare_recoveries( + &rt, + dl_idx, + p_idx, + BitField::try_from_bits(sectors_info.iter().map(|s| s.sector_number)).unwrap(), + TokenAmount::zero(), + ) + .unwrap(); + + let dl = h.get_deadline(&rt, dl_idx); + let p = dl.load_partition(&rt.store, p_idx).unwrap(); + assert_eq!(p.faults, p.recoveries); + h.check_state(&rt); +} + +#[test] +fn fault_and_terminate_after_move() { + let (mut h, rt) = setup(); + + let sectors_info = h.commit_and_prove_sectors( + &rt, + 1, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + orig_deadline_id, + dest_deadline_id, + bitfield_from_slice(&[partition_id]), + || {}, + ); + assert!(result.is_ok()); + + let st = h.get_state(&rt); + let (dl_idx, _) = st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + assert!(dl_idx == dest_deadline_id); + + h.check_state(&rt); + + // fault and terminate + + h.declare_faults(&rt, §ors_info); + + // A miner will pay the minimum of termination fee and locked funds. Add some locked funds to ensure + // correct fee calculation is used. + h.apply_rewards(&rt, BIG_REWARDS.clone(), TokenAmount::zero()); + let state: State = rt.get_state(); + let initial_locked_funds = state.locked_funds; + + let expected_fee = calc_expected_fee_for_termination(&h, &rt, sectors_info[0].clone()); + let sectors = bitfield_from_slice(&[sectors_info[0].sector_number]); + h.terminate_sectors(&rt, §ors, expected_fee.clone()); + + // expect sector to be marked as terminated and the early termination queue to be empty (having been fully processed) + let state: State = rt.get_state(); + let (_, mut partition) = h.find_sector(&rt, sectors_info[0].sector_number); + let terminated = partition.terminated.get(sectors_info[0].sector_number); + assert!(terminated); + + let (result, _) = partition.pop_early_terminations(rt.store(), 1000).unwrap(); + assert!(result.is_empty()); + + // expect fee to have been unlocked and burnt + assert_eq!(initial_locked_funds - expected_fee, state.locked_funds); + + //expect pledge requirement to have been decremented + assert!(state.initial_pledge.is_zero()); + + h.check_state(&rt); +} + +fn calc_expected_fee_for_termination( + h: &ActorHarness, + rt: &MockRuntime, + sector: SectorOnChainInfo, +) -> TokenAmount { + let sector_power = qa_power_for_sector(sector.seal_proof.sector_size().unwrap(), §or); + let day_reward = expected_reward_for_power( + &h.epoch_reward_smooth, + &h.epoch_qa_power_smooth, + §or_power, + EPOCHS_IN_DAY, + ); + let twenty_day_reward = expected_reward_for_power( + &h.epoch_reward_smooth, + &h.epoch_qa_power_smooth, + §or_power, + INITIAL_PLEDGE_PROJECTION_PERIOD, + ); + let sector_age = *rt.epoch.borrow() - sector.activation; + pledge_penalty_for_termination( + &day_reward, + sector_age, + &twenty_day_reward, + &h.epoch_qa_power_smooth, + §or_power, + &h.epoch_reward_smooth, + &TokenAmount::zero(), + 0, + ) +} + +#[test] +fn directly_terminate_after_move() { + let (mut h, rt) = setup(); + + let sectors_info = h.commit_and_prove_sectors( + &rt, + 1, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + orig_deadline_id, + dest_deadline_id, + bitfield_from_slice(&[partition_id]), + || {}, + ); + assert!(result.is_ok()); + + let st = h.get_state(&rt); + let (dl_idx, _) = st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + assert!(dl_idx == dest_deadline_id); + + h.check_state(&rt); + + // directly terminate + + // A miner will pay the minimum of termination fee and locked funds. Add some locked funds to ensure + // correct fee calculation is used. + h.apply_rewards(&rt, BIG_REWARDS.clone(), TokenAmount::zero()); + let state: State = rt.get_state(); + let initial_locked_funds = state.locked_funds; + + let expected_fee = calc_expected_fee_for_termination(&h, &rt, sectors_info[0].clone()); + let sectors = bitfield_from_slice(&[sectors_info[0].sector_number]); + h.terminate_sectors(&rt, §ors, expected_fee.clone()); + + // expect sector to be marked as terminated and the early termination queue to be empty (having been fully processed) + let state: State = rt.get_state(); + let (_, mut partition) = h.find_sector(&rt, sectors_info[0].sector_number); + let terminated = partition.terminated.get(sectors_info[0].sector_number); + assert!(terminated); + + let (result, _) = partition.pop_early_terminations(rt.store(), 1000).unwrap(); + assert!(result.is_empty()); + + // expect fee to have been unlocked and burnt + assert_eq!(initial_locked_funds - expected_fee, state.locked_funds); + + //expect pledge requirement to have been decremented + assert!(state.initial_pledge.is_zero()); + + h.check_state(&rt); +} + +#[test] +fn fault_and_expire_after_move() { + let (mut h, rt) = setup(); + + let sectors_info = h.commit_and_prove_sectors( + &rt, + 1, + DEFAULT_SECTOR_EXPIRATION, + vec![vec![10], vec![20]], + true, + ); + h.advance_and_submit_posts(&rt, §ors_info); + + let st = h.get_state(&rt); + let (orig_deadline_id, partition_id) = + st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + + h.advance_to_epoch_with_cron(&rt, nearest_safe_epoch(&rt, &h, orig_deadline_id)); + let dest_deadline_id = + farthest_possible_to_deadline(&rt, orig_deadline_id, h.current_deadline(&rt)); + + let result = h.move_partitions( + &rt, + orig_deadline_id, + dest_deadline_id, + bitfield_from_slice(&[partition_id]), + || {}, + ); + assert!(result.is_ok()); + + let st = h.get_state(&rt); + let (dl_idx, partition_id) = st.find_sector(&rt.store, sectors_info[0].sector_number).unwrap(); + assert!(dl_idx == dest_deadline_id); + + h.check_state(&rt); + + // fault and expire + + h.declare_faults(&rt, §ors_info); + + let st = h.get_state(&rt); + let quant = st.quant_spec_for_deadline(rt.policy(), dl_idx); + + let current_deadline = h.current_deadline(&rt); + + let target_deadline = new_deadline_info( + rt.policy(), + if current_deadline.index < orig_deadline_id { + current_deadline.period_start - rt.policy().wpost_proving_period + } else { + current_deadline.period_start + }, + orig_deadline_id, + *rt.epoch.borrow(), + ); + let fault_expiration_epoch = target_deadline.last() + rt.policy.fault_max_age; + let new_expiration = quant.quantize_up(fault_expiration_epoch); + + // assert that new expiration exists + let (_, mut partition) = h.get_deadline_and_partition(&rt, dl_idx, partition_id); + let expiration_set = + partition.pop_expired_sectors(rt.store(), new_expiration - 1, quant).unwrap(); + assert!(expiration_set.is_empty()); + + let expiration_set = partition + .pop_expired_sectors(rt.store(), quant.quantize_up(new_expiration), quant) + .unwrap(); + assert_eq!(expiration_set.len(), 1); + assert!(expiration_set.early_sectors.get(sectors_info[0].sector_number)); + + h.check_state(&rt); +} diff --git a/actors/miner/tests/util.rs b/actors/miner/tests/util.rs index 472695dcc..2926abc9d 100644 --- a/actors/miner/tests/util.rs +++ b/actors/miner/tests/util.rs @@ -2057,8 +2057,18 @@ impl ActorHarness { let mut deal_ids: Vec = Vec::new(); let mut sector_infos: Vec = Vec::new(); + let mut has_active_sector = false; for sector in sectors.iter() { + let (_, partition) = self.find_sector(&rt, sector); + let non_active = partition.terminated.get(sector) + || partition.faults.get(sector) + || partition.unproven.get(sector); + if !non_active { + has_active_sector = true; + } + let sector = self.get_sector(rt, sector); + deal_ids.extend(sector.deal_ids.iter()); sector_infos.push(sector); } @@ -2116,14 +2126,17 @@ impl ActorHarness { raw_byte_delta: -sector_power.raw.clone(), quality_adjusted_delta: -sector_power.qa.clone(), }; - rt.expect_send_simple( - STORAGE_POWER_ACTOR_ADDR, - UPDATE_CLAIMED_POWER_METHOD, - IpldBlock::serialize_cbor(¶ms).unwrap(), - TokenAmount::zero(), - None, - ExitCode::OK, - ); + + if has_active_sector { + rt.expect_send_simple( + STORAGE_POWER_ACTOR_ADDR, + UPDATE_CLAIMED_POWER_METHOD, + IpldBlock::serialize_cbor(¶ms).unwrap(), + TokenAmount::zero(), + None, + ExitCode::OK, + ); + } // create declarations let state: State = rt.get_state(); From d2202ab5d0669a492c332f867e7ac9796992cfd7 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 26 Sep 2023 18:07:07 +0800 Subject: [PATCH 48/54] fix for review --- actors/miner/src/deadline_state.rs | 18 +++++++++--------- actors/miner/src/expiration_queue.rs | 10 +++++++--- actors/miner/src/lib.rs | 10 +++++----- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 8dc04075d..2d61edf86 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -124,6 +124,9 @@ impl Deadlines { if !moving_partition.faults.is_empty() || !moving_partition.unproven.is_empty() { return Err(actor_error!(forbidden, "partition with faults or unproven sectors are not allowed to move, partition_idx {}", orig_partition_idx))?; } + if orig_deadline.early_terminations.get(orig_partition_idx) { + return Err(actor_error!(forbidden, "partition with early terminated sectors are not allowed to move, partition_idx {}", orig_partition_idx))?; + } if !moving_partition.faulty_power.is_zero() { return Err(actor_error!( illegal_state, @@ -134,19 +137,16 @@ impl Deadlines { let dest_partition_idx = first_dest_partition_idx + i as u64; - let all_sectors = moving_partition.sectors.len(); - let live_sectors = moving_partition.live_sectors().len(); - if orig_deadline.early_terminations.get(orig_partition_idx) { - return Err(actor_error!(forbidden, "partition with early terminated sectors are not allowed to move, partition_idx {}", orig_partition_idx))?; - } + // sector_count is both total sector count and total live sector count, since no sector is faulty here. + let sector_count = moving_partition.sectors.len(); // start updating orig/dest `Deadline` here - orig_deadline.total_sectors -= all_sectors; - orig_deadline.live_sectors -= live_sectors; + orig_deadline.total_sectors -= sector_count; + orig_deadline.live_sectors -= sector_count; - dest_deadline.total_sectors += all_sectors; - dest_deadline.live_sectors += live_sectors; + dest_deadline.total_sectors += sector_count; + dest_deadline.live_sectors += sector_count; orig_partitions.set(orig_partition_idx, Partition::new(store)?)?; dest_partitions.set(dest_partition_idx, moving_partition)?; diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index a141949f6..e5e0cd783 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -644,16 +644,16 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { Ok(()) } + /// Note that the `epoch` parameter doesn't quantize, and assumes the entry for the epoch is non-empty. fn remove( &mut self, - raw_epoch: ChainEpoch, + epoch: ChainEpoch, on_time_sectors: &BitField, early_sectors: &BitField, active_power: &PowerPair, faulty_power: &PowerPair, pledge: &TokenAmount, ) -> anyhow::Result<()> { - let epoch = raw_epoch; let mut expiration_set = self .amt .get(epoch.try_into()?) @@ -792,6 +792,10 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { let mut expiration_groups = Vec::::with_capacity(sectors.len()); for sector in sectors { + // precheck to save unnecessary call to `for_each_while_ranged` below + if !all_remaining.contains(§or.sector_number) { + continue; + } // scan [sector.expiration, sector.expiration+EPOCHS_IN_DAY] for active sectors. // since a sector may have been moved from another deadline, the possible range for an active sector is [sector.expiration, sector.expiration+EPOCHS_IN_DAY] let end_epoch = (sector.expiration + EPOCHS_IN_DAY) as u64; @@ -817,7 +821,7 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { // by here it's guaranteed this is true: // all_remaining.contains(§or.sector_number) && es.on_time_sectors.get(sector.sector_number) // so group.sector_epoch_set.sectors should not be empty - return Err(anyhow!("should never happen")); + return Err(anyhow!("there's a bug in `group_expiration_set` function")); } expiration_groups.push(group); diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index e139bebc4..6eac87bf2 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2741,7 +2741,7 @@ impl Actor { ExitCode::USR_ILLEGAL_STATE, format!("failed to load deadline {}", params.orig_deadline), )?; - // only try to do synchronous Window Post verification if the from deadline is in dispute window + // only try to do immediate (non-optimistic) Window Post verification if the from deadline is in dispute window // note that as window post is batched, the best we can do is to verify only those that contains at least one partition being moved. // besides, after verification, the corresponding PostProof is deleted, leaving other PostProof intact. // thus it's necessary that for those moved partitions, there's an empty partition left in place to keep the partitions from re-indexing. @@ -2762,7 +2762,7 @@ impl Actor { )?; // Find the proving period start for the deadline in question. - let target_deadline = + let prev_orig_deadline = nearest_occured_deadline_info(policy, ¤t_deadline, params.orig_deadline); // Load sectors for the dispute. @@ -2796,14 +2796,14 @@ impl Actor { if !partition.faults.is_empty() { return Err(actor_error!( forbidden, - "cannot move partition {}: has faults", + "cannot move partition {}: had faults at last Window PoST", partition_idx ))?; } if !partition.unproven.is_empty() { return Err(actor_error!( forbidden, - "cannot move partition {}: has unproven", + "cannot move partition {}: had unproven at last Window PoST", partition_idx ))?; } @@ -2824,7 +2824,7 @@ impl Actor { if !verify_windowed_post( rt, - target_deadline.challenge, + prev_orig_deadline.challenge, §or_infos, window_proof.proofs.clone(), ) From d276241507407550c3b5cbb101916aac6117480b Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Tue, 26 Sep 2023 22:09:58 +0800 Subject: [PATCH 49/54] add a comment about not re-quantizing when moving expirations_epochs --- actors/miner/src/deadline_state.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index 2d61edf86..d4af2562c 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -153,6 +153,8 @@ impl Deadlines { } // update expirations_epochs Cid of Deadline. + // Note that when moving a partition from `orig_expirations_epochs` to `dest_expirations_epochs`, + // we explicitly keep the `dest_epoch` the same as `orig_epoch`, this is by design of not re-quantizing. { let mut epochs_to_remove = Vec::::new(); let mut orig_expirations_epochs: Array = From b2dc66c4b7b9fbd13b7e97582ad1028bfd38cfc3 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Wed, 27 Sep 2023 09:40:16 +0800 Subject: [PATCH 50/54] minor optimization --- actors/miner/src/expiration_queue.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index e5e0cd783..34152a7a6 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -806,8 +806,8 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { } if !es.on_time_sectors.get(sector.sector_number) { - // continue scan for this sector - return Ok(true); + // continue scan for this sector if not already at end_epoch + return Ok(epoch < end_epoch); } let group = group_expiration_set( From fcc4cc0ed943d5e63f2811c86d0d9dfd1f351b42 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Thu, 28 Sep 2023 10:09:19 +0800 Subject: [PATCH 51/54] avoid scanning the same range twice --- actors/miner/src/expiration_queue.rs | 51 +++++++++++++++------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index 34152a7a6..3765a36e2 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -783,31 +783,36 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { let mut sectors_by_number = BTreeMap::::new(); let mut all_remaining = BTreeSet::::new(); + let mut declared_expirations = BTreeSet::::new(); for sector in sectors { + declared_expirations.insert(sector.expiration); all_remaining.insert(sector.sector_number); sectors_by_number.insert(sector.sector_number, sector); } let mut expiration_groups = Vec::::with_capacity(sectors.len()); - for sector in sectors { - // precheck to save unnecessary call to `for_each_while_ranged` below - if !all_remaining.contains(§or.sector_number) { - continue; - } - // scan [sector.expiration, sector.expiration+EPOCHS_IN_DAY] for active sectors. - // since a sector may have been moved from another deadline, the possible range for an active sector is [sector.expiration, sector.expiration+EPOCHS_IN_DAY] - let end_epoch = (sector.expiration + EPOCHS_IN_DAY) as u64; - self.amt.for_each_while_ranged(Some(sector.expiration as u64), None, |epoch, es| { - if epoch > end_epoch || !all_remaining.contains(§or.sector_number) { - // no need to scan for this sector any more - return Ok(false); - } + let mut old_end = 0i64; + for expiration in declared_expirations.iter() { + // Basically we're scanning [sector.expiration, sector.expiration+EPOCHS_IN_DAY] for active sectors. + // Since a sector may have been moved from another deadline, the possible range for an active sector is [sector.expiration, sector.expiration+EPOCHS_IN_DAY]. + // + // And we're also trying to avoid scanning the same range twice by choosing a proper `start_at`. - if !es.on_time_sectors.get(sector.sector_number) { - // continue scan for this sector if not already at end_epoch - return Ok(epoch < end_epoch); + let start_at = if *expiration > old_end { + *expiration + } else { + // +1 since the range is inclusive + old_end + 1 + }; + let new_end = (expiration + EPOCHS_IN_DAY) as u64; + + // scan range [start_at, new_end] for active sectors of interest + self.amt.for_each_while_ranged(Some(start_at as u64), None, |epoch, es| { + if epoch > new_end { + // no need to scan any more + return Ok(false); } let group = group_expiration_set( @@ -817,17 +822,15 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { es, epoch as ChainEpoch, ); - if group.sector_epoch_set.sectors.is_empty() { - // by here it's guaranteed this is true: - // all_remaining.contains(§or.sector_number) && es.on_time_sectors.get(sector.sector_number) - // so group.sector_epoch_set.sectors should not be empty - return Err(anyhow!("there's a bug in `group_expiration_set` function")); + + if !group.sector_epoch_set.sectors.is_empty() { + expiration_groups.push(group); } - expiration_groups.push(group); - // no need to scan for this sector any more - Ok(false) + Ok(!all_remaining.is_empty()) })?; + + old_end = new_end as i64; } // If sectors remain, traverse next in epoch order. Remaining sectors should be From 7619065c2a6dc0bfa55443f8ffe5728ebed2cbcd Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Thu, 28 Sep 2023 20:57:51 +0800 Subject: [PATCH 52/54] 1. review fix 2. add check for `max_partitions_per_deadline` --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- actors/miner/src/deadline_state.rs | 8 ++++++++ actors/miner/src/expiration_queue.rs | 4 ++-- actors/miner/src/lib.rs | 1 + 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0233c278..71655f629 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2087,9 +2087,9 @@ dependencies = [ [[package]] name = "fvm_ipld_amt" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0b0ee51ca8defa9717a72e1d35c8cbb85bd8320a835911410b63b9a63dffec" +checksum = "5fea333475130094f27ce67809aae3f69eb5247541d835950b7c5da733dbbb34" dependencies = [ "anyhow", "cid 0.10.1", diff --git a/Cargo.toml b/Cargo.toml index c83543736..eb75594d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,7 +120,7 @@ fvm_ipld_encoding = "0.4.0" fvm_ipld_blockstore = "0.2.0" fvm_ipld_hamt = "0.7.0" fvm_ipld_kamt = "0.3.0" -fvm_ipld_amt = { version = "0.6.1" } +fvm_ipld_amt = { version = "0.6.2" } fvm_ipld_bitfield = "0.6.0" # workspace diff --git a/actors/miner/src/deadline_state.rs b/actors/miner/src/deadline_state.rs index d4af2562c..be6ed791f 100644 --- a/actors/miner/src/deadline_state.rs +++ b/actors/miner/src/deadline_state.rs @@ -105,6 +105,7 @@ impl Deadlines { } pub fn move_partitions( + policy: &Policy, store: &BS, orig_deadline: &mut Deadline, dest_deadline: &mut Deadline, @@ -115,6 +116,13 @@ impl Deadlines { // even though we're moving partitions intact, we still need to update orig/dest `Deadline` accordingly. + if dest_partitions.count() + partitions.len() > policy.max_partitions_per_deadline { + return Err(actor_error!( + forbidden, + "partitions in dest_deadline will exceed max_partitions_per_deadline" + ))?; + } + let first_dest_partition_idx = dest_partitions.count(); for (i, orig_partition_idx) in partitions.iter().enumerate() { let moving_partition = orig_partitions diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index 3765a36e2..754cf5a17 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -806,7 +806,7 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { // +1 since the range is inclusive old_end + 1 }; - let new_end = (expiration + EPOCHS_IN_DAY) as u64; + let new_end = (expiration + EPOCHS_IN_DAY - 1) as u64; // scan range [start_at, new_end] for active sectors of interest self.amt.for_each_while_ranged(Some(start_at as u64), None, |epoch, es| { @@ -827,7 +827,7 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { expiration_groups.push(group); } - Ok(!all_remaining.is_empty()) + Ok(epoch < new_end && !all_remaining.is_empty()) })?; old_end = new_end as i64; diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 6eac87bf2..72806b7dc 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2848,6 +2848,7 @@ impl Actor { )?; Deadlines::move_partitions( + policy, store, &mut orig_deadline, &mut dest_deadline, From 6e91261a2ab8f92bc0f1cf5a06a4338c9ca2643a Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Thu, 28 Sep 2023 21:43:49 +0800 Subject: [PATCH 53/54] fix comment --- actors/miner/src/expiration_queue.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actors/miner/src/expiration_queue.rs b/actors/miner/src/expiration_queue.rs index 754cf5a17..207222431 100644 --- a/actors/miner/src/expiration_queue.rs +++ b/actors/miner/src/expiration_queue.rs @@ -795,8 +795,8 @@ impl<'db, BS: Blockstore> ExpirationQueue<'db, BS> { let mut old_end = 0i64; for expiration in declared_expirations.iter() { - // Basically we're scanning [sector.expiration, sector.expiration+EPOCHS_IN_DAY] for active sectors. - // Since a sector may have been moved from another deadline, the possible range for an active sector is [sector.expiration, sector.expiration+EPOCHS_IN_DAY]. + // Basically we're scanning [sector.expiration, sector.expiration+EPOCHS_IN_DAY) for active sectors. + // Since a sector may have been moved from another deadline, the possible range for an active sector is [sector.expiration, sector.expiration+EPOCHS_IN_DAY). // // And we're also trying to avoid scanning the same range twice by choosing a proper `start_at`. From 99642572098400e6bbdff27c5126714781350fce Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Fri, 29 Sep 2023 17:47:40 +0800 Subject: [PATCH 54/54] use with_context_code --- actors/miner/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/actors/miner/src/lib.rs b/actors/miner/src/lib.rs index 72806b7dc..48537f84e 100644 --- a/actors/miner/src/lib.rs +++ b/actors/miner/src/lib.rs @@ -2736,11 +2736,11 @@ impl Actor { "conditions not satisfied for deadline_available_for_move", )?; - let mut orig_deadline = - deadlines.load_deadline(store, params.orig_deadline).context_code( - ExitCode::USR_ILLEGAL_STATE, - format!("failed to load deadline {}", params.orig_deadline), - )?; + let mut orig_deadline = deadlines + .load_deadline(store, params.orig_deadline) + .with_context_code(ExitCode::USR_ILLEGAL_STATE, || { + format!("failed to load deadline {}", params.orig_deadline) + })?; // only try to do immediate (non-optimistic) Window Post verification if the from deadline is in dispute window // note that as window post is batched, the best we can do is to verify only those that contains at least one partition being moved. // besides, after verification, the corresponding PostProof is deleted, leaving other PostProof intact.