From e8d7248093036b042f58cf80b321e3cc0cf857fa Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Thu, 6 May 2021 17:24:09 -0400 Subject: [PATCH 01/11] Implement the simple Lengauer-Tarjan algorithm This replaces the previous implementation with the simple variant of Lengauer-Tarjan, which performs better in the general case. Performance on the keccak benchmark is about equivalent between the two, but we don't see regressions (and indeed see improvements) on other benchmarks, even on a partially optimized implementation. The implementation here follows that of the pseudocode in "Linear-Time Algorithms for Dominators and Related Problems" thesis by Loukas Georgiadis. The next few commits will optimize the implementation as suggested in the thesis. Several related works are cited in the comments within the implementation, as well. Implement the simple Lengauer-Tarjan algorithm This replaces the previous implementation (from #34169), which has not been optimized since, with the simple variant of Lengauer-Tarjan which performs better in the general case. A previous attempt -- not kept in commit history -- attempted a replacement with a bitset-based implementation, but this led to regressions on perf.rust-lang.org benchmarks and equivalent wins for the keccak benchmark, so was rejected. The implementation here follows that of the pseudocode in "Linear-Time Algorithms for Dominators and Related Problems" thesis by Loukas Georgiadis. The next few commits will optimize the implementation as suggested in the thesis. Several related works are cited in the comments within the implementation, as well. On the keccak benchmark, we were previously spending 15% of our cycles computing the NCA / intersect function; this function is quite expensive, especially on modern CPUs, as it chases pointers on every iteration in a tight loop. With this commit, we spend ~0.05% of our time in dominator computation. --- .../src/graph/dominators/mod.rs | 144 +++++++++++++----- .../src/graph/dominators/tests.rs | 11 ++ 2 files changed, 116 insertions(+), 39 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 1cd170599ba51..b1f62636b8977 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -1,9 +1,8 @@ //! Finding the dominators in a control-flow graph. //! -//! Algorithm based on Keith D. Cooper, Timothy J. Harvey, and Ken Kennedy, -//! "A Simple, Fast Dominance Algorithm", -//! Rice Computer Science TS-06-33870, -//! . +//! Algorithm based on Loukas Georgiadis, +//! "Linear-Time Algorithms for Dominators and Related Problems", +//! ftp://ftp.cs.princeton.edu/techreports/2005/737.pdf use super::iterate::reverse_post_order; use super::ControlFlowGraph; @@ -19,6 +18,11 @@ pub fn dominators(graph: G) -> Dominators { dominators_given_rpo(graph, &rpo) } +struct PreOrderFrame { + node: Node, + iter: Iter, +} + fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Dominators { let start_node = graph.start_node(); assert_eq!(rpo[0], start_node); @@ -29,53 +33,115 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin post_order_rank[node] = index; } - let mut immediate_dominators = IndexVec::from_elem_n(None, graph.num_nodes()); - immediate_dominators[start_node] = Some(start_node); - - let mut changed = true; - while changed { - changed = false; - - for &node in &rpo[1..] { - let mut new_idom = None; - for pred in graph.predecessors(node) { - if immediate_dominators[pred].is_some() { - // (*) dominators for `pred` have been calculated - new_idom = Some(if let Some(new_idom) = new_idom { - intersect(&post_order_rank, &immediate_dominators, new_idom, pred) - } else { - pred - }); - } + let mut visited = BitSet::new_empty(graph.num_nodes()); + let mut parent: IndexVec> = + IndexVec::from_elem_n(None, graph.num_nodes()); + let mut pre_order_index: IndexVec> = + IndexVec::from_elem_n(None, graph.num_nodes()); + let mut pre_order_nodes = Vec::with_capacity(rpo.len()); + + let mut stack = vec![PreOrderFrame { + node: graph.start_node(), + iter: graph.successors(graph.start_node()), + }]; + visited.insert(graph.start_node()); + let mut idx = 0; + pre_order_index[graph.start_node()] = Some(0); + idx += 1; + pre_order_nodes.push(graph.start_node()); + + 'recurse: while let Some(frame) = stack.last_mut() { + while let Some(successor) = frame.iter.next() { + if visited.insert(successor) { + parent[successor] = Some(frame.node); + pre_order_index[successor] = Some(idx); + pre_order_nodes.push(successor); + idx += 1; + + stack.push(PreOrderFrame { node: successor, iter: graph.successors(successor) }); + continue 'recurse; } + } + stack.pop(); + } - if new_idom != immediate_dominators[node] { - immediate_dominators[node] = new_idom; - changed = true; - } + let mut ancestor = IndexVec::from_elem_n(None, graph.num_nodes()); + let mut idom = IndexVec::from_elem_n(graph.start_node(), graph.num_nodes()); + let mut semi = IndexVec::from_fn_n(std::convert::identity, graph.num_nodes()); + let mut label = semi.clone(); + let mut bucket = IndexVec::from_elem_n(vec![], graph.num_nodes()); + + for &w in pre_order_nodes[1..].iter().rev() { + semi[w] = w; + for v in graph.predecessors(w) { + let x = eval(&pre_order_index, &mut ancestor, &semi, &mut label, v); + semi[w] = if pre_order_index[semi[w]].unwrap() < pre_order_index[semi[x]].unwrap() { + semi[w] + } else { + semi[x] + }; + } + // semi[w] is now semidominator(w). + + bucket[semi[w]].push(w); + + link(&mut ancestor, &parent, w); + let z = parent[w].unwrap(); + for v in std::mem::take(&mut bucket[z]) { + let y = eval(&pre_order_index, &mut ancestor, &semi, &mut label, v); + idom[v] = if pre_order_index[semi[y]] < pre_order_index[z] { y } else { z }; + } + } + for &w in pre_order_nodes.iter().skip(1) { + if idom[w] != semi[w] { + idom[w] = idom[idom[w]]; + } + } + + let mut immediate_dominators = IndexVec::from_elem_n(None, graph.num_nodes()); + for (node, idom_slot) in immediate_dominators.iter_enumerated_mut() { + if pre_order_index[node].is_some() { + *idom_slot = Some(idom[node]); } } Dominators { post_order_rank, immediate_dominators } } -fn intersect( - post_order_rank: &IndexVec, - immediate_dominators: &IndexVec>, - mut node1: Node, - mut node2: Node, -) -> Node { - while node1 != node2 { - while post_order_rank[node1] < post_order_rank[node2] { - node1 = immediate_dominators[node1].unwrap(); - } +fn eval( + pre_order_index: &IndexVec>, + ancestor: &mut IndexVec>, + semi: &IndexVec, + label: &mut IndexVec, + node: N, +) -> N { + if ancestor[node].is_some() { + compress(pre_order_index, ancestor, semi, label, node); + label[node] + } else { + node + } +} - while post_order_rank[node2] < post_order_rank[node1] { - node2 = immediate_dominators[node2].unwrap(); +fn compress( + pre_order_index: &IndexVec>, + ancestor: &mut IndexVec>, + semi: &IndexVec, + label: &mut IndexVec, + v: N, +) { + let u = ancestor[v].unwrap(); + if ancestor[u].is_some() { + compress(pre_order_index, ancestor, semi, label, u); + if pre_order_index[semi[label[u]]] < pre_order_index[semi[label[v]]] { + label[v] = label[u]; } + ancestor[v] = ancestor[u]; } +} - node1 +fn link(ancestor: &mut IndexVec>, parent: &IndexVec>, w: N) { + ancestor[w] = Some(parent[w].unwrap()); } #[derive(Clone, Debug)] diff --git a/compiler/rustc_data_structures/src/graph/dominators/tests.rs b/compiler/rustc_data_structures/src/graph/dominators/tests.rs index 1160df5186b36..ff31d8f7fdcf6 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/tests.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/tests.rs @@ -32,3 +32,14 @@ fn paper() { assert_eq!(immediate_dominators[5], Some(6)); assert_eq!(immediate_dominators[6], Some(6)); } + +#[test] +fn paper_slt() { + // example from the paper: + let graph = TestGraph::new( + 1, + &[(1, 2), (1, 3), (2, 3), (2, 7), (3, 4), (3, 6), (4, 5), (5, 4), (6, 7), (7, 8), (8, 5)], + ); + + dominators(&graph); +} From c82fe0efb4591f1a39e135152db77a43a6820c51 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 May 2021 14:02:24 -0400 Subject: [PATCH 02/11] Optimization: Merge parent and ancestor arrays As the paper indicates, the unprocessed vertices in the DFS tree and processed vertices are disjoint, and we can use them in the same space, tracking only the index of the split. --- .../src/graph/dominators/mod.rs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index b1f62636b8977..eb6e2b296cd9d 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -65,16 +65,16 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin stack.pop(); } - let mut ancestor = IndexVec::from_elem_n(None, graph.num_nodes()); let mut idom = IndexVec::from_elem_n(graph.start_node(), graph.num_nodes()); let mut semi = IndexVec::from_fn_n(std::convert::identity, graph.num_nodes()); let mut label = semi.clone(); let mut bucket = IndexVec::from_elem_n(vec![], graph.num_nodes()); + let mut lastlinked = None; for &w in pre_order_nodes[1..].iter().rev() { semi[w] = w; for v in graph.predecessors(w) { - let x = eval(&pre_order_index, &mut ancestor, &semi, &mut label, v); + let x = eval(&pre_order_index, &mut parent, lastlinked, &semi, &mut label, v); semi[w] = if pre_order_index[semi[w]].unwrap() < pre_order_index[semi[x]].unwrap() { semi[w] } else { @@ -91,6 +91,10 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin let y = eval(&pre_order_index, &mut ancestor, &semi, &mut label, v); idom[v] = if pre_order_index[semi[y]] < pre_order_index[z] { y } else { z }; } + + // Optimization: We share the parent array between processed and not + // processed elements; lastlinked represents the divider. + lastlinked = Some(w); } for &w in pre_order_nodes.iter().skip(1) { if idom[w] != semi[w] { @@ -111,28 +115,39 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin fn eval( pre_order_index: &IndexVec>, ancestor: &mut IndexVec>, + lastlinked: Option, semi: &IndexVec, label: &mut IndexVec, node: N, ) -> N { - if ancestor[node].is_some() { - compress(pre_order_index, ancestor, semi, label, node); + if is_processed(pre_order_index, node, lastlinked) { + compress(pre_order_index, ancestor, lastlinked, semi, label, node); label[node] } else { node } } +fn is_processed( + pre_order_index: &IndexVec>, + v: N, + lastlinked: Option, +) -> bool { + if let Some(ll) = lastlinked { pre_order_index[v] >= pre_order_index[ll] } else { false } +} + fn compress( pre_order_index: &IndexVec>, ancestor: &mut IndexVec>, + lastlinked: Option, semi: &IndexVec, label: &mut IndexVec, v: N, ) { + assert!(is_processed(pre_order_index, v, lastlinked)); let u = ancestor[v].unwrap(); - if ancestor[u].is_some() { - compress(pre_order_index, ancestor, semi, label, u); + if is_processed(pre_order_index, u, lastlinked) { + compress(pre_order_index, ancestor, lastlinked, semi, label, u); if pre_order_index[semi[label[u]]] < pre_order_index[semi[label[v]]] { label[v] = label[u]; } @@ -140,10 +155,6 @@ fn compress( } } -fn link(ancestor: &mut IndexVec>, parent: &IndexVec>, w: N) { - ancestor[w] = Some(parent[w].unwrap()); -} - #[derive(Clone, Debug)] pub struct Dominators { post_order_rank: IndexVec, From 7379d24ebc1ed0bbd8f33b9767b1390ea1a7177d Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 May 2021 14:05:32 -0400 Subject: [PATCH 03/11] Optimization: process buckets only once --- .../src/graph/dominators/mod.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index eb6e2b296cd9d..99119610f93e6 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -72,6 +72,14 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin let mut lastlinked = None; for &w in pre_order_nodes[1..].iter().rev() { + // Optimization: process buckets just once. We need not explicitly empty + // the bucket here, but mem::take is pretty cheap. + let z = parent[w].unwrap(); + for v in std::mem::take(&mut bucket[z]) { + let y = eval(&pre_order_index, &mut parent, lastlinked, &semi, &mut label, v); + idom[v] = if pre_order_index[semi[y]] < pre_order_index[z] { y } else { z }; + } + semi[w] = w; for v in graph.predecessors(w) { let x = eval(&pre_order_index, &mut parent, lastlinked, &semi, &mut label, v); @@ -85,13 +93,6 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin bucket[semi[w]].push(w); - link(&mut ancestor, &parent, w); - let z = parent[w].unwrap(); - for v in std::mem::take(&mut bucket[z]) { - let y = eval(&pre_order_index, &mut ancestor, &semi, &mut label, v); - idom[v] = if pre_order_index[semi[y]] < pre_order_index[z] { y } else { z }; - } - // Optimization: We share the parent array between processed and not // processed elements; lastlinked represents the divider. lastlinked = Some(w); From 92186cb5c9228f64c7f7c832a0b61aa05e14802d Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 May 2021 14:06:05 -0400 Subject: [PATCH 04/11] Avoid inserting into buckets if not necessary --- .../rustc_data_structures/src/graph/dominators/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 99119610f93e6..074b5a16c12f5 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -91,7 +91,13 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin } // semi[w] is now semidominator(w). - bucket[semi[w]].push(w); + // Optimization: Do not insert into buckets if parent[w] = semi[w], as + // we then immediately know the idom. + if parent[w].unwrap() != semi[w] { + bucket[semi[w]].push(w); + } else { + idom[w] = parent[w].unwrap(); + } // Optimization: We share the parent array between processed and not // processed elements; lastlinked represents the divider. From 7d12767dc590e544d500d034f466a7b2d0d82d57 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 8 May 2021 19:13:11 -0400 Subject: [PATCH 05/11] Use preorder indices for data structures This largely avoids remapping from and to the 'real' indices, with the exception of predecessor lookup and the final merge back, and is conceptually better. --- .../src/graph/dominators/mod.rs | 91 ++++++++----------- 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 074b5a16c12f5..b819b7f419ad9 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -34,60 +34,53 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin } let mut visited = BitSet::new_empty(graph.num_nodes()); - let mut parent: IndexVec> = - IndexVec::from_elem_n(None, graph.num_nodes()); - let mut pre_order_index: IndexVec> = - IndexVec::from_elem_n(None, graph.num_nodes()); - let mut pre_order_nodes = Vec::with_capacity(rpo.len()); + let mut parent: IndexVec> = IndexVec::from_elem_n(None, rpo.len()); - let mut stack = vec![PreOrderFrame { - node: graph.start_node(), - iter: graph.successors(graph.start_node()), - }]; + let mut stack = vec![PreOrderFrame { node: 0, iter: graph.successors(graph.start_node()) }]; visited.insert(graph.start_node()); - let mut idx = 0; - pre_order_index[graph.start_node()] = Some(0); - idx += 1; - pre_order_nodes.push(graph.start_node()); + let mut pre_order_to_real = Vec::with_capacity(rpo.len()); + let mut real_to_pre_order: IndexVec> = + IndexVec::from_elem_n(None, graph.num_nodes()); + pre_order_to_real.push(graph.start_node()); + real_to_pre_order[graph.start_node()] = Some(0); + let mut idx = 1; 'recurse: while let Some(frame) = stack.last_mut() { while let Some(successor) = frame.iter.next() { if visited.insert(successor) { - parent[successor] = Some(frame.node); - pre_order_index[successor] = Some(idx); - pre_order_nodes.push(successor); - idx += 1; + parent[idx] = Some(frame.node); + pre_order_to_real.push(successor); + real_to_pre_order[successor] = Some(idx); - stack.push(PreOrderFrame { node: successor, iter: graph.successors(successor) }); + stack.push(PreOrderFrame { node: idx, iter: graph.successors(successor) }); + idx += 1; continue 'recurse; } } stack.pop(); } - let mut idom = IndexVec::from_elem_n(graph.start_node(), graph.num_nodes()); - let mut semi = IndexVec::from_fn_n(std::convert::identity, graph.num_nodes()); + let mut idom = IndexVec::from_elem_n(0, pre_order_to_real.len()); + let mut semi = IndexVec::from_fn_n(std::convert::identity, pre_order_to_real.len()); let mut label = semi.clone(); - let mut bucket = IndexVec::from_elem_n(vec![], graph.num_nodes()); + let mut bucket = IndexVec::from_elem_n(vec![], pre_order_to_real.len()); let mut lastlinked = None; - for &w in pre_order_nodes[1..].iter().rev() { - // Optimization: process buckets just once. We need not explicitly empty - // the bucket here, but mem::take is pretty cheap. + for w in (1..pre_order_to_real.len()).rev() { + // Optimization: process buckets just once, at the start of the + // iteration. Do not explicitly empty the bucket (even though it will + // not be used again), to save some instructions. let z = parent[w].unwrap(); - for v in std::mem::take(&mut bucket[z]) { - let y = eval(&pre_order_index, &mut parent, lastlinked, &semi, &mut label, v); - idom[v] = if pre_order_index[semi[y]] < pre_order_index[z] { y } else { z }; + for &v in bucket[z].iter() { + let y = eval(&mut parent, lastlinked, &semi, &mut label, v); + idom[v] = if semi[y] < z { y } else { z }; } semi[w] = w; - for v in graph.predecessors(w) { - let x = eval(&pre_order_index, &mut parent, lastlinked, &semi, &mut label, v); - semi[w] = if pre_order_index[semi[w]].unwrap() < pre_order_index[semi[x]].unwrap() { - semi[w] - } else { - semi[x] - }; + for v in graph.predecessors(pre_order_to_real[w]) { + let v = real_to_pre_order[v].unwrap(); + let x = eval(&mut parent, lastlinked, &semi, &mut label, v); + semi[w] = std::cmp::min(semi[w], semi[x]); } // semi[w] is now semidominator(w). @@ -103,59 +96,51 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin // processed elements; lastlinked represents the divider. lastlinked = Some(w); } - for &w in pre_order_nodes.iter().skip(1) { + for w in 1..pre_order_to_real.len() { if idom[w] != semi[w] { idom[w] = idom[idom[w]]; } } let mut immediate_dominators = IndexVec::from_elem_n(None, graph.num_nodes()); - for (node, idom_slot) in immediate_dominators.iter_enumerated_mut() { - if pre_order_index[node].is_some() { - *idom_slot = Some(idom[node]); - } + for (idx, node) in pre_order_to_real.iter().enumerate() { + immediate_dominators[*node] = Some(pre_order_to_real[idom[idx]]); } Dominators { post_order_rank, immediate_dominators } } fn eval( - pre_order_index: &IndexVec>, ancestor: &mut IndexVec>, lastlinked: Option, semi: &IndexVec, label: &mut IndexVec, node: N, ) -> N { - if is_processed(pre_order_index, node, lastlinked) { - compress(pre_order_index, ancestor, lastlinked, semi, label, node); + if is_processed(node, lastlinked) { + compress(ancestor, lastlinked, semi, label, node); label[node] } else { node } } -fn is_processed( - pre_order_index: &IndexVec>, - v: N, - lastlinked: Option, -) -> bool { - if let Some(ll) = lastlinked { pre_order_index[v] >= pre_order_index[ll] } else { false } +fn is_processed(v: N, lastlinked: Option) -> bool { + if let Some(ll) = lastlinked { v >= ll } else { false } } fn compress( - pre_order_index: &IndexVec>, ancestor: &mut IndexVec>, lastlinked: Option, semi: &IndexVec, label: &mut IndexVec, v: N, ) { - assert!(is_processed(pre_order_index, v, lastlinked)); + assert!(is_processed(v, lastlinked)); let u = ancestor[v].unwrap(); - if is_processed(pre_order_index, u, lastlinked) { - compress(pre_order_index, ancestor, lastlinked, semi, label, u); - if pre_order_index[semi[label[u]]] < pre_order_index[semi[label[v]]] { + if is_processed(u, lastlinked) { + compress(ancestor, lastlinked, semi, label, u); + if semi[label[u]] < semi[label[v]] { label[v] = label[u]; } ancestor[v] = ancestor[u]; From 8991002644dc23588ca57c99d33001a13ec24061 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 8 May 2021 19:50:50 -0400 Subject: [PATCH 06/11] Remove separate RPO traversal This integrates the preorder and postorder traversals into one. --- .../src/graph/dominators/mod.rs | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index b819b7f419ad9..043e706c39877 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -4,7 +4,6 @@ //! "Linear-Time Algorithms for Dominators and Related Problems", //! ftp://ftp.cs.princeton.edu/techreports/2005/737.pdf -use super::iterate::reverse_post_order; use super::ControlFlowGraph; use rustc_index::vec::{Idx, IndexVec}; use std::cmp::Ordering; @@ -12,38 +11,26 @@ use std::cmp::Ordering; #[cfg(test)] mod tests; -pub fn dominators(graph: G) -> Dominators { - let start_node = graph.start_node(); - let rpo = reverse_post_order(&graph, start_node); - dominators_given_rpo(graph, &rpo) -} - struct PreOrderFrame { node: Node, iter: Iter, } -fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Dominators { - let start_node = graph.start_node(); - assert_eq!(rpo[0], start_node); - +pub fn dominators(graph: G) -> Dominators { // compute the post order index (rank) for each node let mut post_order_rank = IndexVec::from_elem_n(0, graph.num_nodes()); - for (index, node) in rpo.iter().rev().cloned().enumerate() { - post_order_rank[node] = index; - } - let mut visited = BitSet::new_empty(graph.num_nodes()); - let mut parent: IndexVec> = IndexVec::from_elem_n(None, rpo.len()); + let mut parent: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); let mut stack = vec![PreOrderFrame { node: 0, iter: graph.successors(graph.start_node()) }]; visited.insert(graph.start_node()); - let mut pre_order_to_real = Vec::with_capacity(rpo.len()); + let mut pre_order_to_real = Vec::with_capacity(graph.num_nodes()); let mut real_to_pre_order: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); pre_order_to_real.push(graph.start_node()); real_to_pre_order[graph.start_node()] = Some(0); let mut idx = 1; + let mut post_order_idx = 0; 'recurse: while let Some(frame) = stack.last_mut() { while let Some(successor) = frame.iter.next() { @@ -57,6 +44,9 @@ fn dominators_given_rpo(graph: G, rpo: &[G::Node]) -> Domin continue 'recurse; } } + post_order_rank[pre_order_to_real[frame.node]] = post_order_idx; + post_order_idx += 1; + stack.pop(); } From 345ada0e1b17d21157b3963121c8238d39c763c0 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 May 2021 12:56:58 -0400 Subject: [PATCH 07/11] Optimize: reuse the real-to-preorder mapping as the visited set --- compiler/rustc_data_structures/src/graph/dominators/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 043e706c39877..9d864057c85f6 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -19,11 +19,9 @@ struct PreOrderFrame { pub fn dominators(graph: G) -> Dominators { // compute the post order index (rank) for each node let mut post_order_rank = IndexVec::from_elem_n(0, graph.num_nodes()); - let mut visited = BitSet::new_empty(graph.num_nodes()); let mut parent: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); let mut stack = vec![PreOrderFrame { node: 0, iter: graph.successors(graph.start_node()) }]; - visited.insert(graph.start_node()); let mut pre_order_to_real = Vec::with_capacity(graph.num_nodes()); let mut real_to_pre_order: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); @@ -34,10 +32,10 @@ pub fn dominators(graph: G) -> Dominators { 'recurse: while let Some(frame) = stack.last_mut() { while let Some(successor) = frame.iter.next() { - if visited.insert(successor) { + if real_to_pre_order[successor].is_none() { + real_to_pre_order[successor] = Some(idx); parent[idx] = Some(frame.node); pre_order_to_real.push(successor); - real_to_pre_order[successor] = Some(idx); stack.push(PreOrderFrame { node: idx, iter: graph.successors(successor) }); idx += 1; From cc63ec32fb1921ce51425c90cded1f62f7971487 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 May 2021 18:59:52 -0400 Subject: [PATCH 08/11] Use variables rather than lengths directly --- .../src/graph/dominators/mod.rs | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 9d864057c85f6..e67986ae85c7a 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -27,18 +27,19 @@ pub fn dominators(graph: G) -> Dominators { IndexVec::from_elem_n(None, graph.num_nodes()); pre_order_to_real.push(graph.start_node()); real_to_pre_order[graph.start_node()] = Some(0); - let mut idx = 1; let mut post_order_idx = 0; 'recurse: while let Some(frame) = stack.last_mut() { while let Some(successor) = frame.iter.next() { if real_to_pre_order[successor].is_none() { - real_to_pre_order[successor] = Some(idx); - parent[idx] = Some(frame.node); + let pre_order_idx = pre_order_to_real.len(); + + real_to_pre_order[successor] = Some(pre_order_idx); + parent[pre_order_idx] = Some(frame.node); pre_order_to_real.push(successor); + stack + .push(PreOrderFrame { node: pre_order_idx, iter: graph.successors(successor) }); - stack.push(PreOrderFrame { node: idx, iter: graph.successors(successor) }); - idx += 1; continue 'recurse; } } @@ -48,13 +49,15 @@ pub fn dominators(graph: G) -> Dominators { stack.pop(); } - let mut idom = IndexVec::from_elem_n(0, pre_order_to_real.len()); - let mut semi = IndexVec::from_fn_n(std::convert::identity, pre_order_to_real.len()); + let reachable_vertices = pre_order_to_real.len(); + + let mut idom = IndexVec::from_elem_n(0, reachable_vertices); + let mut semi = IndexVec::from_fn_n(std::convert::identity, reachable_vertices); let mut label = semi.clone(); - let mut bucket = IndexVec::from_elem_n(vec![], pre_order_to_real.len()); + let mut bucket = IndexVec::from_elem_n(vec![], reachable_vertices); let mut lastlinked = None; - for w in (1..pre_order_to_real.len()).rev() { + for w in (1..reachable_vertices).rev() { // Optimization: process buckets just once, at the start of the // iteration. Do not explicitly empty the bucket (even though it will // not be used again), to save some instructions. @@ -84,7 +87,7 @@ pub fn dominators(graph: G) -> Dominators { // processed elements; lastlinked represents the divider. lastlinked = Some(w); } - for w in 1..pre_order_to_real.len() { + for w in 1..reachable_vertices { if idom[w] != semi[w] { idom[w] = idom[idom[w]]; } From 2b6305977219af72d445c4014bbcbdf136e581f4 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 May 2021 19:10:17 -0400 Subject: [PATCH 09/11] Create newtype around the pre order index --- .../src/graph/dominators/mod.rs | 73 +++++++++++-------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index e67986ae85c7a..e1a32d53e9277 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -11,39 +11,45 @@ use std::cmp::Ordering; #[cfg(test)] mod tests; -struct PreOrderFrame { - node: Node, +struct PreOrderFrame { + pre_order_idx: PreorderIndex, iter: Iter, } +rustc_index::newtype_index! { + struct PreorderIndex { .. } +} + pub fn dominators(graph: G) -> Dominators { // compute the post order index (rank) for each node let mut post_order_rank = IndexVec::from_elem_n(0, graph.num_nodes()); - let mut parent: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); + let mut parent: IndexVec> = + IndexVec::from_elem_n(None, graph.num_nodes()); - let mut stack = vec![PreOrderFrame { node: 0, iter: graph.successors(graph.start_node()) }]; - let mut pre_order_to_real = Vec::with_capacity(graph.num_nodes()); - let mut real_to_pre_order: IndexVec> = + let mut stack = vec![PreOrderFrame { + pre_order_idx: PreorderIndex::new(0), + iter: graph.successors(graph.start_node()), + }]; + let mut pre_order_to_real: IndexVec = + IndexVec::with_capacity(graph.num_nodes()); + let mut real_to_pre_order: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); pre_order_to_real.push(graph.start_node()); - real_to_pre_order[graph.start_node()] = Some(0); + real_to_pre_order[graph.start_node()] = Some(PreorderIndex::new(0)); let mut post_order_idx = 0; 'recurse: while let Some(frame) = stack.last_mut() { while let Some(successor) = frame.iter.next() { if real_to_pre_order[successor].is_none() { - let pre_order_idx = pre_order_to_real.len(); - + let pre_order_idx = pre_order_to_real.push(successor); real_to_pre_order[successor] = Some(pre_order_idx); - parent[pre_order_idx] = Some(frame.node); - pre_order_to_real.push(successor); - stack - .push(PreOrderFrame { node: pre_order_idx, iter: graph.successors(successor) }); + parent[pre_order_idx] = Some(frame.pre_order_idx); + stack.push(PreOrderFrame { pre_order_idx, iter: graph.successors(successor) }); continue 'recurse; } } - post_order_rank[pre_order_to_real[frame.node]] = post_order_idx; + post_order_rank[pre_order_to_real[frame.pre_order_idx]] = post_order_idx; post_order_idx += 1; stack.pop(); @@ -51,13 +57,13 @@ pub fn dominators(graph: G) -> Dominators { let reachable_vertices = pre_order_to_real.len(); - let mut idom = IndexVec::from_elem_n(0, reachable_vertices); + let mut idom = IndexVec::from_elem_n(PreorderIndex::new(0), reachable_vertices); let mut semi = IndexVec::from_fn_n(std::convert::identity, reachable_vertices); let mut label = semi.clone(); let mut bucket = IndexVec::from_elem_n(vec![], reachable_vertices); let mut lastlinked = None; - for w in (1..reachable_vertices).rev() { + for w in (PreorderIndex::new(1)..PreorderIndex::new(reachable_vertices)).rev() { // Optimization: process buckets just once, at the start of the // iteration. Do not explicitly empty the bucket (even though it will // not be used again), to save some instructions. @@ -87,27 +93,28 @@ pub fn dominators(graph: G) -> Dominators { // processed elements; lastlinked represents the divider. lastlinked = Some(w); } - for w in 1..reachable_vertices { + for w in PreorderIndex::new(1)..PreorderIndex::new(reachable_vertices) { if idom[w] != semi[w] { idom[w] = idom[idom[w]]; } } let mut immediate_dominators = IndexVec::from_elem_n(None, graph.num_nodes()); - for (idx, node) in pre_order_to_real.iter().enumerate() { + for (idx, node) in pre_order_to_real.iter_enumerated() { immediate_dominators[*node] = Some(pre_order_to_real[idom[idx]]); } Dominators { post_order_rank, immediate_dominators } } -fn eval( - ancestor: &mut IndexVec>, - lastlinked: Option, - semi: &IndexVec, - label: &mut IndexVec, - node: N, -) -> N { +#[inline] +fn eval( + ancestor: &mut IndexVec>, + lastlinked: Option, + semi: &IndexVec, + label: &mut IndexVec, + node: PreorderIndex, +) -> PreorderIndex { if is_processed(node, lastlinked) { compress(ancestor, lastlinked, semi, label, node); label[node] @@ -116,16 +123,18 @@ fn eval( } } -fn is_processed(v: N, lastlinked: Option) -> bool { +#[inline] +fn is_processed(v: PreorderIndex, lastlinked: Option) -> bool { if let Some(ll) = lastlinked { v >= ll } else { false } } -fn compress( - ancestor: &mut IndexVec>, - lastlinked: Option, - semi: &IndexVec, - label: &mut IndexVec, - v: N, +#[inline] +fn compress( + ancestor: &mut IndexVec>, + lastlinked: Option, + semi: &IndexVec, + label: &mut IndexVec, + v: PreorderIndex, ) { assert!(is_processed(v, lastlinked)); let u = ancestor[v].unwrap(); From 31874800702537252a2d1450d4b3c6e2d2321c22 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Mon, 10 May 2021 10:50:08 -0400 Subject: [PATCH 10/11] Avoid using Option where values are always Some --- .../src/graph/dominators/mod.rs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index e1a32d53e9277..104a4119f3265 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -23,8 +23,11 @@ rustc_index::newtype_index! { pub fn dominators(graph: G) -> Dominators { // compute the post order index (rank) for each node let mut post_order_rank = IndexVec::from_elem_n(0, graph.num_nodes()); - let mut parent: IndexVec> = - IndexVec::from_elem_n(None, graph.num_nodes()); + + // We allocate capacity for the full set of nodes, because most of the time + // most of the nodes *are* reachable. + let mut parent: IndexVec = + IndexVec::with_capacity(graph.num_nodes()); let mut stack = vec![PreOrderFrame { pre_order_idx: PreorderIndex::new(0), @@ -35,6 +38,7 @@ pub fn dominators(graph: G) -> Dominators { let mut real_to_pre_order: IndexVec> = IndexVec::from_elem_n(None, graph.num_nodes()); pre_order_to_real.push(graph.start_node()); + parent.push(PreorderIndex::new(0)); // the parent of the root node is the root for now. real_to_pre_order[graph.start_node()] = Some(PreorderIndex::new(0)); let mut post_order_idx = 0; @@ -43,7 +47,7 @@ pub fn dominators(graph: G) -> Dominators { if real_to_pre_order[successor].is_none() { let pre_order_idx = pre_order_to_real.push(successor); real_to_pre_order[successor] = Some(pre_order_idx); - parent[pre_order_idx] = Some(frame.pre_order_idx); + parent.push(frame.pre_order_idx); stack.push(PreOrderFrame { pre_order_idx, iter: graph.successors(successor) }); continue 'recurse; @@ -67,7 +71,7 @@ pub fn dominators(graph: G) -> Dominators { // Optimization: process buckets just once, at the start of the // iteration. Do not explicitly empty the bucket (even though it will // not be used again), to save some instructions. - let z = parent[w].unwrap(); + let z = parent[w]; for &v in bucket[z].iter() { let y = eval(&mut parent, lastlinked, &semi, &mut label, v); idom[v] = if semi[y] < z { y } else { z }; @@ -83,10 +87,10 @@ pub fn dominators(graph: G) -> Dominators { // Optimization: Do not insert into buckets if parent[w] = semi[w], as // we then immediately know the idom. - if parent[w].unwrap() != semi[w] { + if parent[w] != semi[w] { bucket[semi[w]].push(w); } else { - idom[w] = parent[w].unwrap(); + idom[w] = parent[w]; } // Optimization: We share the parent array between processed and not @@ -109,7 +113,7 @@ pub fn dominators(graph: G) -> Dominators { #[inline] fn eval( - ancestor: &mut IndexVec>, + ancestor: &mut IndexVec, lastlinked: Option, semi: &IndexVec, label: &mut IndexVec, @@ -130,14 +134,14 @@ fn is_processed(v: PreorderIndex, lastlinked: Option) -> bool { #[inline] fn compress( - ancestor: &mut IndexVec>, + ancestor: &mut IndexVec, lastlinked: Option, semi: &IndexVec, label: &mut IndexVec, v: PreorderIndex, ) { assert!(is_processed(v, lastlinked)); - let u = ancestor[v].unwrap(); + let u = ancestor[v]; if is_processed(u, lastlinked) { compress(ancestor, lastlinked, semi, label, u); if semi[label[u]] < semi[label[v]] { From 15483ccf9d0a0befb472e4dd3a1bfba754a1cd11 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Mon, 10 May 2021 15:50:50 -0400 Subject: [PATCH 11/11] Annotate comments onto the LT algorithm --- .../src/graph/dominators/mod.rs | 104 +++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs index 104a4119f3265..0d2ae115cb0d2 100644 --- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -2,7 +2,12 @@ //! //! Algorithm based on Loukas Georgiadis, //! "Linear-Time Algorithms for Dominators and Related Problems", -//! ftp://ftp.cs.princeton.edu/techreports/2005/737.pdf +//! +//! +//! Additionally useful is the original Lengauer-Tarjan paper on this subject, +//! "A Fast Algorithm for Finding Dominators in a Flowgraph" +//! Thomas Lengauer and Robert Endre Tarjan. +//! use super::ControlFlowGraph; use rustc_index::vec::{Idx, IndexVec}; @@ -42,6 +47,14 @@ pub fn dominators(graph: G) -> Dominators { real_to_pre_order[graph.start_node()] = Some(PreorderIndex::new(0)); let mut post_order_idx = 0; + // Traverse the graph, collecting a number of things: + // + // * Preorder mapping (to it, and back to the actual ordering) + // * Postorder mapping (used exclusively for rank_partial_cmp on the final product) + // * Parents for each vertex in the preorder tree + // + // These are all done here rather than through one of the 'standard' + // graph traversals to help make this fast. 'recurse: while let Some(frame) = stack.last_mut() { while let Some(successor) = frame.iter.next() { if real_to_pre_order[successor].is_none() { @@ -67,26 +80,95 @@ pub fn dominators(graph: G) -> Dominators { let mut bucket = IndexVec::from_elem_n(vec![], reachable_vertices); let mut lastlinked = None; + // We loop over vertices in reverse preorder. This implements the pseudocode + // of the simple Lengauer-Tarjan algorithm. A few key facts are noted here + // which are helpful for understanding the code (full proofs and such are + // found in various papers, including one cited at the top of this file). + // + // For each vertex w (which is not the root), + // * semi[w] is a proper ancestor of the vertex w (i.e., semi[w] != w) + // * idom[w] is an ancestor of semi[w] (i.e., idom[w] may equal semi[w]) + // + // An immediate dominator of w (idom[w]) is a vertex v where v dominates w + // and every other dominator of w dominates v. (Every vertex except the root has + // a unique immediate dominator.) + // + // A semidominator for a given vertex w (semi[w]) is the vertex v with minimum + // preorder number such that there exists a path from v to w in which all elements (other than w) have + // preorder numbers greater than w (i.e., this path is not the tree path to + // w). for w in (PreorderIndex::new(1)..PreorderIndex::new(reachable_vertices)).rev() { // Optimization: process buckets just once, at the start of the // iteration. Do not explicitly empty the bucket (even though it will // not be used again), to save some instructions. + // + // The bucket here contains the vertices whose semidominator is the + // vertex w, which we are guaranteed to have found: all vertices who can + // be semidominated by w must have a preorder number exceeding w, so + // they have been placed in the bucket. + // + // We compute a partial set of immediate dominators here. let z = parent[w]; for &v in bucket[z].iter() { + // This uses the result of Lemma 5 from section 2 from the original + // 1979 paper, to compute either the immediate or relative dominator + // for a given vertex v. + // + // eval returns a vertex y, for which semi[y] is minimum among + // vertices semi[v] +> y *> v. Note that semi[v] = z as we're in the + // z bucket. + // + // Given such a vertex y, semi[y] <= semi[v] and idom[y] = idom[v]. + // If semi[y] = semi[v], though, idom[v] = semi[v]. + // + // Using this, we can either set idom[v] to be: + // * semi[v] (i.e. z), if semi[y] is z + // * idom[y], otherwise + // + // We don't directly set to idom[y] though as it's not necessarily + // known yet. The second preorder traversal will cleanup by updating + // the idom for any that were missed in this pass. let y = eval(&mut parent, lastlinked, &semi, &mut label, v); idom[v] = if semi[y] < z { y } else { z }; } + // This loop computes the semi[w] for w. semi[w] = w; for v in graph.predecessors(pre_order_to_real[w]) { let v = real_to_pre_order[v].unwrap(); + + // eval returns a vertex x from which semi[x] is minimum among + // vertices semi[v] +> x *> v. + // + // From Lemma 4 from section 2, we know that the semidominator of a + // vertex w is the minimum (by preorder number) vertex of the + // following: + // + // * direct predecessors of w with preorder number less than w + // * semidominators of u such that u > w and there exists (v, w) + // such that u *> v + // + // This loop therefore identifies such a minima. Note that any + // semidominator path to w must have all but the first vertex go + // through vertices numbered greater than w, so the reverse preorder + // traversal we are using guarantees that all of the information we + // might need is available at this point. + // + // The eval call will give us semi[x], which is either: + // + // * v itself, if v has not yet been processed + // * A possible 'best' semidominator for w. let x = eval(&mut parent, lastlinked, &semi, &mut label, v); semi[w] = std::cmp::min(semi[w], semi[x]); } - // semi[w] is now semidominator(w). + // semi[w] is now semidominator(w) and won't change any more. // Optimization: Do not insert into buckets if parent[w] = semi[w], as // we then immediately know the idom. + // + // If we don't yet know the idom directly, then push this vertex into + // our semidominator's bucket, where it will get processed at a later + // stage to compute its immediate dominator. if parent[w] != semi[w] { bucket[semi[w]].push(w); } else { @@ -97,6 +179,14 @@ pub fn dominators(graph: G) -> Dominators { // processed elements; lastlinked represents the divider. lastlinked = Some(w); } + + // Finalize the idoms for any that were not fully settable during initial + // traversal. + // + // If idom[w] != semi[w] then we know that we've stored vertex y from above + // into idom[w]. It is known to be our 'relative dominator', which means + // that it's one of w's ancestors and has the same immediate dominator as w, + // so use that idom. for w in PreorderIndex::new(1)..PreorderIndex::new(reachable_vertices) { if idom[w] != semi[w] { idom[w] = idom[idom[w]]; @@ -111,6 +201,16 @@ pub fn dominators(graph: G) -> Dominators { Dominators { post_order_rank, immediate_dominators } } +/// Evaluate the link-eval virtual forest, providing the currently minimum semi +/// value for the passed `node` (which may be itself). +/// +/// This maintains that for every vertex v, `label[v]` is such that: +/// +/// ```text +/// semi[eval(v)] = min { semi[label[u]] | root_in_forest(v) +> u *> v } +/// ``` +/// +/// where `+>` is a proper ancestor and `*>` is just an ancestor. #[inline] fn eval( ancestor: &mut IndexVec,