diff --git a/src/librustc_mir/lib.rs b/src/librustc_mir/lib.rs index 5e42ba3279027..3712cfd89e83d 100644 --- a/src/librustc_mir/lib.rs +++ b/src/librustc_mir/lib.rs @@ -26,6 +26,7 @@ Rust MIR: a lowered representation of Rust. Also: an experiment! #![feature(try_blocks)] #![feature(associated_type_bounds)] #![feature(range_is_empty)] +#![feature(step_trait)] #![feature(stmt_expr_attributes)] #![feature(trait_alias)] #![recursion_limit = "256"] diff --git a/src/librustc_mir/transform/fragment_locals.rs b/src/librustc_mir/transform/fragment_locals.rs new file mode 100644 index 0000000000000..00051243e2b4c --- /dev/null +++ b/src/librustc_mir/transform/fragment_locals.rs @@ -0,0 +1,397 @@ +use crate::transform::{MirPass, MirSource}; +use rustc::mir::visit::{MutVisitor, NonUseContext, PlaceContext, Visitor}; +use rustc::mir::*; +use rustc::ty::layout::VariantIdx; +use rustc::ty::util::IntTypeExt; +use rustc::ty::{self, Ty, TyCtxt}; +use rustc_index::vec::IndexVec; +use rustc_span::Span; +use std::collections::BTreeMap; +use std::iter::Step; +use std::ops::Range; + +pub struct FragmentLocals; + +impl MirPass<'tcx> for FragmentLocals { + fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut BodyAndCache<'tcx>) { + let mut collector = FragmentTreeCollector { + tcx, + locals: body.local_decls.iter().map(|decl| FragmentTree::new(decl.ty)).collect(), + }; + + // Can't fragment return and arguments. + collector.locals[RETURN_PLACE].make_opaque(); + for arg in body.args_iter() { + collector.locals[arg].make_opaque(); + } + + collector.visit_body(read_only!(body)); + + let replacements = collector + .locals + .iter_enumerated_mut() + .map(|(local, root)| { + // Don't rename locals that are entirely opaque. + match root.kind { + FragmentTreeNodeKind::OpaqueLeaf { .. } => local..local.add_one(), + FragmentTreeNodeKind::Nested(_) => { + let source_info = body.local_decls[local].source_info; + let first = body.local_decls.next_index(); + root.assign_locals(&mut body.local_decls, source_info); + first..body.local_decls.next_index() + } + } + }) + .collect::>>(); + + // Expand `Storage{Live,Dead}` statements to refer to the replacement locals. + for bb in body.basic_blocks_mut() { + bb.expand_statements(|stmt| { + let (local, is_live) = match stmt.kind { + StatementKind::StorageLive(local) => (local, true), + StatementKind::StorageDead(local) => (local, false), + _ => return None, + }; + let range = replacements[local].clone(); + // FIXME(eddyb) `Range` should itself be iterable. + let range = (range.start.as_u32()..range.end.as_u32()).map(Local::from_u32); + let source_info = stmt.source_info; + Some(range.map(move |new_local| Statement { + source_info, + kind: if is_live { + StatementKind::StorageLive(new_local) + } else { + StatementKind::StorageDead(new_local) + }, + })) + }); + } + drop(replacements); + + // Lastly, replace all the opaque nodes with their new locals. + let mut replacer = FragmentTreeReplacer { tcx, span: body.span, locals: collector.locals }; + replacer.visit_body(body); + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum Fragment { + Discriminant, + Field(Option, Field), +} + +struct FragmentTree<'tcx> { + ty: Ty<'tcx>, + kind: FragmentTreeNodeKind<'tcx>, +} + +// FIXME(eddyb) find a shorter name for this +enum FragmentTreeNodeKind<'tcx> { + /// This node needs to remain compact, e.g. due to accesses / borrows. + OpaqueLeaf { replacement_local: Option }, + + /// This node can be fragmented into separate locals for its fields. + Nested(BTreeMap>), +} + +impl FragmentTree<'tcx> { + fn new(ty: Ty<'tcx>) -> Self { + let mut node = FragmentTree { ty, kind: FragmentTreeNodeKind::Nested(BTreeMap::new()) }; + + if let ty::Adt(adt_def, _) = ty.kind { + // Unions have (observably) overlapping members, so don't fragment them. + if adt_def.is_union() { + node.make_opaque(); + } + } + + node + } + + fn fragment(&mut self, fragment: Fragment, ty: Ty<'tcx>) -> Option<&mut Self> { + match self.kind { + FragmentTreeNodeKind::Nested(ref mut fragments) => { + Some(fragments.entry(fragment).or_insert_with(|| FragmentTree::new(ty))) + } + FragmentTreeNodeKind::OpaqueLeaf { .. } => None, + } + } + + fn discriminant(&mut self, tcx: TyCtxt<'tcx>) -> Option<&mut Self> { + match self.ty.kind { + ty::Adt(adt_def, _) if adt_def.is_enum() => { + let discr_ty = adt_def.repr.discr_type().to_ty(tcx); + self.fragment(Fragment::Discriminant, discr_ty) + } + _ => None, + } + } + + fn make_opaque(&mut self) { + if let FragmentTreeNodeKind::Nested(_) = self.kind { + self.kind = FragmentTreeNodeKind::OpaqueLeaf { replacement_local: None }; + } + } + + fn project( + mut self: &'a mut Self, + mut proj_elems: &'tcx [PlaceElem<'tcx>], + ) -> (&'a mut Self, &'tcx [PlaceElem<'tcx>]) { + let mut variant_index = None; + while let [elem, rest @ ..] = proj_elems { + if let FragmentTreeNodeKind::OpaqueLeaf { .. } = self.kind { + break; + } + + match *elem { + ProjectionElem::Field(f, ty) => { + let field = Fragment::Field(variant_index, f); + // FIXME(eddyb) use `self.fragment(field)` post-Polonius(?). + match self.kind { + FragmentTreeNodeKind::Nested(ref mut fragments) => { + self = fragments.entry(field).or_insert_with(|| FragmentTree::new(ty)); + } + FragmentTreeNodeKind::OpaqueLeaf { .. } => unreachable!(), + } + } + + ProjectionElem::Downcast(..) => {} + + // FIXME(eddyb) support indexing by constants. + ProjectionElem::ConstantIndex { .. } | ProjectionElem::Subslice { .. } | + // Can't support without alias analysis. + ProjectionElem::Index(_) | ProjectionElem::Deref => { + // If we can't project, we must be opaque. + self.make_opaque(); + break; + } + } + + proj_elems = rest; + variant_index = match *elem { + ProjectionElem::Downcast(_, v) => Some(v), + _ => None, + }; + } + + (self, proj_elems) + } + + fn assign_locals( + &mut self, + local_decls: &mut IndexVec>, + source_info: SourceInfo, + ) { + match self.kind { + FragmentTreeNodeKind::OpaqueLeaf { ref mut replacement_local } => { + let mut decl = LocalDecl::new_internal(self.ty, source_info.span); + decl.source_info = source_info; + *replacement_local = Some(local_decls.push(decl)); + } + FragmentTreeNodeKind::Nested(ref mut fragments) => { + for fragment in fragments.values_mut() { + fragment.assign_locals(local_decls, source_info); + } + } + } + } +} + +struct FragmentTreeCollector<'tcx> { + tcx: TyCtxt<'tcx>, + locals: IndexVec>, +} + +impl FragmentTreeCollector<'tcx> { + fn place_node(&'a mut self, place: &Place<'tcx>) -> Option<&'a mut FragmentTree<'tcx>> { + let (node, proj_elems) = self.locals[place.local].project(place.projection); + if proj_elems.is_empty() { Some(node) } else { None } + } +} + +impl Visitor<'tcx> for FragmentTreeCollector<'tcx> { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _: Location) { + // Borrows of fields might be used to access the entire local, + // by unsafe code, so it's better for the time being to remain + // conservative, until such uses have been definitely deemed UB. + if context.is_borrow() { + self.locals[place.local].make_opaque(); + } + + if let Some(node) = self.place_node(place) { + if context.is_use() { + node.make_opaque(); + } + + // FIXME(eddyb) implement debuginfo support for fragmented locals. + if let PlaceContext::NonUse(NonUseContext::VarDebugInfo) = context { + node.make_opaque(); + } + } + } + + // Special-case `(Set)Discriminant(place)` to only mark `Fragment::Discriminant` as opaque. + fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { + let tcx = self.tcx; + + if let Rvalue::Discriminant(ref place) = *rvalue { + if let Some(node) = self.place_node(place) { + if let Some(discr) = node.discriminant(tcx) { + discr.make_opaque(); + } + } + } else { + self.super_rvalue(rvalue, location); + } + } + + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + let tcx = self.tcx; + + if let StatementKind::SetDiscriminant { ref place, .. } = statement.kind { + if let Some(node) = self.place_node(place) { + if let Some(discr) = node.discriminant(tcx) { + discr.make_opaque(); + } + } + } else { + self.super_statement(statement, location); + } + } +} + +struct FragmentTreeReplacer<'tcx> { + tcx: TyCtxt<'tcx>, + span: Span, + locals: IndexVec>, +} + +impl FragmentTreeReplacer<'tcx> { + fn replace( + &mut self, + place: &Place<'tcx>, + ) -> Option, &mut FragmentTree<'tcx>>> { + let base_node = &mut self.locals[place.local]; + + // Avoid identity replacements, which would re-intern projections. + if let FragmentTreeNodeKind::OpaqueLeaf { replacement_local: None } = base_node.kind { + return None; + } + + let (node, proj_elems) = base_node.project(place.projection); + + Some(match node.kind { + FragmentTreeNodeKind::OpaqueLeaf { replacement_local } => Ok(Place { + local: replacement_local.expect("missing replacement"), + projection: self.tcx.intern_place_elems(proj_elems), + }), + + // HACK(eddyb) this only exists to support `(Set)Discriminant` below. + FragmentTreeNodeKind::Nested(_) => { + assert_eq!(proj_elems, &[]); + + Err(node) + } + }) + } +} + +impl MutVisitor<'tcx> for FragmentTreeReplacer<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, _: Location) { + if let Some(place_replacement) = self.replace(place) { + match place_replacement { + Ok(place_replacement) => *place = place_replacement, + // HACK(eddyb) this only exists to support `(Set)Discriminant` below. + Err(_) => unreachable!(), + } + } + } + + // Special-case `(Set)Discriminant(place)` to use `discr_local` for `place`. + fn visit_rvalue(&mut self, rvalue: &mut Rvalue<'tcx>, location: Location) { + let tcx = self.tcx; + let span = self.span; + + if let Rvalue::Discriminant(ref mut place) = rvalue { + if let Some(place_replacement) = self.replace(place) { + match place_replacement { + Ok(place_replacement) => *place = place_replacement, + Err(node) => { + let discr = if let Some(discr) = node.discriminant(tcx) { + let discr = match discr.kind { + FragmentTreeNodeKind::OpaqueLeaf { replacement_local } => { + replacement_local + } + FragmentTreeNodeKind::Nested(_) => unreachable!(), + }; + Operand::Copy(Place::from(discr.expect("missing discriminant"))) + } else { + // Non-enums don't have discriminants other than `0u8`. + let discr_value = ty::Const::from_bits( + tcx, + 0, + ty::ParamEnv::empty().and(tcx.types.u8), + ); + Operand::Constant(box Constant { + span, + user_ty: None, + literal: discr_value, + }) + }; + *rvalue = Rvalue::Use(discr); + } + } + } + } else { + self.super_rvalue(rvalue, location); + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + self.span = statement.source_info.span; + + let tcx = self.tcx; + let span = self.span; + + if let StatementKind::SetDiscriminant { ref mut place, variant_index } = statement.kind { + if let Some(place_replacement) = self.replace(place) { + match place_replacement { + Ok(place_replacement) => **place = place_replacement, + Err(node) => { + if let Some(discr) = node.discriminant(tcx) { + let discr_ty = discr.ty; + let discr_local = match discr.kind { + FragmentTreeNodeKind::OpaqueLeaf { replacement_local } => { + replacement_local + } + FragmentTreeNodeKind::Nested(_) => unreachable!(), + }; + let discr_place = + Place::from(discr_local.expect("missing discriminant")); + let discr_value = ty::Const::from_bits( + tcx, + node.ty.discriminant_for_variant(tcx, variant_index).unwrap().val, + ty::ParamEnv::empty().and(discr_ty), + ); + let discr_rvalue = Rvalue::Use(Operand::Constant(box Constant { + span, + user_ty: None, + literal: discr_value, + })); + statement.kind = StatementKind::Assign(box (discr_place, discr_rvalue)); + } else { + // Non-enums don't have discriminants to set. + statement.kind = StatementKind::Nop; + } + } + } + } + } else { + self.super_statement(statement, location); + } + } +} diff --git a/src/librustc_mir/transform/mod.rs b/src/librustc_mir/transform/mod.rs index 3c37eccc1843b..169df061064c2 100644 --- a/src/librustc_mir/transform/mod.rs +++ b/src/librustc_mir/transform/mod.rs @@ -24,6 +24,7 @@ pub mod deaggregator; pub mod dump_mir; pub mod elaborate_drops; pub mod erase_regions; +pub mod fragment_locals; pub mod generator; pub mod inline; pub mod instcombine; @@ -311,6 +312,14 @@ fn run_optimization_passes<'tcx>( &const_prop::ConstProp, &simplify_branches::SimplifyBranches::new("after-const-prop"), &deaggregator::Deaggregator, + // FIXME(eddyb) move this (and `Deaggregator` above) to happen + // before `ConstProp`, to maximize its effectiveness. + // NB: it's not known yet if `ConstProp` can create new + // opportunities for this optimization, or any others. + // Perhaps we should do a crater run erroring if any optimizations + // performed better with `ConstProp` running right before them + // (this would require a lot of MIR cloning, but seems doable). + &fragment_locals::FragmentLocals, ©_prop::CopyPropagation, &simplify_branches::SimplifyBranches::new("after-copy-prop"), &remove_noop_landing_pads::RemoveNoopLandingPads, diff --git a/src/test/incremental/hashes/enum_constructors.rs b/src/test/incremental/hashes/enum_constructors.rs index 575b2e92966ea..10ec262e3ccd0 100644 --- a/src/test/incremental/hashes/enum_constructors.rs +++ b/src/test/incremental/hashes/enum_constructors.rs @@ -96,7 +96,7 @@ pub fn change_constructor_path_struct_like() { } #[cfg(not(cfail1))] -#[rustc_clean(cfg="cfail2", except="HirBody,optimized_mir,mir_built,typeck_tables_of")] +#[rustc_clean(cfg="cfail2", except="HirBody,mir_built,typeck_tables_of")] #[rustc_clean(cfg="cfail3")] pub fn change_constructor_path_struct_like() { let _ = Enum2::Struct { @@ -119,7 +119,7 @@ pub fn change_constructor_variant_struct_like() { } #[cfg(not(cfail1))] -#[rustc_clean(cfg="cfail2", except="HirBody,optimized_mir,mir_built")] +#[rustc_clean(cfg="cfail2", except="HirBody,mir_built")] #[rustc_clean(cfg="cfail3")] pub fn change_constructor_variant_struct_like() { let _ = Enum2::Struct2 { @@ -197,7 +197,7 @@ pub fn change_constructor_path_tuple_like() { #[cfg(not(cfail1))] #[rustc_clean( cfg="cfail2", - except="HirBody,optimized_mir,mir_built,typeck_tables_of" + except="HirBody,mir_built,typeck_tables_of" )] #[rustc_clean(cfg="cfail3")] pub fn change_constructor_path_tuple_like() { @@ -215,7 +215,7 @@ pub fn change_constructor_variant_tuple_like() { #[cfg(not(cfail1))] #[rustc_clean( cfg="cfail2", - except="HirBody,optimized_mir,mir_built,typeck_tables_of" + except="HirBody,mir_built,typeck_tables_of" )] #[rustc_clean(cfg="cfail3")] pub fn change_constructor_variant_tuple_like() { @@ -278,7 +278,7 @@ pub fn change_constructor_path_c_like() { } #[cfg(not(cfail1))] -#[rustc_clean(cfg="cfail2", except="HirBody,optimized_mir,mir_built,typeck_tables_of")] +#[rustc_clean(cfg="cfail2", except="HirBody,mir_built,typeck_tables_of")] #[rustc_clean(cfg="cfail3")] pub fn change_constructor_path_c_like() { let _ = Clike2::B; @@ -293,7 +293,7 @@ pub fn change_constructor_variant_c_like() { } #[cfg(not(cfail1))] -#[rustc_clean(cfg="cfail2", except="HirBody,optimized_mir,mir_built")] +#[rustc_clean(cfg="cfail2", except="HirBody,mir_built")] #[rustc_clean(cfg="cfail3")] pub fn change_constructor_variant_c_like() { let _ = Clike::C; diff --git a/src/test/mir-opt/simplify-arm-identity.rs b/src/test/mir-opt/simplify-arm-identity.rs index a8fa64255fb9a..06d1bea969745 100644 --- a/src/test/mir-opt/simplify-arm-identity.rs +++ b/src/test/mir-opt/simplify-arm-identity.rs @@ -28,20 +28,21 @@ fn main() { // StorageLive(_1); // ((_1 as Foo).0: u8) = const 0u8; // discriminant(_1) = 0; -// StorageLive(_2); +// StorageLive(_6); +// StorageLive(_7); // _3 = discriminant(_1); // switchInt(move _3) -> [0isize: bb3, 1isize: bb1, otherwise: bb2]; // } // bb1: { -// ((_2 as Foo).0: u8) = const 0u8; -// discriminant(_2) = 0; +// _7 = const 0u8; +// _6 = const 0isize; // goto -> bb4; // } // ... // bb3: { // _4 = ((_1 as Foo).0: u8); -// ((_2 as Foo).0: u8) = move _4; -// discriminant(_2) = 0; +// _7 = move _4; +// _6 = const 0isize; // goto -> bb4; // } // ... @@ -54,20 +55,21 @@ fn main() { // StorageLive(_1); // ((_1 as Foo).0: u8) = const 0u8; // discriminant(_1) = 0; -// StorageLive(_2); +// StorageLive(_6); +// StorageLive(_7); // _3 = discriminant(_1); // switchInt(move _3) -> [0isize: bb3, 1isize: bb1, otherwise: bb2]; // } // bb1: { -// ((_2 as Foo).0: u8) = const 0u8; -// discriminant(_2) = 0; +// _7 = const 0u8; +// _6 = const 0isize; // goto -> bb4; // } // ... // bb3: { // _4 = ((_1 as Foo).0: u8); -// ((_2 as Foo).0: u8) = move _4; -// discriminant(_2) = 0; +// _7 = move _4; +// _6 = const 0isize; // goto -> bb4; // } // ...