Skip to content

Commit

Permalink
[engine] rm python graphmaker; create dot formatted display (#4295)
Browse files Browse the repository at this point in the history
### Problem

Printing rule graphs has not been converted to Rust. This can make it hard to visualize what the rule sets look like. Also, the Python rule graph construction is effectively dead code already.

### Solution

Port rule graph visualization and visualization based testing to the rust side of the engine implementation.

### Result

Now we're setup to use the RuleGraph from the Rust side for functions as described in #4303 and #4304.
  • Loading branch information
baroquebobcat authored Mar 7, 2017
1 parent 8aeadcd commit 45507e2
Show file tree
Hide file tree
Showing 12 changed files with 650 additions and 947 deletions.
2 changes: 1 addition & 1 deletion src/python/pants/build_graph/build_file_address_mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from pants.build_graph.address_lookup_error import AddressLookupError
from pants.build_graph.address_mapper import AddressMapper
from pants.build_graph.build_file_parser import BuildFileParser
from pants.util.dirutil import fast_relpath, longest_dir_prefix, join_specs
from pants.util.dirutil import fast_relpath, join_specs, longest_dir_prefix


logger = logging.getLogger(__name__)
Expand Down
6 changes: 6 additions & 0 deletions src/python/pants/engine/addressable.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ class Exactly(TypeConstraint):
def satisfied_by_type(self, obj_type):
return obj_type in self._types

def graph_str(self):
if len(self.types) == 1:
return self.types[0].__name__
else:
return repr(self)


class SubclassesOf(TypeConstraint):
"""Objects of the exact type as well as any sub-types are allowed."""
Expand Down
603 changes: 3 additions & 600 deletions src/python/pants/engine/rules.py

Large diffs are not rendered by default.

36 changes: 33 additions & 3 deletions src/python/pants/engine/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from pants.build_graph.address import Address, BuildFileAddress
from pants.engine.addressable import SubclassesOf
from pants.engine.fs import FileContent, FilesContent, Path, PathGlobs, Snapshot
from pants.engine.isolated_process import create_snapshot_singletons, _Snapshots
from pants.engine.isolated_process import _Snapshots, create_snapshot_singletons
from pants.engine.nodes import Return, Throw
from pants.engine.rules import RuleIndex
from pants.engine.selectors import (Select, SelectDependencies, SelectLiteral, SelectProjection,
Expand Down Expand Up @@ -84,14 +84,17 @@ def graph_trace(self):
yield line.rstrip()

def assert_ruleset_valid(self):
listed = list(TypeId(self._to_id(t)) for t in self.root_subject_types)
root_type_ids = self._root_type_ids()

raw_value = self._native.lib.validator_run(self._scheduler, listed, len(listed))
raw_value = self._native.lib.validator_run(self._scheduler, root_type_ids, len(root_type_ids))
value = self._from_value(raw_value)

if isinstance(value, Exception):
raise ValueError(str(value))

def _root_type_ids(self):
return list(TypeId(self._to_id(t)) for t in sorted(self.root_subject_types))

def _to_value(self, obj):
return self._native.context.to_value(obj)

Expand Down Expand Up @@ -198,6 +201,33 @@ def _register_tasks(self, tasks):
def visualize_graph_to_file(self, filename):
self._native.lib.graph_visualize(self._scheduler, bytes(filename))

def rule_graph_visualization(self):
root_type_ids = self._root_type_ids()

with temporary_file_path() as path:
self._native.lib.rule_graph_visualize(
self._scheduler,
root_type_ids,
len(root_type_ids),
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()

def rule_subgraph_visualization(self, root_subject_type, product_type):
root_type_id = TypeId(self._to_id(root_subject_type))

product_type_id = TypeConstraint(self._to_id(constraint_for(product_type)))
with temporary_file_path() as path:
self._native.lib.rule_subgraph_visualize(
self._scheduler,
root_type_id,
product_type_id,
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()

def invalidate(self, filenames):
filenames_buf = self._native.context.utf8_buf_buf(filenames)
return self._native.lib.graph_invalidate(self._scheduler, filenames_buf)
Expand Down
4 changes: 4 additions & 0 deletions src/python/pants/engine/subsystem/native.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@
typedef void ExternContext;
// On the rust side the integration is defined in externs.rs
typedef void (*extern_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
typedef Key (*extern_key_for)(ExternContext*, Value*);
typedef Value (*extern_val_for)(ExternContext*, Key*);
Expand Down Expand Up @@ -196,6 +197,9 @@
Value validator_run(RawScheduler*, TypeId*, uint64_t);
void rule_graph_visualize(RawScheduler*, TypeId*, uint64_t, char*);
void rule_subgraph_visualize(RawScheduler*, TypeId, TypeConstraint, char*);
void nodes_destroy(RawNodes*);
'''
)
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/engine/subsystem/native_engine_version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c99656dc56811ab5e99f1bbc312950d2c6a2432e
00bd3b7dfb8532b371806c6c38c52338eae0d82a
4 changes: 4 additions & 0 deletions src/rust/engine/src/externs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ pub fn project_str(value: &Value, field: &str) -> String {
val_to_str(&name_val)
}

pub fn key_to_str(key: &Key) -> String {
val_to_str(&val_for(key))
}

pub fn id_to_str(digest: Id) -> String {
with_externs(|e| {
(e.id_to_str)(e.context, digest).to_string().unwrap_or_else(|e| {
Expand Down
70 changes: 67 additions & 3 deletions src/rust/engine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@ extern crate tar;
extern crate tempdir;

use std::ffi::CStr;
use std::fs::File;
use std::io;
use std::mem;
use std::os::raw;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::Arc;


use context::Core;
use core::{Function, Key, TypeConstraint, TypeId, Value};
use externs::{
Expand Down Expand Up @@ -60,7 +63,7 @@ use externs::{
with_vec,
};
use nodes::Failure;
use rule_graph::{GraphMaker, RootSubjectTypes};
use rule_graph::{GraphMaker, RuleGraph};
use scheduler::{RootResult, Scheduler, ExecutionStat};
use tasks::Tasks;
use types::Types;
Expand Down Expand Up @@ -495,7 +498,7 @@ pub extern fn validator_run(
with_scheduler(scheduler_ptr, |raw| {
with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| {
let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks,
RootSubjectTypes { subject_types: subject_types.clone() });
subject_types.clone());
let graph = graph_maker.full_graph();

match graph.validate() {
Expand All @@ -510,6 +513,67 @@ pub extern fn validator_run(
})
}

#[no_mangle]
pub extern fn rule_graph_visualize(
scheduler_ptr: *mut RawScheduler,
subject_types_ptr: *mut TypeId,
subject_types_len: u64,
path_ptr: *const raw::c_char
) {
with_scheduler(scheduler_ptr, |raw| {
with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| {
let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() };
let path = PathBuf::from(path_str);

let graph = graph_full(raw, subject_types);
write_to_file(path.as_path(), &graph).unwrap_or_else(|e| {
println!("Failed to visualize to {}: {:?}", path.display(), e);
});
})
})
}

#[no_mangle]
pub extern fn rule_subgraph_visualize(
scheduler_ptr: *mut RawScheduler,
subject_type: TypeId,
product_type: TypeConstraint,
path_ptr: *const raw::c_char
) {
with_scheduler(scheduler_ptr, |raw| {
let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() };
let path = PathBuf::from(path_str);

let graph = graph_sub(raw, subject_type, product_type);
write_to_file(path.as_path(), &graph).unwrap_or_else(|e| {
println!("Failed to visualize to {}: {:?}", path.display(), e);
});
})
}


fn graph_full(raw: &mut RawScheduler, subject_types: &Vec<TypeId>) -> RuleGraph {
let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks,
subject_types.clone());
graph_maker.full_graph()
}

fn graph_sub(
raw: &mut RawScheduler,
subject_type: TypeId,
product_type: TypeConstraint
) -> RuleGraph {
let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks,
vec![subject_type.clone()]);
graph_maker.sub_graph(&subject_type, &product_type)
}

fn write_to_file(path: &Path, graph: &RuleGraph) -> io::Result<()> {
let file = File::create(path)?;
let mut f = io::BufWriter::new(file);
graph.visualize(&mut f)
}

fn with_scheduler<F, T>(scheduler_ptr: *mut RawScheduler, f: F) -> T
where F: FnOnce(&mut RawScheduler)->T {
let mut scheduler = unsafe { Box::from_raw(scheduler_ptr) };
Expand Down
Loading

0 comments on commit 45507e2

Please sign in to comment.