Merge branch 'master' into feature/nightly-2019-06-20

README.md

@@ -33,7 +33,10 @@ C2Rust requires LLVM 6, 7, or 8 with its corresponding clang compiler and librar
 - **Arch Linux:**
 
         pacman -S base-devel llvm clang cmake openssl
+        
+- **NixOS / nix:**
 
+        nix-shell
 
 - **OS X:** XCode command-line tools and recent LLVM (we recommend the Homebrew version) are required.
 
@@ -68,6 +71,17 @@ On Gentoo, you need to point the build system to the location of `libclang.so`
 
 If you have trouble with building and installing, or want to build from the latest master, the [developer docs](docs/README-developers.md#building-with-system-llvm-libraries) provide more details on the build system.
 
+### Installing from Git
+
+If you'd like to check our recently developed features or you urgently require a bugfixed version of c2rust
+you can install it directly from Git:
+
+    cargo +nightly-2019-04-12 install --git https://github.com/immunant/c2rust.git c2rust
+
+Please note that the master branch is under constant development and you may expirience issues or crashes.
+
+You should also set `LLVM_CONFIG_PATH` accordingly if required as described above.
+
 ## Translating C to Rust
 
 To translate C files specified in `compile_commands.json` (see below), run the `c2rust` tool with the `transpile` subcommand:
@@ -140,6 +154,10 @@ If you have [bear](https://github.com/rizsotto/Bear) installed, it can be used s
 
     bear <build command>
 
+## Contact
+To report issues with translation or refactoring, please use our [Issue Tracker](https://github.com/immunant/c2rust/issues).
+
+To reach the development team, join our [discord channel](https://discord.gg/ANSrTuu) or email us at [c2rust@immunant.com](mailto:c2rust@immunant.com).
 
 ## FAQ
 
@@ -151,12 +169,6 @@ We run the C preprocessor before translation to Rust. This specializes the code
 
 The translator and refactoring tool support both macOS and Linux. Other features, such as cross checking the functionality between C and Rust code, are currently limited to Linux hosts. 
 
-
-## Contact
-To report issues with the translation, please use our Issue Tracker.
-
-The development team can be reached by email at c2rust@immunant.com.
-
 ## Acknowledgements and Licensing
 
 This material is available under the BSD-3 style license as found in the

c2rust-ast-builder/src/builder.rs

@@ -1733,14 +1733,14 @@ impl Builder {
         )
     }
 
-    pub fn use_multiple_item<Pa, I>(self, path: Pa, inner: Vec<I>) -> P<Item>
+    pub fn use_multiple_item<Pa, I, It>(self, path: Pa, inner: It) -> P<Item>
     where
         Pa: Make<Path>,
         I: Make<Ident>,
+        It: Iterator<Item = I>,
     {
         let path = path.make(&self);
         let inner_trees = inner
-            .into_iter()
             .map(|i| {
                 (
                     UseTree {

c2rust-ast-exporter/src/AstExporter.cpp

@@ -1,3 +1,4 @@
+#include <algorithm>
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
@@ -437,7 +438,9 @@ class TranslateASTVisitor final
     TypeEncoder typeEncoder;
     CborEncoder *encoder;
     Preprocessor &PP;
-    std::unordered_map<string, uint64_t> filenames;
+    std::vector<std::pair<string, SourceLocation>> files;
+    // Mapping from SourceManager FileID to index in files
+    DenseMap<FileID, size_t> file_id_mapping;
     std::set<std::pair<void *, ASTEntryTag>> exportedTags;
     std::unordered_map<MacroInfo*, MacroExpansionInfo> macros;
 
@@ -656,25 +659,37 @@ class TranslateASTVisitor final
                                  std::unordered_map<void *, QualType> *sugared,
                                  Preprocessor &PP)
         : Context(Context), typeEncoder(Context, encoder, sugared, this),
-          encoder(encoder), PP(PP) {}
+          encoder(encoder), PP(PP),
+          files{{"", {}}} {}
 
     // Override the default behavior of the RecursiveASTVisitor
     bool shouldVisitImplicitCode() const { return true; }
 
     // Return the filenames as a vector. Indices correspond to file IDs.
-    std::vector<string> getFilenames() const {
-        // Store filenames in order
-        std::vector<string> ordered_filenames(filenames.size());
-
-        for (auto const &kv : filenames) {
-            ordered_filenames[kv.second] = kv.first;
-        }
-
-        return ordered_filenames;
+    const std::vector<std::pair<string, SourceLocation>> &getFiles() {
+        // Iterate file include locations until fix point
+        auto &manager = Context->getSourceManager();
+        size_t size;
+        do {
+            size = files.size();
+            for (auto const &file : files) {
+                getExporterFileId(manager.getFileID(file.second), false);
+            }
+        } while (size != files.size());
+        return files;
     }
 
     void encodeMacros() {
-        for (auto &I : macros) {
+        // Sort macros by source location
+        std::vector<std::pair<MacroInfo *, MacroExpansionInfo>> macro_vec(
+            macros.begin(), macros.end());
+        std::sort(macro_vec.begin(), macro_vec.end(),
+                  [](const std::pair<MacroInfo *, MacroExpansionInfo> &a,
+                     const std::pair<MacroInfo *, MacroExpansionInfo> &b) {
+                      return a.first->getDefinitionLoc() <
+                             b.first->getDefinitionLoc();
+                  });
+        for (auto &I : macro_vec) {
             auto &Mac = I.first;
             auto &Info = I.second;
             auto Name = Info.Name;
@@ -706,8 +721,23 @@ class TranslateASTVisitor final
 
         auto line = manager.getPresumedLineNumber(loc);
         auto col = manager.getPresumedColumnNumber(loc);
-        auto fileid = manager.getFileID(loc);
-        auto entry = manager.getFileEntryForID(fileid);
+        auto fileid = getExporterFileId(manager.getFileID(loc), isVaList);
+
+        cbor_encode_uint(enc, fileid);
+        cbor_encode_uint(enc, line);
+        cbor_encode_uint(enc, col);
+    }
+
+    uint64_t getExporterFileId(FileID id, bool isVaList) {
+        if (id.isInvalid())
+            return 0;
+
+        auto file = file_id_mapping.find(id);
+        if (file != file_id_mapping.end())
+            return file->second;
+
+        auto &manager = Context->getSourceManager();
+        auto entry = manager.getFileEntryForID(id);
 
         auto filename = string("?");
         if (entry)
@@ -716,12 +746,10 @@ class TranslateASTVisitor final
         if (filename == "?" && isVaList)
             filename = "vararg";
 
-        auto pair =
-            filenames.insert(std::make_pair(filename, filenames.size()));
-
-        cbor_encode_uint(enc, pair.first->second);
-        cbor_encode_uint(enc, line);
-        cbor_encode_uint(enc, col);
+        auto new_id = files.size();
+        files.push_back(std::make_pair(filename, manager.getIncludeLoc(id)));
+        file_id_mapping[id] = new_id;
+        return new_id;
     }
 
     //
@@ -817,8 +845,9 @@ class TranslateASTVisitor final
         LLVM_DEBUG(dbgs() << "Visit ");
         LLVM_DEBUG(DS->dumpColor());
 
-        // We copy only canonical decls and VarDecl's that are extern/local. For
-        // more on the latter, see the comment at the top of `VisitVarDecl`
+        // We copy only canonical decls and VarDecl's that are extern/local.
+        // For more on the latter, see the comment at the top of
+        // `VisitVarDecl`
         std::vector<void *> childIds;
         std::copy_if(DS->decl_begin(), DS->decl_end(),
                      std::back_inserter(childIds), [](Decl *decl) {
@@ -2201,18 +2230,29 @@ class TranslateConsumer : public clang::ASTConsumer {
             cbor_encoder_close_container(&outer, &array);
 
             // 3. Encode all of the visited file names
-            auto filenames = visitor.getFilenames();
-            cbor_encoder_create_array(&outer, &array, filenames.size());
-            for (auto const &name : filenames) {
-                cbor_encode_string(&array, name);
+            auto files = visitor.getFiles();
+            cbor_encoder_create_array(&outer, &array, files.size());
+            for (auto const &file : files) {
+                CborEncoder entry;
+                cbor_encoder_create_array(&array, &entry, 2);
+                cbor_encode_string(&entry, file.first);
+                if (file.second.isValid()) {
+                    CborEncoder locEntry;
+                    cbor_encoder_create_array(&entry, &locEntry, 3);
+                    visitor.encodeSourcePos(&locEntry, file.second);
+                    cbor_encoder_close_container(&entry, &locEntry);
+                } else {
+                    cbor_encode_null(&entry);
+                }
+                cbor_encoder_close_container(&array, &entry);
             }
             cbor_encoder_close_container(&outer, &array);
 
             // 4. Emit comments as array of arrays. Each comment is represented
             // as an array of source position followed by comment string.
             //
-            // Getting all comments will require processing the file with
-            // -fparse-all-comments !
+            // Getting all comments requires -fparse-all-comments (see
+            // augment_argv())!
             auto comments = Context.getRawCommentList().getComments();
             cbor_encoder_create_array(&outer, &array, comments.size());
             for (auto comment : comments) {
@@ -2222,7 +2262,7 @@ class TranslateConsumer : public clang::ASTConsumer {
                 SourceLocation loc = comment->getLocStart();
 #else
                 SourceLocation loc = comment->getBeginLoc();
-#endif                                                // CLANG_VERSION_MAJOR
+#endif // CLANG_VERSION_MAJOR
                 visitor.encodeSourcePos(&entry, loc); // emits 3 values
                 auto raw_text = comment->getRawText(Context.getSourceManager());
                 cbor_encode_byte_string(&entry, raw_text.bytes_begin(),

c2rust-ast-exporter/src/clang_ast.rs

@@ -22,14 +22,18 @@ impl LRValue {
     }
 }
 
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq)]
+pub struct SrcLoc {
+    pub fileid: u64,
+    pub line: u64,
+    pub column: u64,
+}
+
 #[derive(Debug, Clone)]
 pub struct AstNode {
     pub tag: ASTEntryTag,
     pub children: Vec<Option<u64>>,
-    pub fileid: u64,
-    pub line: u64,
-    pub column: u64,
-    pub file_path: Option<PathBuf>,
+    pub loc: SrcLoc,
     pub type_id: Option<u64>,
     pub rvalue: LRValue,
 
@@ -48,12 +52,16 @@ pub struct TypeNode {
 
 #[derive(Debug, Clone)]
 pub struct CommentNode {
-    pub fileid: u64,
-    pub line: u64,
-    pub column: u64,
+    pub loc: SrcLoc,
     pub string: String,
 }
 
+#[derive(Debug, Clone)]
+pub struct SrcFile {
+    pub path: Option<PathBuf>,
+    pub include_loc: Option<SrcLoc>,
+}
+
 impl TypeNode {
     // Masks used to decode the IDs given to type nodes
     pub const ID_MASK: u64 = !0b111;
@@ -68,6 +76,7 @@ pub struct AstContext {
     pub type_nodes: HashMap<u64, TypeNode>,
     pub top_nodes: Vec<u64>,
     pub comments: Vec<CommentNode>,
+    pub files: Vec<SrcFile>,
 }
 
 pub fn expect_opt_str(val: &Value) -> Option<Option<&str>> {
@@ -103,22 +112,34 @@ pub fn process(items: Value) -> error::Result<AstContext> {
     let mut types: HashMap<u64, TypeNode> = HashMap::new();
     let mut comments: Vec<CommentNode> = vec![];
 
-    let (all_nodes, top_nodes, file_paths, raw_comments): (
+    let (all_nodes, top_nodes, files, raw_comments): (
         Vec<Vec<Value>>,
         Vec<u64>,
-        Vec<String>,
+        Vec<(String, Option<(u64, u64, u64)>)>,
         Vec<(u64, u64, u64, ByteBuf)>,
     ) = from_value(items)?;
 
     for (fileid, line, column, bytes) in raw_comments {
         comments.push(CommentNode {
-            fileid,
-            line,
-            column,
+            loc: SrcLoc { fileid, line, column },
             string: String::from_utf8_lossy(&bytes).to_string(),
         })
     }
 
+    let files = files.into_iter()
+        .map(|(path, loc)| {
+            let path = match path.as_str() {
+                "" => None,
+                "?" => None,
+                path => Some(Path::new(path).to_path_buf()),
+            };
+            SrcFile {
+                path,
+                include_loc: loc.map(|(fileid, line, column)| SrcLoc { fileid, line, column }),
+            }
+        })
+        .collect::<Vec<_>>();
+
     for entry in all_nodes {
         let entry_id = entry[0].as_u64().unwrap();
         let tag = entry[1].as_u64().unwrap();
@@ -133,11 +154,6 @@ pub fn process(items: Value) -> error::Result<AstContext> {
 
             let type_id: Option<u64> = expect_opt_u64(&entry[6]).unwrap();
             let fileid = entry[3].as_u64().unwrap();
-            let file_path = match file_paths[fileid as usize].as_str() {
-                "" => None,
-                "?" => None,
-                path => Some(Path::new(path).to_path_buf()),
-            };
 
             let macro_expansions = entry[8]
                 .as_array()
@@ -149,11 +165,12 @@ pub fn process(items: Value) -> error::Result<AstContext> {
             let node = AstNode {
                 tag: import_ast_tag(tag),
                 children,
-                fileid,
-                line: entry[4].as_u64().unwrap(),
-                column: entry[5].as_u64().unwrap(),
+                loc: SrcLoc {
+                    fileid,
+                    line: entry[4].as_u64().unwrap(),
+                    column: entry[5].as_u64().unwrap(),
+                },
                 type_id,
-                file_path,
                 rvalue: if entry[7].as_boolean().unwrap() {
                     LRValue::RValue
                 } else {
@@ -178,5 +195,6 @@ pub fn process(items: Value) -> error::Result<AstContext> {
         ast_nodes: asts,
         type_nodes: types,
         comments,
+        files,
     })
 }