Caliper instrumentation

src/c/runtime/CMakeLists.txt

@@ -13,16 +13,16 @@ set(perfflow_runtime_sources
 )
 
 find_package(OpenSSL REQUIRED)
+find_package(caliper REQUIRED)
 
-set(perfflow_runtime_deps -ljansson
-)
+set(perfflow_runtime_deps -ljansson)
 
 add_library(perfflow_runtime SHARED
             ${perfflow_runtime_sources}
             ${perfflow_runtime_headers}
 )
 
-target_link_libraries(perfflow_runtime ${perfflow_runtime_deps} OpenSSL::SSL OpenSSL::Crypto)
+target_link_libraries(perfflow_runtime ${perfflow_runtime_deps} OpenSSL::SSL OpenSSL::Crypto caliper)
 
 install(TARGETS perfflow_runtime
         EXPORT perfflow_export

src/c/test/CMakeLists.txt

@@ -10,6 +10,7 @@ set(SMOKETESTS
 find_package(MPI REQUIRED)
 find_package(Threads REQUIRED)
 find_package(CUDA REQUIRED)
+find_package(caliper REQUIRED)
 include_directories(${MPI_INCLUDE_PATH})
 
 set(perfflow_deps "-L../runtime -lperfflow_runtime -lcrypto")
@@ -22,13 +23,13 @@ foreach(TEST ${SMOKETESTS})
     target_link_libraries(${TEST} ${MPI_LIBRARIES})
     target_link_libraries(${TEST} pthread)
     set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fPIC")
-    target_link_libraries(${TEST} ${perfflow_deps})
+    target_link_libraries(${TEST} ${perfflow_deps} caliper)
 endforeach()
 
 message(STATUS " [*] Adding test: smoketest_cuda")
 set(CUDA_NVCC_FLAGS  "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so")
 cuda_add_executable(smoketest_cuda smoketest_cuda_wrapper.cpp smoketest_cuda_kernel.cu)
-target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES} cuda)
+target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES} cuda caliper)
 
 configure_file(t0001-cbinding-basic.t.in
     ${CMAKE_CURRENT_BINARY_DIR}/t0001-cbinding-basic.t

src/c/weaver/weave/perfflow_weave.cpp

@@ -16,6 +16,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -67,14 +68,14 @@ bool WeavingPass::insertAfter(Module &m, Function &f, StringRef &a,
         {
             IRBuilder<> builder(inst);
             Value *v1 = builder.CreateGlobalStringPtr(m.getName(), "str");
-            Value *v2 = builder.CreateGlobalStringPtr(f.getName(), "str");
+            Value *FnNameStr = builder.CreateGlobalStringPtr(f.getName(), "str");
             Value *v3 = builder.CreateGlobalStringPtr(StringRef(scope), "str");
             Value *v4 = builder.CreateGlobalStringPtr(StringRef(flow), "str");
             Value *v5 = builder.CreateGlobalStringPtr(StringRef(pcut), "str");
             std::vector<Value *> args;
             args.push_back(ConstantInt::get(Type::getInt32Ty(context), async));
             args.push_back(v1);
-            args.push_back(v2);
+            args.push_back(FnNameStr);
             args.push_back(v3);
             args.push_back(v4);
             args.push_back(v5);
@@ -84,6 +85,46 @@ bool WeavingPass::insertAfter(Module &m, Function &f, StringRef &a,
     return true;
 }
 
+bool WeavingPass::instrumentCaliper(Module &M, Function &F){
+    IRBuilder<> IRB(M.getContext());
+    BasicBlock &Entry = F.getEntryBlock();
+    SplitBlock(&Entry, &*Entry.getFirstInsertionPt());
+
+    IRB.SetInsertPoint(Entry.getTerminator());
+    std::string FunctionName = F.getName().str();
+    auto *FnStr = IRB.CreateGlobalStringPtr(FunctionName);
+    IRB.CreateCall(CaliBeginRegion, {FnStr});
+
+    bool RetFound = false;
+    for (inst_iterator It = inst_begin(F), E = inst_end(F); It != E; ++It) {
+      Instruction *I = &*It;
+      if (!isa<ReturnInst>(I) && !isa<CallInst>(I) && !isa<InvokeInst>(I))
+        continue;
+
+      if ( isa<ReturnInst>(I) ) {
+          IRB.SetInsertPoint(I);
+          IRB.CreateCall(CaliEndRegion, {FnStr});
+          RetFound = true;
+      }
+
+      // This is a call instruction
+      CallBase *CB = dyn_cast<CallBase>(I);
+      if ( CB && CB->doesNotReturn() ){
+          IRB.SetInsertPoint(I);
+          IRB.CreateCall(CaliEndRegion, {FnStr});
+          RetFound = true;
+      }
+    }
+
+    // All functions need to have at least one exit block
+    if (!RetFound) {
+      dbgs() << "Could not find return for " << FunctionName << "\n";
+      abort();
+    }
+
+    return RetFound;
+}
+
 bool WeavingPass::insertBefore(Module &m, Function &f, StringRef &a,
                                int async, std::string &scope, std::string &flow, std::string pcut)
 {
@@ -114,15 +155,15 @@ bool WeavingPass::insertBefore(Module &m, Function &f, StringRef &a,
     auto &entry = f.getEntryBlock();
     IRBuilder<> builder(&entry);
     Value *v1 = builder.CreateGlobalStringPtr(m.getName(), "str");
-    Value *v2 = builder.CreateGlobalStringPtr(f.getName(), "str");
+    Value *FnNameStr = builder.CreateGlobalStringPtr(f.getName(), "str");
     Value *v3 = builder.CreateGlobalStringPtr(StringRef(scope), "str");
     Value *v4 = builder.CreateGlobalStringPtr(StringRef(flow), "str");
     Value *v5 = builder.CreateGlobalStringPtr(StringRef(pcut), "str");
     builder.SetInsertPoint(&entry, entry.begin());
     std::vector<Value *> args;
     args.push_back(ConstantInt::get(Type::getInt32Ty(context), async));
     args.push_back(v1);
-    args.push_back(v2);
+    args.push_back(FnNameStr);
     args.push_back(v3);
     args.push_back(v4);
     args.push_back(v5);
@@ -146,13 +187,31 @@ bool WeavingPass::doInitialization(Module &m)
         return false;
     }
 
+    IRBuilder<> IRB(m.getContext());
+    AttrBuilder AB;
+    AB.addAttribute(Attribute::AlwaysInline);
+    //It was not added in LLVM@10.
+    //AB.addAttribute(Attribute::ArgMemOnly);
+    AttributeList Attrs = AttributeList::get(m.getContext(),
+            AttributeList::FunctionIndex, AB);
+    // Insert Functions on the module
+    CaliBeginRegion = m.getOrInsertFunction(
+        "cali_begin_region", Attrs, IRB.getVoidTy(), IRB.getInt8PtrTy());
+    CaliEndRegion = m.getOrInsertFunction("cali_end_region", Attrs,
+                                          IRB.getVoidTy(), IRB.getInt8PtrTy());
+
+
     bool changed = false;
     auto a = cast<ConstantArray> (annotations->getOperand(0));
     for (unsigned int i = 0; i < a->getNumOperands(); i++)
     {
         auto e = cast<ConstantStruct> (a->getOperand(i));
         if (auto *fn = dyn_cast<Function> (e->getOperand(0)->getOperand(0)))
         {
+            // We insert Caliper Instrumentation before weaver.
+            // Thus weaver will include Caliper overheads
+            changed |= instrumentCaliper(m, *fn);
+
             auto anno = cast<ConstantDataArray>(
                             cast<GlobalVariable>(e->getOperand(1)->getOperand(0))
                             ->getOperand(0))

src/c/weaver/weave/perfflow_weave.hpp

@@ -11,9 +11,13 @@
 #ifndef PERFFLOW_WEAVE_H
 #define PERFFLOW_WEAVE_H
 
+#include "llvm/IR/Attributes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 
 #include <string>
 
@@ -24,6 +28,9 @@ namespace {
 class WeavingPass : public FunctionPass
 {
 public:
+    FunctionCallee CaliBeginRegion;
+    FunctionCallee CaliEndRegion;
+
     static char ID;
     WeavingPass () : FunctionPass (ID) {}
     virtual bool doInitialization (Module &m);
@@ -34,6 +41,8 @@ class WeavingPass : public FunctionPass
                       int async, std::string &scope, std::string &flow, std::string pcut);
     bool insertBefore (Module &m, Function &f, StringRef &a,
                        int async, std::string &scope, std::string &flow, std::string pcut);
+
+    bool instrumentCaliper(Module &M, Function &F);
 };
 
 }