runtime: add json import and export (eunomia-bpf#51)

* runtime: let kernel allocate fd

* add json handler

* Add export test

* fix unit test

* add test for json import and export

* build bpftimetool

* fix examples

* fix extern c

---------

Co-authored-by: Littlefisher619 <i@littlefisher.me>

.github/workflows/test-examples.yml

@@ -5,6 +5,7 @@ on:
     branches: "master"
   pull_request: 
     branches: "master"
+
 jobs: 
   build-and-test:
     runs-on: ubuntu-22.04

.github/workflows/test-runtime.yml

@@ -1,27 +1,28 @@
-name: Run unit tests of bpftime-runtime
-
+name: Build and test runtime
 
 on:
   push:
-    branches: "*"
+    branches: "master"
   pull_request: 
-    # Why wildcard? Because we want to see test results when examing prs from non-master branches..
-    branches: "*"
-
+    branches: "master"
 
 jobs:
-  build-test-target-and-run:
+  build:
     runs-on: ubuntu-22.04
+
     steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-      - name: Install dependencies
-        run: | 
-          sudo apt-get install binutils-dev libboost1.74-all-dev libelf-dev zlib1g-dev libyaml-cpp-dev gcc-12 g++-12 llvm
-      - name: Build the target
-        run: |
-          CC=gcc-12 CXX=g++-12 make build-unit-test
-      - name: Run the unit test
-        run: |
-          make unit-test
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+
+    - name: install deps
+      run: |
+        sudo apt install -y --no-install-recommends \
+        libelf1 libelf-dev zlib1g-dev make git libboost1.74-all-dev \
+        binutils-dev libyaml-cpp-dev  gcc-12 g++-12 llvm
+
+    - name: build runtime
+      run:  CC=gcc-12 CXX=g++-12 make build
+
+    - name: test runtime
+      run:  make unit-test

.github/workflows/test-verifier.yml

@@ -1,4 +1,4 @@
-name: Test whether bpftime-verifier works properly
+name: Build and test verifier
 
 on:
   push:

.github/workflows/test-vm.yml

@@ -1,4 +1,4 @@
-name: Compile and Test VM
+name: Build and Test VM
 
 on:
   push:

CMakeLists.txt

@@ -25,7 +25,7 @@ endif()
 
 if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
   message(STATUS "Enabling ubsan for Debug builds; Processor=${CMAKE_SYSTEM_PROCESSOR}")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
 
   if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm((32.*)|(v6.*)|(v7.*))")
     message(STATUS "Linking libatomic on arm32 machines..")
@@ -84,6 +84,7 @@ set(SPDLOG_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/spdlog/include)
 add_subdirectory(vm)
 add_subdirectory(runtime)
 add_subdirectory(daemon)
+add_subdirectory(tools)
 
 # benchmark that requires bpftime libraries
 add_subdirectory(benchmark)

Makefile

@@ -34,11 +34,13 @@ build-unit-test:
 	cmake -Bbuild  -DBPFTIME_ENABLE_UNIT_TESTING=1 -DCMAKE_BUILD_TYPE:STRING=Debug
 	cmake --build build --config Debug --target bpftime_runtime_tests
 
-unit-test:
+unit-test: ## run catch2 unit tests
+	make -C runtime/test/bpf && cp runtime/test/bpf/*.bpf.o build/runtime/test/
 	./build/runtime/unit-test/bpftime_runtime_tests
+	cd build/runtime/test && ctest -VV
 
 build: ## build the package
-	cmake -Bbuild   -DBPFTIME_ENABLE_UNIT_TESTING=1
+	cmake -Bbuild -DBPFTIME_ENABLE_UNIT_TESTING=1
 	cmake --build build --config Debug
 	cd tools/cli-rs && cargo build
 

README.md

@@ -24,7 +24,7 @@
 
 With `bpftime`, you can build eBPF applications using familiar tools like clang and libbpf, and execute them in userspace. For instance, the `malloc` eBPF program traces malloc calls using uprobe and aggregates the counts using a hash map.
 
-You can refer to [documents/build-and-test.md](documents/build-and-test.md) for how to build the project.
+You can refer to [documents/build-and-test.md](documents/build-and-test.md) for how to build the project, or using the container images from [GitHub packages](https://github.com/eunomia-bpf/bpftime/pkgs/container/bpftime).
 
 To get started, you can build and run a libbpf based eBPF program starts with `bpftime` cli:
 

cmake/StandardSettings.cmake

@@ -60,6 +60,6 @@ endif()
 
 option(BPFTIME_ENABLE_ASAN "Enable Address Sanitize to detect memory error." OFF)
 if(BPFTIME_ENABLE_ASAN)
-    add_compile_options(-fsanitize=address)
-    add_link_options(-fsanitize=address)
+    add_compile_options(-fsanitize=address,undefined)
+    add_link_options(-fsanitize=address,undefined)
 endif()

daemon/main.cpp

@@ -1,9 +1,4 @@
 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-// Copyright (c) 2019 Facebook
-// Copyright (c) 2020 Netflix
-//
-// Based on bpf_mocker(8) from BCC by Brendan Gregg and others.
-// 14-Feb-2020   Brendan Gregg   Created this.
 #include <argp.h>
 #include <signal.h>
 #include <stdio.h>

runtime/CMakeLists.txt

@@ -62,6 +62,7 @@ set(sources
   src/handler/epoll_handler.cpp
   src/bpftime_shm.cpp
   src/bpftime_shm_internal.cpp
+  src/bpftime_shm_json.cpp
   src/syscall_table.cpp
   src/bpf_map/array_map.cpp
   src/bpf_map/hash_map.cpp

runtime/include/bpftime_shm.hpp

@@ -47,15 +47,31 @@ void bpftime_initialize_global_shm();
 // destroy the global shared memory
 void bpftime_destroy_global_shm();
 
+// import the global shared memory from json file
+int bpftime_import_global_shm_from_json(const char *filename);
+// export the global shared memory to json file
+int bpftime_export_global_shm_to_json(const char *filename);
+// import a hander to global shared memory from json string
+int bpftime_import_shm_handler_from_json(int fd, const char *json_string);
+
 // create a bpf link in the global shared memory
-int bpftime_link_create(int prog_fd, int target_fd);
+//
+// @param[fd]: fd is the fd allocated by the kernel. if fd is -1, then the
+// function will allocate a new perf event fd.
+int bpftime_link_create(int fd, int prog_fd, int target_fd);
 
 // create a bpf prog in the global shared memory
-int bpftime_progs_create(const ebpf_inst *insn, size_t insn_cnt,
+//
+// @param[fd]: fd is the fd allocated by the kernel. if fd is -1, then the
+// function will allocate a new perf event fd.
+int bpftime_progs_create(int fd, const ebpf_inst *insn, size_t insn_cnt,
 			 const char *prog_name, int prog_type);
 
-// create a bpf map in the global shared memory	
-int bpftime_maps_create(const char *name, bpftime::bpf_map_attr attr);
+// create a bpf map in the global shared memory
+//
+// @param[fd]: fd is the fd allocated by the kernel. if fd is -1, then the
+// function will allocate a new perf event fd.
+int bpftime_maps_create(int fd, const char *name, bpftime::bpf_map_attr attr);
 
 // get the bpf map info from the global shared memory
 int bpftime_map_get_info(int fd, bpftime::bpf_map_attr *out_attr,
@@ -65,32 +81,36 @@ int bpftime_map_get_info(int fd, bpftime::bpf_map_attr *out_attr,
 uint32_t bpftime_map_value_size_from_syscall(int fd);
 
 // used by bpf_helper to get the next key
-int bpftime_helper_map_get_next_key(int fd, const void *key,
-					 void *next_key);
+int bpftime_helper_map_get_next_key(int fd, const void *key, void *next_key);
 // used by bpf_helper to lookup the elem
 const void *bpftime_helper_map_lookup_elem(int fd, const void *key);
 // used by bpf_helper to update the elem
-long bpftime_helper_map_update_elem(int fd, const void *key,
-					 const void *value, uint64_t flags);
+long bpftime_helper_map_update_elem(int fd, const void *key, const void *value,
+				    uint64_t flags);
 // used by bpf_helper to delete the elem
 long bpftime_helper_map_delete_elem(int fd, const void *key);
 
 // use from bpf syscall to get the next key
-int bpftime_map_get_next_key(int fd, const void *key,
-					  void *next_key);
+int bpftime_map_get_next_key(int fd, const void *key, void *next_key);
 // use from bpf syscall to lookup the elem
 const void *bpftime_map_lookup_elem(int fd, const void *key);
 // use from bpf syscall to update the elem
-long bpftime_map_update_elem(int fd, const void *key,
-					  const void *value, uint64_t flags);
+long bpftime_map_update_elem(int fd, const void *key, const void *value,
+			     uint64_t flags);
 // use from bpf syscall to delete the elem
 long bpftime_map_delete_elem(int fd, const void *key);
 
 // create uprobe in the global shared memory
-int bpftime_uprobe_create(int pid, const char *name, uint64_t offset,
+//
+// @param[fd]: fd is the fd allocated by the kernel. if fd is -1, then the
+// function will allocate a new perf event fd.
+int bpftime_uprobe_create(int fd, int pid, const char *name, uint64_t offset,
 			  bool retprobe, size_t ref_ctr_off);
 // create tracepoint in the global shared memory
-int bpftime_tracepoint_create(int pid, int32_t tp_id);
+//
+// @param[fd]: fd is the fd allocated by the kernel. if fd is -1, then the
+// function will allocate a new perf event fd.
+int bpftime_tracepoint_create(int fd, int pid, int32_t tp_id);
 
 int bpftime_perf_event_enable(int fd);
 int bpftime_perf_event_disable(int fd);
@@ -121,7 +141,6 @@ int bpftime_add_software_perf_event(int cpu, int32_t sample_type,
 int bpftime_is_software_perf_event(int fd);
 void *bpftime_get_software_perf_event_raw_buffer(int fd, size_t expected_size);
 int bpftime_perf_event_output(int fd, const void *buf, size_t sz);
-
 }
 
 #endif // BPFTIME_SHM_CPP_H

runtime/src/attach/bpf_attach_ctx.cpp

@@ -37,6 +37,7 @@ static int load_prog_and_helpers(bpftime_prog *prog, agent_config &config)
 	}
 	return prog->bpftime_prog_load(config.jit_enabled);
 }
+
 int bpf_attach_ctx::init_attach_ctx_from_handlers(agent_config &config)
 {
 	const handler_manager *manager =
@@ -46,6 +47,7 @@ int bpf_attach_ctx::init_attach_ctx_from_handlers(agent_config &config)
 	}
 	return init_attach_ctx_from_handlers(manager, config);
 }
+
 // Check whether there is a syscall trace program. Use the global
 // handler manager
 bool bpf_attach_ctx::check_exist_syscall_trace_program()

runtime/src/bpf_map/map_common_def.hpp

@@ -7,6 +7,7 @@
 #include <boost/interprocess/containers/vector.hpp>
 #include <functional>
 #include <sched.h>
+
 namespace bpftime
 {
 

runtime/src/bpftime_shm.cpp

@@ -12,21 +12,21 @@
 
 using namespace bpftime;
 
-int bpftime_link_create(int prog_fd, int target_fd)
+int bpftime_link_create(int fd, int prog_fd, int target_fd)
 {
-	return shm_holder.global_shared_memory.add_bpf_link(prog_fd, target_fd);
+	return shm_holder.global_shared_memory.add_bpf_link(fd, prog_fd, target_fd);
 }
 
-int bpftime_progs_create(const ebpf_inst *insn, size_t insn_cnt,
+int bpftime_progs_create(int fd, const ebpf_inst *insn, size_t insn_cnt,
 			 const char *prog_name, int prog_type)
 {
-	return shm_holder.global_shared_memory.add_bpf_prog(
+	return shm_holder.global_shared_memory.add_bpf_prog(fd, 
 		insn, insn_cnt, prog_name, prog_type);
 }
 
-int bpftime_maps_create(const char *name, bpftime::bpf_map_attr attr)
+int bpftime_maps_create(int fd, const char *name, bpftime::bpf_map_attr attr)
 {
-	return shm_holder.global_shared_memory.add_bpf_map(name, attr);
+	return shm_holder.global_shared_memory.add_bpf_map(fd, name, attr);
 }
 uint32_t bpftime_map_value_size_from_syscall(int fd)
 {
@@ -81,16 +81,16 @@ int bpftime_map_get_next_key(int fd, const void *key,
 		fd, key, next_key, true);
 }
 
-int bpftime_uprobe_create(int pid, const char *name, uint64_t offset,
+int bpftime_uprobe_create(int fd, int pid, const char *name, uint64_t offset,
 			  bool retprobe, size_t ref_ctr_off)
 {
-	return shm_holder.global_shared_memory.add_uprobe(
+	return shm_holder.global_shared_memory.add_uprobe(fd, 
 		pid, name, offset, retprobe, ref_ctr_off);
 }
 
-int bpftime_tracepoint_create(int pid, int32_t tp_id)
+int bpftime_tracepoint_create(int fd, int pid, int32_t tp_id)
 {
-	return shm_holder.global_shared_memory.add_tracepoint(pid, tp_id);
+	return shm_holder.global_shared_memory.add_tracepoint(fd, pid, tp_id);
 }
 
 int bpftime_perf_event_enable(int fd)