docs: add more example and usage (#15)

* update

* update

.gitignore

@@ -14,3 +14,7 @@ test.bin
 *.ll
 bpf_conformance
 coverage.info
+example/standalone/standalone
+example/inline/standalone
+example/inline/inline.o
+example/load-llvm-ir/bpf_module.o

README.md

@@ -24,6 +24,10 @@ For a comprehensive userspace eBPF runtime that includes support for maps, helpe
     - [Use llvmbpf as a AOT compiler](#use-llvmbpf-as-a-aot-compiler)
     - [load eBPF bytecode from ELF file](#load-ebpf-bytecode-from-elf-file)
     - [Maps and data relocation support](#maps-and-data-relocation-support)
+    - [Build into standalone binary for deployment](#build-into-standalone-binary-for-deployment)
+  - [optimizaion](#optimizaion)
+    - [inline the maps and helper function](#inline-the-maps-and-helper-function)
+    - [Use original LLVM IR from C code](#use-original-llvm-ir-from-c-code)
   - [Test](#test)
     - [Unit test](#unit-test)
     - [Test with bpf-conformance](#test-with-bpf-conformance)
@@ -68,12 +72,19 @@ void run_ebpf_prog(const void *code, size_t code_len)
 
 ### Use llvmbpf as a AOT compiler
 
+Build with cli:
+
+```sh
+sudo apt-get install libelf1 libelf-dev
+cmake -B build  -DBUILD_LLVM_AOT_CLI=1 
+```
+
 You can use the cli to generate the LLVM IR from eBPF bytecode:
 
 ```console
 # ./build/cli/bpftime-vm build .github/assets/sum.bpf.o -emit-llvm > test.bpf.ll
 # opt -O3 -S test.bpf.ll -opaque-pointers  -o test.opt.ll
-# cat test.opt.ll 
+# cat test.opt.ll
 ; ModuleID = 'test.bpf.ll'
 source_filename = "bpf-jit"
 
@@ -124,6 +135,8 @@ Load and run a AOTed eBPF program:
 [2024-08-10 14:57:16.991] [info] [main.cpp:136] Program executed successfully. Return value: 6
 ```
 
+See [Build into standalone binary for deployment](#build-into-standalone-binary-for-deployment) for more details.
+
 ### load eBPF bytecode from ELF file
 
 You can use llvmbpf together with libbpf to load the eBPF bytecode directly from `bpf.o` ELF file. For example:
@@ -280,6 +293,181 @@ Reference:
 - <https://prototype-kernel.readthedocs.io/en/latest/bpf/ebpf_maps.html>
 - <https://www.ietf.org/archive/id/draft-ietf-bpf-isa-00.html#name-64-bit-immediate-instructio>
 
+### Build into standalone binary for deployment
+
+You can build the eBPF program into a standalone binary, which does not rely on any external libraries, and can be exec like nomal c code with helper and maps support.
+
+This can help:
+
+- Easily deploy the eBPF program to any machine without the need to install any dependencies.
+- Avoid the overhead of loading the eBPF bytecode and maps at runtime.
+- Suitable for microcontroller or embedded systems, which does not have a OS.
+
+Take [https://github.com/eunomia-bpf/bpftime/blob/master/example/xdp-counter/](https://github.com/eunomia-bpf/bpftime/blob/master/example/xdp-counter/) as an example:
+
+In the bpftime project:
+
+```sh
+# load the eBPF program with bpftime
+LD_PRELOAD=build/runtime/syscall-server/libbpftime-syscall-server.so example/xdp-counter/xdp-counter example/xdp-counter/.output/xdp-counter.bpf.o veth1
+# dump the map and eBPF bytecode define
+./build/tools/bpftimetool/bpftimetool export res.json
+# build the eBPF program into llvm IR
+./build/tools/aot/bpftime-aot compile --emit_llvm 1>xdp-counter.ll
+```
+
+You can see [example/xdp-counter.json](example/xdp-counter.json) for an example json file dump by bpftime.
+
+The result xdp-counter.ll can be found in [example/standalone/xdp-counter.ll](example/standalone/xdp-counter.ll).
+
+Then you can write a C code and compile it with the llvm IR:
+
+```c
+#include <stdint.h>
+#include <stdio.h>
+#include <inttypes.h>
+
+int bpf_main(void* ctx, uint64_t size);
+
+uint32_t ctl_array[2] = { 0, 0 };
+uint64_t cntrs_array[2] = { 0, 0 };
+
+void *_bpf_helper_ext_0001(uint64_t map_fd, void *key)
+{
+  printf("bpf_map_lookup_elem %lu\n", map_fd);
+  if (map_fd == 5) {
+    return &ctl_array[*(uint32_t *)key];
+  } else if (map_fd == 6) {
+    return &cntrs_array[*(uint32_t *)key];
+  } else {
+    return NULL;
+  }
+  return 0;
+}
+
+void* __lddw_helper_map_val(uint64_t val)
+{
+    printf("map_val %lu\n", val);
+    if (val == 5) {
+        return (void *)ctl_array;
+    } else if (val == 6) {
+        return (void *)cntrs_array;
+    } else {
+        return NULL;
+    }
+}
+
+uint8_t bpf_mem[] = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 };
+
+int main() {
+    printf("The value of cntrs_array[0] is %" PRIu64 "\n", cntrs_array[0]);
+    printf("calling ebpf program...\n");
+    bpf_main(bpf_mem, sizeof(bpf_mem));
+    printf("The value of cntrs_array[0] is %" PRIu64 "\n", cntrs_array[0]);
+    printf("calling ebpf program...\n");
+    bpf_main(bpf_mem, sizeof(bpf_mem));
+    printf("The value of cntrs_array[0] is %" PRIu64 "\n", cntrs_array[0]);
+    return 0;
+}
+```
+
+Compile the C code with the llvm IR:
+
+```sh
+clang -g main.c xdp-counter.ll -o standalone 
+```
+
+And you can run the `standalone` eBPF program directly.
+
+## optimizaion
+
+Based on the AOT compiler, we can apply some optimization strategies:
+
+### inline the maps and helper function
+
+Inline the maps and helper function into the eBPF program, so that the eBPF program can be optimized with `const propagation`, `dead code elimination`, etc by the LLVM optimizer. llvmbpf can also eliminate the cost of function calls.
+
+Prepare a C code:
+
+```c
+
+uint32_t ctl_array[2] = { 0, 0 };
+uint64_t cntrs_array[2] = { 0, 0 };
+
+void *_bpf_helper_ext_0001(uint64_t map_fd, void *key)
+{
+  if (map_fd == 5) {
+    return &ctl_array[*(uint32_t *)key];
+  } else if (map_fd == 6) {
+    return &cntrs_array[*(uint32_t *)key];
+  } else {
+    return NULL;
+  }
+  return 0;
+}
+
+void* __lddw_helper_map_val(uint64_t val)
+{
+    if (val == 5) {
+        return (void *)ctl_array;
+    } else if (val == 6) {
+        return (void *)cntrs_array;
+    } else {
+        return NULL;
+    }
+}
+```
+
+Merge the modules with `llvm-link` and inline them:
+
+```sh
+clang -S -O3 -emit-llvm libmap.c -o libmap.ll
+llvm-link -S -o xdp-counter-inline.ll xdp-counter.ll libmap.ll
+opt --always-inline -S xdp-counter-inline.ll -o xdp-counter-inline.ll
+clang -O3 -g -c xdp-counter-inline.ll -o inline.o
+```
+
+Run the code with cli:
+
+```c
+./build/cli/bpftime-vm run example/inline/inline.o test.bin
+```
+
+Or you can compile as standalone binary and link with the C code:
+
+```console
+$ clang -O3 example/inline/inline.o example/inline/main.c -o inline
+$ /workspaces/llvmbpf/inline
+calling ebpf program...
+return value = 1
+```
+
+### Use original LLVM IR from C code
+
+eBPF is a instruction set define for verification, but may not be the best for performance.
+
+llvmbpf also support using the original LLVM IR from C code. See [example/load-llvm-ir](example/load-llvm-ir) for an example. You can:
+
+- Compile the C code to eBPF for verify
+- Compile the C code to LLVM IR and native code for execution in the VM.
+
+The C code:
+
+```c
+int _bpf_helper_ext_0006(const char *fmt, ... );
+
+int bpf_main(void* ctx, int size) {
+    _bpf_helper_ext_0006("hello world: %d\n", size);
+    return 0;
+}
+```
+
+You can compile it with `clang -g -c bpf_module.c -o bpf_module.o`, and Run the code with cli:
+
+```c
+./build/cli/bpftime-vm run example/load-llvm-ir/bpf_module.o test.bin
+```
+
 ## Test
 
 ### Unit test

cli/main.cpp

@@ -58,6 +58,20 @@ static bool has_argument(int argc, const char **argv, const std::string &option)
 	return false;
 }
 
+uint64_t bpftime_trace_printk(uint64_t fmt, uint64_t fmt_size, ...)
+{
+	const char *fmt_str = (const char *)fmt;
+	va_list args;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
+#pragma GCC diagnostic ignored "-Wvarargs"
+	va_start(args, fmt_str);
+	long ret = vprintf(fmt_str, args);
+#pragma GCC diagnostic pop
+	va_end(args);
+	return 0;
+}
+
 static int build_ebpf_program(const std::string &ebpf_elf,
 			      const std::filesystem::path &output,
 			      bool emit_llvm)
@@ -130,6 +144,8 @@ static int run_ebpf_program(const std::filesystem::path &elf,
 	file.close();
 
 	llvmbpf_vm vm;
+	vm.register_external_function(6, "bpf_trace_printk",
+					      (void *)bpftime_trace_printk);
 	auto func = vm.load_aot_object(file_buffer);
 	if (!func) {
 		SPDLOG_CRITICAL("Failed to load AOT object from ELF file: {}",

example/inline/Makefile

@@ -0,0 +1,5 @@
+inline.o: libmap.c
+	clang -S -O3 -emit-llvm libmap.c -o libmap.ll
+	llvm-link -S -o xdp-counter-inline.ll xdp-counter.ll libmap.ll
+	opt -passes=always-inline -S xdp-counter-inline.ll -o xdp-counter-inline.ll
+	clang -O3 -g -c xdp-counter-inline.ll -o inline.o

example/inline/libmap.c

@@ -0,0 +1,29 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <inttypes.h>
+
+uint32_t ctl_array[2] = { 0, 0 };
+uint64_t cntrs_array[2] = { 0, 0 };
+
+void *_bpf_helper_ext_0001(uint64_t map_fd, void *key)
+{
+	if (map_fd == 5) {
+		return &ctl_array[*(uint32_t *)key];
+	} else if (map_fd == 6) {
+		return &cntrs_array[*(uint32_t *)key];
+	} else {
+		return NULL;
+	}
+	return 0;
+}
+
+void* __lddw_helper_map_val(uint64_t val)
+{
+    if (val == 5) {
+        return (void *)ctl_array;
+    } else if (val == 6) {
+        return (void *)cntrs_array;
+    } else {
+        return NULL;
+    }
+}

example/inline/libmap.ll

@@ -0,0 +1,56 @@
+; ModuleID = 'libmap.c'
+source_filename = "libmap.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+@ctl_array = dso_local global [2 x i32] zeroinitializer, align 4
+@cntrs_array = dso_local global [2 x i64] zeroinitializer, align 16
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) uwtable
+define dso_local ptr @_bpf_helper_ext_0001(i64 noundef %0, ptr nocapture noundef readonly %1) local_unnamed_addr #0 {
+  switch i64 %0, label %11 [
+    i64 5, label %3
+    i64 6, label %7
+  ]
+
+3:                                                ; preds = %2
+  %4 = load i32, ptr %1, align 4, !tbaa !5
+  %5 = zext i32 %4 to i64
+  %6 = getelementptr inbounds [2 x i32], ptr @ctl_array, i64 0, i64 %5
+  br label %11
+
+7:                                                ; preds = %2
+  %8 = load i32, ptr %1, align 4, !tbaa !5
+  %9 = zext i32 %8 to i64
+  %10 = getelementptr inbounds [2 x i64], ptr @cntrs_array, i64 0, i64 %9
+  br label %11
+
+11:                                               ; preds = %2, %7, %3
+  %12 = phi ptr [ %6, %3 ], [ %10, %7 ], [ null, %2 ]
+  ret ptr %12
+}
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable
+define dso_local noundef ptr @__lddw_helper_map_val(i64 noundef %0) local_unnamed_addr #1 {
+  %2 = icmp eq i64 %0, 6
+  %3 = select i1 %2, ptr @cntrs_array, ptr null
+  %4 = icmp eq i64 %0, 5
+  %5 = select i1 %4, ptr @ctl_array, ptr %3
+  ret ptr %5
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+attributes #1 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"Ubuntu clang version 18.1.3 (1ubuntu1)"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}

example/inline/main.c

@@ -0,0 +1,14 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <inttypes.h>
+
+int bpf_main(void* ctx, uint64_t size);
+
+unsigned char bpf_mem[] = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 };
+
+int main() {
+    printf("calling ebpf program...\n");
+    int res = bpf_main(bpf_mem, sizeof(bpf_mem));
+    printf("return value = %d\n", res);
+    return 0;
+}