diff --git a/.gitignore b/.gitignore index eb8c87c..deda60f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ opentrace vmlinux.h *.o arch +*.csv +*.json diff --git a/dune b/dune index f6fff69..84ae7d0 100644 --- a/dune +++ b/dune @@ -1,9 +1,11 @@ (executable (name opentrace) (public_name opentrace) - (preprocess (pps ppx_blob)) - (preprocessor_deps (file opentrace.bpf.o)) - (libraries unix libbpf libbpf_maps)) + (preprocess + (pps ppx_blob)) + (preprocessor_deps + (file opentrace.bpf.o)) + (libraries eio_main jsonm cmdliner unix libbpf libbpf_maps)) (rule (mode @@ -12,7 +14,7 @@ (deps arch opentrace.bpf.c) (action (system - "NIX_HARDENING_ENABLE=\"\" clang -g -O2 -target bpf -I/usr/include/%{architecture}-linux-gnu/ -c opentrace.bpf.c -D__TARGET_ARCH_%{read:arch}"))) + "NIX_HARDENING_ENABLE=\"\" clang -g -O3 -target bpf -I/usr/include/%{architecture}-linux-gnu/ -c opentrace.bpf.c -D__TARGET_ARCH_%{read:arch}"))) (rule (mode diff --git a/dune-project b/dune-project index f229754..b48dc2c 100644 --- a/dune-project +++ b/dune-project @@ -1,3 +1,17 @@ (lang dune 3.14) (name opentrace) + +(generate_opam_files true) + +(package + (name opentrace) + (synopsis "Trace programs for the files they read and write") + (description "") + (depends + (ocaml (>= 5.2.0)) + eio_main + jsonm + libbpf + libbpf_maps)) + diff --git a/opentrace.bpf.c b/opentrace.bpf.c index 748b7c5..3f2cf92 100644 --- a/opentrace.bpf.c +++ b/opentrace.bpf.c @@ -4,13 +4,14 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; const volatile int pid_target = 0; +const volatile int cgroup_target = 0; struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); } rb SEC(".maps"); -#define FILE_NAME_LEN 1024 +#define FILE_NAME_LEN 256 #define OPEN_KIND 0 #define OPENAT_KIND 1 @@ -20,36 +21,39 @@ struct { struct open_event { uint32_t e_pid; + uint64_t e_cgid; + char e_comm[TASK_COMM_LEN]; int e_kind; int e_flags; uint32_t e_mode; char e_filename[FILE_NAME_LEN]; + int e_ret; }; +// Temporary memory between the syscall event and the exit +// of the event. +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 10240); + __type(key, u32); // PID + __type(value, struct open_event); // The event +} tmpmap SEC(".maps"); + SEC("tracepoint/syscalls/sys_enter_openat") int tracepoint__syscalls__sys_enter_openat(struct trace_event_raw_sys_enter* ctx) { u64 id = bpf_get_current_pid_tgid(); u32 pid = id >> 32; - - char filename[FILE_NAME_LEN]; - struct open_event *oet; - - oet = bpf_ringbuf_reserve(&rb, sizeof(struct open_event), 0); - if (!oet) - return 0; if (pid_target && pid_target != pid) return false; - // Fill the open event - oet->e_pid = id; - oet->e_kind = OPENAT_KIND; - oet->e_flags = (int)ctx->args[2]; - oet->e_mode = (__u32)ctx->args[3]; - bpf_probe_read(oet->e_filename, sizeof(filename), (char *) ctx->args[1]); - - bpf_ringbuf_submit(oet, 0); + struct open_event oet = {0}; + oet.e_flags = (int)ctx->args[2]; + oet.e_mode = (__u32)ctx->args[3]; + bpf_probe_read(oet.e_filename, FILE_NAME_LEN, (char *) ctx->args[1]); + bpf_map_update_elem(&tmpmap, &pid, &oet, BPF_NOEXIST); + return 0; } @@ -59,27 +63,19 @@ int tracepoint__syscalls__sys_enter_openat2(struct trace_event_raw_sys_enter* ct u64 id = bpf_get_current_pid_tgid(); u32 pid = id >> 32; - char filename[FILE_NAME_LEN]; - struct open_event *oet; - - oet = bpf_ringbuf_reserve(&rb, sizeof(struct open_event), 0); - if (!oet) - return 0; - if (pid_target && pid_target != pid) return false; - struct open_how how = {}; + struct open_event oet = {0}; + struct open_how how = {0}; bpf_probe_read_user(&how, sizeof(how), (void *)ctx->args[2]); - oet->e_flags = (int)how.flags; - oet->e_mode = (__u32)how.mode; - oet->e_kind = OPENAT2_KIND; - // Fill the open event - oet->e_pid = id; + oet.e_flags = (int)how.flags; + oet.e_mode = (__u32)how.mode; + bpf_probe_read(oet.e_filename, FILE_NAME_LEN, (char *) ctx->args[1]); - bpf_probe_read(oet->e_filename, sizeof(filename), (char *) ctx->args[1]); - - bpf_ringbuf_submit(oet, 0); + // Add it to the hash map + bpf_map_update_elem(&tmpmap, &pid, &oet, BPF_NOEXIST); + return 0; } @@ -88,25 +84,68 @@ int tracepoint__syscalls__sys_enter_open(struct trace_event_raw_sys_enter* ctx) { u64 id = bpf_get_current_pid_tgid(); u32 pid = id >> 32; - - char filename[FILE_NAME_LEN]; - struct open_event *oet; + if (pid_target && pid_target != pid) + return false; + + struct open_event oet = {0}; + oet.e_flags = (int)ctx->args[2]; + oet.e_mode = (__u32)ctx->args[3]; + bpf_probe_read(oet.e_filename, FILE_NAME_LEN, (char *) ctx->args[1]); + + // Add it to the hash map + bpf_map_update_elem(&tmpmap, &pid, &oet, BPF_NOEXIST); + + return 0; +} + +// SEC("tracepoint/syscalls/sys_exit_open") +// int tracepoint__syscalls__sys_exit_open(struct syscall_trace_exit* ctx) +// { +// return complete_event(OPEN_KIND, ctx); +// } + +SEC("tracepoint/syscalls/sys_exit_openat") +int tracepoint__syscalls__sys_exit_openat(struct syscall_trace_exit* ctx) +{ + struct open_event *ev; + u64 id = bpf_get_current_pid_tgid(); + u32 pid = id >> 32; + + struct open_event *oet; oet = bpf_ringbuf_reserve(&rb, sizeof(struct open_event), 0); if (!oet) return 0; - if (pid_target && pid_target != pid) - return false; + // Find our event + ev = bpf_map_lookup_elem(&tmpmap, &pid); + if (!ev) { + // Tidy up + bpf_ringbuf_discard(oet, 0); + bpf_map_delete_elem(&tmpmap, &pid); + return 0; + } + + oet->e_ret = ctx->ret; + oet->e_cgid = bpf_get_current_cgroup_id(); + bpf_probe_read_str(&oet->e_filename, sizeof(oet->e_filename), ev->e_filename); + bpf_get_current_comm(oet->e_comm, TASK_COMM_LEN); + oet->e_flags = ev->e_flags; + oet->e_pid = pid; + oet->e_mode = ev->e_mode; + oet->e_kind = OPENAT_KIND; - oet->e_flags = (int)ctx->args[2]; - oet->e_mode = (__u32)ctx->args[3]; - oet->e_kind = OPEN_KIND; - // Fill the open event - oet->e_pid = id; - - bpf_probe_read(oet->e_filename, sizeof(filename), (char *) ctx->args[1]); + // Tidy up + bpf_map_delete_elem(&tmpmap, &pid); + /* emit event */ bpf_ringbuf_submit(oet, 0); - return 0; + + return 0; +} + +SEC("tracepoint/syscalls/sys_exit_openat2") +int tracepoint__syscalls__sys_exit_openat2(struct syscall_trace_exit* ctx) +{ + return 0; } diff --git a/opentrace.ml b/opentrace.ml index fe00954..f26d051 100644 --- a/opentrace.ml +++ b/opentrace.ml @@ -1,6 +1,8 @@ open Libbpf open Libbpf_maps +type format = Csv | Json + let obj_path = "opentrace.bpf.o" let obj_file = [%blob "opentrace.bpf.o"] @@ -8,9 +10,24 @@ let program_names = [ "tracepoint__syscalls__sys_enter_openat"; "tracepoint__syscalls__sys_enter_openat2"; - (* "tracepoint__syscalls__sys_enter_open"; *) + "tracepoint__syscalls__sys_exit_openat"; + "tracepoint__syscalls__sys_exit_openat2"; ] +let json_to_lexemes json : Jsonm.lexeme list = + let rec loop acc = function + | `String _ as s -> s :: acc + | `Int i -> `Float (Int.to_float i) :: acc + | `O assoc -> + let lexemes = + List.fold_left + (fun vacc (k, v) -> loop (`Name k :: vacc) v) + (`Os :: acc) assoc + in + `Oe :: lexemes + in + loop [] json |> List.rev + module Open_event = struct open Ctypes @@ -31,10 +48,13 @@ module Open_event = struct let t : t structure typ = Ctypes.structure "event" let ( -: ) ty label = Ctypes.field t label ty let pid = uint32_t -: "e_pid" + let cgid = uint64_t -: "e_cgid" + let comm = array 16 char -: "e_comm" let kind = int -: "e_kind" let flags = int -: "e_flags" let mode = uint32_t -: "e_mode" let filename = array 256 char -: "e_filename" + let ret = int -: "e_ret" let () = seal t let char_array_as_string a = @@ -49,16 +69,44 @@ module Open_event = struct with Exit -> Buffer.contents b let get_pid s = getf s pid |> Unsigned.UInt32.to_int + let get_cgid s = getf s cgid |> Unsigned.UInt64.to_int64 + let get_comm s = getf s comm |> char_array_as_string let get_flags s = getf s flags let get_mode s = getf s mode |> Unsigned.UInt32.to_int let get_fname s = getf s filename |> char_array_as_string let get_kind s = getf s kind |> kind_of_int + let get_ret s = getf s ret + let csv_header = "pid,cgid,comm,kind,flags,mode,filename,return\n" + + let to_csv_row event = + Format.sprintf "%i,%Ld,%S,%s,%i,%i,\"%s\",%i%!" (get_pid event) + (get_cgid event) (get_comm event) + (get_kind event |> kind_to_string) + (get_flags event) (get_mode event) (get_fname event) (get_ret event) + + let to_json event = + `O + [ + ("pid", `Int (get_pid event)); + ("cgid", `Int (Int64.to_int @@ get_cgid event)); + ("comm", `String (get_comm event)); + ("kind", `String (get_kind event |> kind_to_string)); + ("flags", `Int (get_flags event)); + ("mode", `Int (get_mode event)); + ("fname", `String (get_fname event)); + ("ret", `Int (get_ret event)); + ] end -let () = +let run_ring_buffer bpf_callback = let dir = Filename.temp_dir "opentrace-" "" in let full_obj_path = Filename.concat dir obj_path in - Out_channel.with_open_bin full_obj_path (fun oc -> Out_channel.output_string oc obj_file); + Out_channel.with_open_bin full_obj_path (fun oc -> + Out_channel.output_string oc obj_file); + with_bpf_object_open_load_link ~obj_path:full_obj_path ~program_names + bpf_callback + +let ringbuffer_polling_callback ~poll rb_cb exit_cb = let bpf_callback obj _links = (* Set signal handlers *) let exitting = ref true in @@ -66,25 +114,195 @@ let () = Sys.(set_signal sigint sig_handler); Sys.(set_signal sigterm sig_handler); - (* Print header *) - Format.printf "pid,kind,flags,mode,filename\n"; - let map = Libbpf.bpf_object_find_map_by_name obj "rb" in let callback : RingBuffer.callback = fun _ data _ -> let event = Ctypes.(!@(from_voidp Open_event.t data)) in - Format.printf "%i,%s,%i,%i,\"%s\"\n%!" (Open_event.get_pid event) - (Open_event.get_kind event |> Open_event.kind_to_string) - (Open_event.get_flags event) - (Open_event.get_mode event) - (Open_event.get_fname event); - 0 + rb_cb event in RingBuffer.init map ~callback @@ fun rb -> while !exitting do - Unix.sleepf 1.0; let _ : int = RingBuffer.poll rb ~timeout:1 in - () + exit_cb exitting; + Unix.sleepf poll done in - with_bpf_object_open_load_link ~obj_path:full_obj_path ~program_names bpf_callback + bpf_callback + +let all poll no_header = + if no_header then () else Format.printf "%s" Open_event.csv_header; + let callback event = + Format.printf "%s\n%!" (Open_event.to_csv_row event); + 0 + in + let bpf_callback = ringbuffer_polling_callback ~poll callback (fun _ -> ()) in + run_ring_buffer bpf_callback + +let exec format output user poll (prog, args) = + let output = + match output with + | Some file -> file + | None -> + let ext = match format with Csv -> "csv" | Json -> "json" in + "trace." ^ ext + in + let uid = + match user with + | None -> Unix.getenv "SUDO_UID" |> int_of_string + | Some user -> ( + match int_of_string_opt user with + | Some uid -> uid + | None -> (Unix.getpwnam user).pw_uid) + in + assert (uid <> 0); + let start_process = Condition.create () in + let mutex = Mutex.create () in + let pid = Atomic.make None in + let exit_status = Atomic.make None in + let _domain = + Domain.spawn @@ fun () -> + Eio_main.run @@ fun env -> + Mutex.lock mutex; + Condition.wait start_process mutex; + Eio.Switch.run @@ fun sw -> + let p = + Eio.Process.spawn ~sw ~uid (Eio.Stdenv.process_mgr env) (prog :: args) + in + Atomic.set pid (Some (Eio.Process.pid p)); + let status = Eio.Process.await p in + Atomic.set exit_status (Some status) + in + Out_channel.with_open_bin output @@ fun oc -> + let encoder = Jsonm.encoder ~minify:false (`Channel oc) in + let encode l = + Jsonm.encode encoder (`Lexeme l) |> function `Ok -> () | _ -> assert false + in + let finish () = + Jsonm.encode encoder `End |> function `Ok -> () | _ -> assert false + in + let () = + (* Header *) + match format with + | Csv -> Out_channel.output_string oc Open_event.csv_header + | Json -> encode `As + in + let callback event = + match Atomic.get pid with + | None -> 0 + | Some pid -> + (if Int.equal (Open_event.get_pid event) pid then + match format with + | Csv -> + Out_channel.output_string oc (Open_event.to_csv_row event); + Out_channel.output_char oc '\n' + | Json -> + List.iter encode (json_to_lexemes (Open_event.to_json event))); + 0 + in + let spawned = ref false in + let exit_callback exitting = + if not !spawned then ( + Condition.broadcast start_process; + spawned := true); + Option.iter + (fun _ -> + exitting := false; + match format with + | Json -> + encode `Ae; + finish () + | _ -> ()) + (Atomic.get exit_status) + in + let bpf_callback = ringbuffer_polling_callback ~poll callback exit_callback in + run_ring_buffer bpf_callback + +open Cmdliner +open Cmdliner.Term.Syntax + +let polling = + let doc = "The number of seconds to sleep between polls of the ringbuffer" in + Arg.(value & opt float 0.1 & info [ "p"; "poll" ] ~doc ~docv:"POLL") + +let no_header = + let doc = "Disable printing the CSV header" in + Arg.(value & flag & info [ "no-header" ] ~doc) + +let user = + let doc = "Username or UID to execute program as" in + Arg.(value & opt (some string) None & info [ "u"; "user" ] ~doc ~docv:"USER") + +let format_conv : format Arg.conv = + let of_string s = + match String.lowercase_ascii s with + | "csv" -> Ok Csv + | "json" -> Ok Json + | _ -> Error (`Msg ("Unknown format: " ^ s)) + in + let to_string = function Csv -> "csv" | Json -> "json" in + let pp ppf v = Fmt.string ppf (to_string v) in + Arg.conv ~docv:"FORMAT" (of_string, pp) + +let format = + let doc = "Output format" in + Arg.(value & opt format_conv Csv & info [ "f"; "format" ] ~docv:"FORMAT" ~doc) + +let output = + let doc = + "Output file for trace. Defaults to trace. depending on the \ + $(format)." + in + Arg.( + value & opt (some string) None & info [ "o"; "output" ] ~docv:"OUTPUT" ~doc) + +let all_cmd = + let doc = "Trace all open system calls" in + let man = + [ + `P + "All calls to open will be traced and written to standard out in CSV \ + format."; + ] + in + Cmd.v (Cmd.info ~doc ~man "all") + @@ + let+ polling = polling and+ no_header = no_header in + all polling no_header + +let exec_cmd = + let doc = "Execute a program and trace its open system calls" in + let man = + [ + `P + "$(b,opentrace exec -- COMMAND) will execute COMMAND and trace only \ + those open calls from that program."; + `P "Opentrace will include the children of the main process."; + ] + in + Cmd.v (Cmd.info ~doc ~man "exec") + @@ + let+ prog = + Arg.(required & pos 0 (some string) None & Arg.info [] ~docv:"PROG") + and+ user = user + and+ format = format + and+ output = output + and+ args = Arg.(value & pos_right 0 string [] & Arg.info [] ~docv:"ARGS") + and+ poll = polling in + exec format output user poll (prog, args) + +let opentrace_cmd = + let doc = "Trace all open system calls" in + let man = + [ + `S Manpage.s_description; + `P "$(cmd) traces all open system calls"; + `P + "$(cmd) can be used either to run an executable directly or to trace \ + all open calls"; + ] + in + let default = Term.(ret (const (`Help (`Auto, None)))) in + Cmd.group (Cmd.info ~doc ~man "opentrace") ~default [ all_cmd; exec_cmd ] + +let main () = Cmd.eval opentrace_cmd +let () = if !Sys.interactive then () else exit (main ()) diff --git a/opentrace.opam b/opentrace.opam index 3e53aae..aff7e62 100644 --- a/opentrace.opam +++ b/opentrace.opam @@ -1,16 +1,14 @@ +# This file is generated by dune, edit dune-project instead opam-version: "2.0" -synopsis: "Trace the opening of files" -description: "A linux tool using eBPF for tracing calls to open files" -maintainer: ["Patrick Ferris "] -authors: ["Patrick Ferris "] -license: "MIT" -homepage: "https://tangled.sh/@patrick.sirref.org/opentrace" +synopsis: "Trace programs for the files they read and write" +description: "" depends: [ - "dune" {>= "3.17"} - "ocaml" + "dune" {>= "3.14"} + "ocaml" {>= "5.2.0"} + "eio_main" + "jsonm" "libbpf" "libbpf_maps" - "ppx_blob" "odoc" {with-doc} ] build: [ @@ -27,8 +25,3 @@ build: [ "@doc" {with-doc} ] ] -homepage: "https://tangled.sh/@patrick.sirref.org/opentrace" -pin-depends:[ - [ "libbpf.dev" "git+https://github.com/patricoferris/ocaml-libbpf#alpine" ] - [ "libbpf_maps.dev" "git+https://github.com/patricoferris/ocaml-libbpf#alpine" ] -]