$ ./syscall_tracer -p 1234
[myapp:1234] openat(AT_FDCWD, "/etc/passwd", O_RDONLY) = 3       [2.1us]
[myapp:1234] fstat(3, {...}) = 0                                  [0.8us]
[myapp:1234] read(3, "root:x:0:0:..."..., 4096) = 2847           [1.5us]
[myapp:1234] close(3) = 0                                        [0.6us]
# The [duration] column shows wall-clock time for each syscall
# Long durations on read/write indicate I/O bottlenecks
# Long durations on futex indicate lock contention

// Track allocations by hooking libc's malloc() with a uprobe
// Uprobes are like kprobes but for userspace functions -- the kernel
// patches a breakpoint into the target function's code
SEC("uprobe/libc.so:malloc")
int trace_malloc(struct pt_regs *ctx)
{
    u64 size = PT_REGS_PARM1(ctx);  // malloc's first arg = requested size
    u64 id = bpf_get_current_pid_tgid();
    
    // Store the requested size temporarily, keyed by thread ID
    // We need this because we cannot get the return value (the pointer)
    // until the function returns -- that is what the uretprobe is for
    bpf_map_update_elem(&alloc_sizes, &id, &size, BPF_ANY);
    return 0;
}

// Capture the return value of malloc (the allocated pointer)
SEC("uretprobe/libc.so:malloc")
int trace_malloc_ret(struct pt_regs *ctx)
{
    u64 addr = PT_REGS_RC(ctx);  // Return value = pointer to allocated memory
    u64 id = bpf_get_current_pid_tgid();
    
    // Look up the size we stored in the entry probe
    u64 *size = bpf_map_lookup_elem(&alloc_sizes, &id);
    if (!size)
        return 0;  // Missed the entry -- can happen under extreme load
    
    // Record the allocation with full context
    struct alloc_info info = {
        .size = *size,
        .timestamp = bpf_ktime_get_ns(),
    };
    // Capture the userspace call stack -- this is HOW we identify
    // which code path leaked. Without stack traces, "you have a leak"
    // is useless. With them, you know exactly which function allocated
    // the leaked memory.
    bpf_get_stack(ctx, &info.stack, sizeof(info.stack), BPF_F_USER_STACK);
    
    // Store allocation keyed by address -- free() will remove it
    bpf_map_update_elem(&allocations, &addr, &info, BPF_ANY);
    bpf_map_delete_elem(&alloc_sizes, &id);  // Clean up temp storage
    
    return 0;
}

// Track deallocations -- remove the allocation record
SEC("uprobe/libc.so:free")
int trace_free(struct pt_regs *ctx)
{
    u64 addr = PT_REGS_PARM1(ctx);  // free's first arg = pointer to free
    
    // Remove from allocations map -- this pointer is no longer outstanding
    // If the address is not in the map, this is a free of memory we
    // did not track (allocated before we attached), which is fine
    bpf_map_delete_elem(&allocations, &addr);
    
    return 0;
}
// After the measurement period, iterate the allocations map in userspace.
// Every remaining entry is memory that was allocated but never freed
// during our observation window -- a candidate leak.

$ ./memleak -p 1234 30
Attaching to process 1234...
Tracing for 30 seconds...

[08:23:15] Top outstanding allocations:

24576 bytes in 12 allocations from:
    malloc+0x0              # libc entry point
    json_parse+0x123        # <-- this function allocates but never frees
    handle_request+0x456    # called from request handler
    main+0x789

8192 bytes in 4 allocations from:
    malloc+0x0
    create_buffer+0x55      # allocates buffers that outlive the function
    process_data+0x123
    worker_thread+0x456

# Stack traces are symbolized using /proc/<pid>/maps and DWARF debug info
# Without debug symbols, you get raw addresses -- still useful with addr2line

// Set up a perf event that samples the CPU at ~100 Hz
struct perf_event_attr attr = {
    .type = PERF_TYPE_SOFTWARE,
    .config = PERF_COUNT_SW_CPU_CLOCK,  // Sample on CPU clock
    .sample_period = 10000000,  // ~100 Hz (10ms period in nanoseconds)
    .sample_type = PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_TID,
    // CALLCHAIN: capture the full stack trace at each sample
    // TID: record which thread was running
    .exclude_kernel = 0,  // Include kernel stacks (essential for I/O analysis)
    .exclude_user = 0,    // Include user stacks (your application code)
};

// Open the perf event -- pid=-1,cpu=N means "profile all processes on CPU N"
// For per-process profiling, use pid=target,cpu=-1
int fd = perf_event_open(&attr, pid, -1, -1, 0);

// Memory-map the ring buffer for zero-copy sample reading
// The kernel writes samples directly here, userspace reads without syscalls
void *mmap_base = mmap(NULL, page_size * (1 + 2^n),
                       PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

// Process samples from the ring buffer
struct perf_event_header *header;
while ((header = get_next_sample(mmap_base)) != NULL) {
    if (header->type == PERF_RECORD_SAMPLE) {
        // Each sample contains a callchain: an array of instruction pointers
        // from the current function up to the entry point
        process_sample((struct sample_event *)header);
    }
    // Also handle PERF_RECORD_MMAP2 events -- these tell you when shared
    // libraries are loaded, which you need to map addresses to symbol names
}

# Step 1: Collapse stacks -- aggregate identical call chains
stacks = {}
for sample in samples:
    # Reverse the callchain so it reads bottom-up (main -> ... -> leaf)
    # Symbolize addresses using the process's /proc/pid/maps
    stack_str = ";".join(reversed(sample.callchain))
    stacks[stack_str] = stacks.get(stack_str, 0) + 1

# Step 2: Output Brendan Gregg's folded stack format
# Each line is "frame1;frame2;frame3 count"
for stack, count in stacks.items():
    print(f"{stack} {count}")

# Step 3: Pipe through flamegraph.pl to generate SVG
# cat folded.txt | flamegraph.pl > profile.svg
# The width of each box is proportional to how often that function
# appeared in samples -- wider = more CPU time

// Hook tcp_v4_connect -- fires when a process initiates a TCP connection
SEC("kprobe/tcp_v4_connect")
int trace_connect(struct pt_regs *ctx)
{
    struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx);
    u64 ts = bpf_ktime_get_ns();  // Record when the connect() was initiated
    
    // Store the start timestamp keyed by the socket pointer
    // We will look this up when the connection reaches ESTABLISHED state
    bpf_map_update_elem(&connect_start, &sk, &ts, BPF_ANY);
    return 0;
}

// Hook tcp_rcv_state_process -- fires on every TCP state transition
// This is the TCP state machine handler in the kernel
SEC("kprobe/tcp_rcv_state_process")
int trace_state_change(struct pt_regs *ctx)
{
    struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx);
    // BPF_CORE_READ handles kernel struct field access safely across versions
    int state = BPF_CORE_READ(sk, __sk_common.skc_state);
    
    if (state == TCP_ESTABLISHED) {
        // Connection completed -- compute handshake latency
        u64 *start_ts = bpf_map_lookup_elem(&connect_start, &sk);
        if (start_ts) {
            u64 latency = bpf_ktime_get_ns() - *start_ts;
            // latency is in nanoseconds. Typical LAN: 100-500us.
            // Cross-region: 10-100ms. If > 1s, there is a problem.
            emit_event(sk, latency);
            bpf_map_delete_elem(&connect_start, &sk);
        }
    }
    return 0;
}

// Also consider hooking:
// tcp_retransmit_skb -- fires on every retransmission (packet loss indicator)
// tcp_close -- fires when connection ends (measure total connection duration)
// tcp_rcv_established -- fires on every received packet (throughput analysis)