// syscall_counter.bpf.c
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>

// Why BPF_MAP_TYPE_HASH? We need O(1) lookups by PID, and the PID
// space is sparse (not all values 0..32768 are active). A hash map
// is the right fit. max_entries caps memory usage -- if we hit 10240
// tracked PIDs, new ones simply won't be inserted (graceful degradation).
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10240);
    __type(key, u32);   // PID -- upper 32 bits of pid_tgid
    __type(value, u64); // Cumulative syscall count for this PID
} syscall_count SEC(".maps");

// Separate map for process names so the user-space reader can
// display human-friendly output without calling /proc/<pid>/comm
// (which itself triggers more syscalls and file I/O).
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10240);
    __type(key, u32);       // PID
    __type(value, char[16]); // comm (16 bytes is the kernel TASK_COMM_LEN)
} pid_comm SEC(".maps");

// Why raw_syscalls/sys_enter? This tracepoint fires for EVERY syscall
// regardless of type. It sits at the very top of the syscall entry
// path in the kernel (entry_SYSCALL_64 -> do_syscall_64 -> tracepoint).
// Using a tracepoint (not a kprobe) gives us a stable ABI -- the
// tracepoint format is versioned, so this code survives kernel upgrades.
SEC("tracepoint/raw_syscalls/sys_enter")
int tracepoint__raw_syscalls__sys_enter(struct trace_event_raw_sys_enter *ctx)
{
    // bpf_get_current_pid_tgid() returns (tgid << 32 | tid).
    // We shift right by 32 to get the tgid, which is what user space
    // calls the "PID". The lower 32 bits are the thread ID.
    u32 pid = bpf_get_current_pid_tgid() >> 32;
    u64 *count = bpf_map_lookup_elem(&syscall_count, &pid);
    
    if (count) {
        // Hot path: PID already tracked. This is a simple atomic
        // increment. Note: on a HASH map this is NOT per-CPU, so
        // concurrent threads in the same tgid could race here.
        // For a production tool, use PERCPU_HASH (see below).
        (*count)++;
    } else {
        u64 initial = 1;
        bpf_map_update_elem(&syscall_count, &pid, &initial, BPF_ANY);
        
        // Store comm only on first sighting of this PID.
        // bpf_get_current_comm reads from current->comm in the
        // kernel task_struct -- it is always 16 bytes or fewer.
        char comm[16];
        bpf_get_current_comm(&comm, sizeof(comm));
        bpf_map_update_elem(&pid_comm, &pid, &comm, BPF_ANY);
    }
    
    return 0;
}

// GPL license is required for tracepoint and kprobe BPF programs.
// Without this, the verifier rejects the program at load time.
char LICENSE[] SEC("license") = "GPL";

// Use per-CPU hash for counting. Each CPU gets its own copy of the
// value, eliminating lock contention entirely. The kernel allocates
// NR_CPUS copies of each value behind the scenes. User space reads
// all per-CPU values and sums them -- a small cost paid infrequently.
struct {
    __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
    __uint(max_entries, 10240);
    __type(key, u32);
    __type(value, u64);
} syscall_count SEC(".maps");

# 1. Check for scheduling issues -- are we being preempted?
# perf sched records scheduler events (context switches, migrations)
# and reports which tasks waited longest to get back on a CPU.
perf sched latency -p <pid>

# 2. Check for off-CPU time -- where is the process blocked?
# offcputime uses eBPF to capture stack traces at every point the
# process is descheduled. Long off-CPU stacks point at I/O, locks,
# or page faults. Think of it as the inverse of a CPU profiler.
sudo offcputime-bpfcc -p <pid> 5

# 3. Check for I/O latency -- is the disk the bottleneck?
# biolatency hooks into the block I/O layer (blk_mq_start_request /
# blk_account_io_done) and builds a latency histogram. Spikes in
# the 10ms+ bucket usually mean disk contention or a slow device.
sudo biolatency-bpfcc 5

# 4. Check for memory pressure -- are we swapping or faulting?
# VmSwap > 0 means pages have been evicted to disk. Even small
# amounts of swap can cause tail latency spikes because swap I/O
# is synchronous in the page fault path.
cat /proc/<pid>/status | grep -E "VmRSS|VmSwap"
sar -B 1 5  # Page faults -- look at pgmajfault (disk-backed)

# 5. Check for lock contention -- last because it is expensive.
# perf lock instruments the kernel's locking primitives (mutexes,
# spinlocks, rwlocks) and shows which locks are most contended.
sudo perf lock record -p <pid> -- sleep 5
sudo perf lock report

# Trace slow syscalls -- this is the single most useful one-liner
# for latency debugging. It instruments the raw_syscalls tracepoints,
# which fire at the very beginning and end of EVERY syscall. We
# record a timestamp on entry, compute the delta on exit, and only
# print if the syscall took longer than our threshold (10ms here).
# The @slow map gives us a histogram of which syscalls are slowest.
sudo bpftrace -e '
tracepoint:raw_syscalls:sys_enter /pid == $TARGET/ {
    @start[tid] = nsecs;
}
tracepoint:raw_syscalls:sys_exit /@start[tid]/ {
    $lat = (nsecs - @start[tid]) / 1000000;
    if ($lat > 10) {
        printf("%s syscall %d took %d ms\n", comm, args->id, $lat);
        @slow[args->id] = count();
    }
    delete(@start[tid]);
}'
// Replace $TARGET with the PID. Run for 30-60 seconds during a spike.
// If you see futex or epoll_wait dominating @slow, look at lock
// contention or slow downstream dependencies. If read/write dominate,
// look at disk I/O or network.

# Java
jstat -gc <pid> 1000

# Go - trace runtime events
GODEBUG=gctrace=1 ./myapp

Usage < memory.high
└── Normal operation -- kernel allocates pages freely

Usage > memory.high (the "throttle" zone)
└── Kernel activates direct reclaim on the allocating task
└── The process is forced to help reclaim pages before getting new ones
└── Reclaim pressure increases (page cache evicted first, then anon pages)
└── Application slows down -- this is INTENTIONAL back-pressure
└── Unlike memory.max, this does NOT trigger OOM -- it slows you down

Usage > memory.max (the "hard wall")
└── Allocation fails -- the kernel cannot give you more memory
└── OOM killer invoked WITHIN this cgroup (not system-wide)
└── Process with highest oom_score_adj in the cgroup is killed
└── If all processes die, the cgroup is empty and gets cleaned up

# Check kernel logs -- the OOM killer always leaves a trace in dmesg.
# The kernel's oom_kill_process() function logs the victim PID, its
# memory usage breakdown, and every process in the cgroup at kill time.
dmesg | grep -i "oom\|killed"

# Check container events -- Docker's event stream captures OOM events
# from the cgroup notification. This tells you WHEN it happened.
docker events --filter 'event=oom'

# Get detailed OOM context from journald -- the -A 10 grabs the
# full memory dump that the kernel prints, including per-process
# RSS breakdowns for every task in the cgroup.
journalctl -k | grep -A 10 "invoked oom-killer"

Memory cgroup out of memory: Killed process 12345 (myapp)
total-pgfault:15000 total-pgmajfault:50
anon-rss:1048576kB file-rss:0kB shmem-rss:0kB

Key indicators (a senior engineer reads these like a doctor reads blood work):
- anon-rss: Heap, stack, anonymous mmap -- this is YOUR code's allocations
- file-rss: Mapped files (shared libraries, mmap'd data) -- can be evicted
- shmem-rss: Shared memory segments (tmpfs, IPC) -- charged to the cgroup
- total-pgmajfault: Major page faults = disk I/O. High values here mean
  the system was already under pressure before the OOM event.

# Track allocations over time
docker stats <container>

# Get detailed memory breakdown
cat /sys/fs/cgroup/<path>/memory.stat

# Profile with BPF
sudo memleak-bpfcc -p <pid>

resources:
  requests:
    memory: "256Mi"  # Used by the scheduler for bin-packing decisions
  limits:
    memory: "512Mi"  # Maps directly to memory.max in cgroup v2
    # Why 2x? Because page cache + GC overhead + kernel slab can
    # easily double your application's "heap" memory usage.
    # If your app reports 200Mi heap, you need 400-512Mi as the limit.

┌─────────────────────────────────────────────────────────────────────┐
│                     PACKET RECEIVE PATH                              │
├─────────────────────────────────────────────────────────────────────┤
│                                                                      │
│  1. NIC receives packet                                              │
│     └─→ DMA to ring buffer in memory                                │
│     └─→ Raise interrupt                                             │
│                                                                      │
│  2. Interrupt handler (hardirq)                                      │
│     └─→ Acknowledge interrupt                                       │
│     └─→ Schedule NAPI poll (softirq)                                │
│     └─→ Disable further interrupts for this queue                  │
│                                                                      │
│  3. NAPI poll (softirq context)                                     │
│     └─→ Poll ring buffer for packets                                │
│     └─→ Allocate sk_buff structures                                 │
│     └─→ Process up to budget packets                                │
│     └─→ Re-enable interrupts if done                                │
│                                                                      │
│  4. Network stack processing                                         │
│     └─→ XDP (if attached) - earliest hook                          │
│     └─→ tc ingress                                                   │
│     └─→ netfilter/iptables                                          │
│     └─→ IP routing                                                   │
│     └─→ TCP/UDP processing                                          │
│                                                                      │
│  5. Socket layer                                                     │
│     └─→ Socket buffer (sk_buff queue)                               │
│     └─→ Wake up waiting application                                 │
│                                                                      │
│  6. Application                                                      │
│     └─→ read()/recv() copies to user space                         │
│     └─→ Process data                                                │
│                                                                      │
└─────────────────────────────────────────────────────────────────────┘

# Enable RSS (Receive Side Scaling) -- this tells the NIC to hash
# incoming packets (by 5-tuple: src/dst IP, src/dst port, protocol)
# and distribute them across multiple RX queues. Each queue gets its
# own interrupt, so multiple CPUs process packets in parallel.
# Without RSS, one CPU handles ALL packets -- a guaranteed bottleneck.
ethtool -L eth0 combined 8

# Configure interrupt coalescing -- instead of firing an interrupt
# for every single packet, wait up to 50 microseconds OR 64 packets
# before interrupting. This trades a tiny bit of latency for a huge
# reduction in interrupt overhead at high packet rates.
ethtool -C eth0 rx-usecs 50 rx-frames 64

# Pin interrupts to CPUs -- ensure each NIC queue's interrupt is
# handled by a specific CPU. This keeps packet processing cache-hot
# on that CPU. Without pinning, the kernel may migrate interrupts
# across CPUs, causing cache misses on the packet data.
echo 1 > /proc/irq/XX/smp_affinity

# Increase socket buffer sizes -- the default rmem_max (212992 bytes)
# is far too small for high-throughput applications. At 10Gbps, the
# kernel can fill that buffer in ~170 microseconds. If the application
# does not read fast enough, packets are dropped at the socket layer.
sysctl -w net.core.rmem_max=26214400  # 25 MB

# Enable busy polling -- instead of sleeping in epoll_wait and being
# woken by a softirq, the application actively polls the NIC driver
# for new packets. This eliminates the softirq-to-application wakeup
# latency (~5-10us) at the cost of burning CPU cycles while polling.
# Only worth it for latency-sensitive apps with dedicated CPU cores.
sysctl -w net.core.busy_poll=50   # poll for 50us before sleeping
sysctl -w net.core.busy_read=50   # same for synchronous reads

# Tune NAPI GRO flush timeout -- GRO (Generic Receive Offload)
# coalesces multiple small packets into fewer large ones before
# passing them up the stack. This timeout controls how long the
# kernel waits for more packets to coalesce.
echo 64 > /sys/class/net/eth0/gro_flush_timeout

// SO_REUSEPORT lets multiple sockets bind to the same port.
// The kernel distributes incoming connections across them using
// a hash. Each thread gets its own socket and its own accept queue,
// eliminating the thundering herd problem where all threads wake
// up for a single incoming connection.
int opt = 1;
setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));

// recvmmsg receives multiple messages in a single syscall.
// Each syscall has ~1us of overhead (user-to-kernel transition,
// register save/restore). Batching amortizes this across BATCH_SIZE
// messages. At 1M pps, this alone can save ~50% of CPU time.
struct mmsghdr msgs[BATCH_SIZE];
int ret = recvmmsg(fd, msgs, BATCH_SIZE, 0, NULL);

# /etc/default/grub
GRUB_CMDLINE_LINUX="
    isolcpus=2,3,4,5              # Remove from CFS scheduler's CPU mask --
                                   # the scheduler will NEVER place a regular
                                   # task on these CPUs unless explicitly pinned
    nohz_full=2,3,4,5             # Disable the periodic timer tick (HZ) on
                                   # these CPUs when only one task is running.
                                   # Without this, the kernel interrupts every
                                   # 1-4ms for scheduling housekeeping.
    rcu_nocbs=2,3,4,5             # Offload RCU (Read-Copy-Update) callbacks
                                   # to housekeeping CPUs. RCU is the kernel's
                                   # lock-free synchronization mechanism. Its
                                   # callbacks can add 10-50us jitter.
    irqaffinity=0,1               # Keep ALL hardware IRQs on CPUs 0,1 only.
                                   # A NIC interrupt on an isolated CPU would
                                   # cause 1-5us of jitter per interrupt.
    intel_pstate=disable          # Disable dynamic frequency scaling entirely.
                                   # P-state transitions take 10-100us and
                                   # cause unpredictable performance variation.
    processor.max_cstate=0        # Disable ALL C-states -- CPU never sleeps.
                                   # Waking from C6 takes 100-200us. For sub-ms
                                   # latency, even C1 (1-2us) is too much.
    idle=poll                     # When idle, spin in a tight loop instead of
                                   # entering any idle state. Burns power but
                                   # guarantees zero wake-up latency.
    nosoftlockup                  # Don't check for soft lockups on isolated
                                   # CPUs. The watchdog itself causes jitter.
    nmi_watchdog=0                # Disable the NMI watchdog (perf-based hard
                                   # lockup detector). NMIs cannot be masked
                                   # and cause ~1us interruptions.
    audit=0                       # Disable the audit subsystem entirely.
                                   # Audit hooks in the syscall path add
                                   # measurable overhead per system call.
    skew_tick=1                   # Randomize timer ticks across CPUs so they
                                   # don't all fire simultaneously and cause
                                   # a burst of cache line bouncing.
"

# Pin critical threads to isolated CPUs
taskset -c 2,3,4,5 ./trading_app

# Move all other processes away
for pid in $(ps -eo pid --no-headers); do
    taskset -p 0x3 $pid 2>/dev/null  # CPUs 0,1
done

# Move all IRQs to housekeeping CPUs
for irq in /proc/irq/*/smp_affinity; do
    echo 3 > $irq 2>/dev/null  # CPUs 0,1
done

# Check no IRQs on isolated CPUs
cat /proc/interrupts | awk '{print $1, $3, $4}'

# Check no kernel threads on isolated CPUs
ps -eo pid,psr,comm | grep -E "^\s*[0-9]+\s+[2-5]"

# Check timer behavior
perf stat -C 2,3,4,5 -e irq_vectors:local_timer_entry sleep 10

# Use cyclictest
cyclictest -m -p 99 -i 100 -h 1000 -D 1m -a 2 -t 1

# Interpret results:
# Min: 1 μs    (good)
# Avg: 2 μs    (good)
# Max: 50 μs   (acceptable for many use cases)
# Max: 500 μs  (investigate!)

┌─────────────────────────────────────────────────────────────────────┐
│                    METRICS COLLECTION ARCHITECTURE                   │
├─────────────────────────────────────────────────────────────────────┤
│                                                                      │
│   Method 1: Poll cgroup files                                        │
│   ─────────────────────────                                         │
│   For each container:                                                │
│     - Read /sys/fs/cgroup/<id>/cpu.stat                            │
│     - Read /sys/fs/cgroup/<id>/memory.current                      │
│     - Read /sys/fs/cgroup/<id>/io.stat                             │
│                                                                      │
│   Pros: Simple, no kernel changes                                   │
│   Cons: File system overhead, 10K reads/second                      │
│                                                                      │
│   ─────────────────────────────────────────────────────────────────  │
│                                                                      │
│   Method 2: eBPF-based collection                                    │
│   ─────────────────────────────                                     │
│   - Hook scheduler for CPU accounting                               │
│   - Hook memory allocator for memory tracking                       │
│   - Hook block layer for I/O                                        │
│   - Aggregate per-cgroup in BPF maps                                │
│                                                                      │
│   Pros: Lower overhead, real-time data                              │
│   Cons: Complex, needs kernel support                               │
│                                                                      │
│   ─────────────────────────────────────────────────────────────────  │
│                                                                      │
│   Recommended: Hybrid approach                                       │
│   - Use cgroup files for infrequent metrics (memory limits)         │
│   - Use eBPF for high-frequency metrics (CPU, I/O)                  │
│   - Batch reads, use inotify for changes                            │
│                                                                      │
└─────────────────────────────────────────────────────────────────────┘

// Key optimization: open file descriptors ONCE at startup, then
// reuse them on every collection cycle. Each open() in the cgroup
// filesystem traverses the VFS path lookup, allocates an fd, and
// creates a struct file -- about 2-5us per call. For 10K containers
// with 3 files each, that is 30K opens = 60-150ms of pure overhead
// per cycle. Keeping FDs open amortizes this to zero.
struct container_fds {
    int cpu_stat_fd;
    int memory_current_fd;
    int io_stat_fd;
};

void collect_metrics(struct container_fds *fds, int count) {
    for (int i = 0; i < count; i++) {
        // Use pread with offset 0 instead of lseek + read.
        // pread is atomic and avoids the overhead of maintaining
        // the file position. For cgroup pseudo-files, reading from
        // offset 0 always returns the current value -- the kernel
        // regenerates the content on each read (these are not real
        // files on disk; they are backed by seq_file handlers that
        // call into the cgroup controller code).
        char buf[4096];
        pread(fds[i].cpu_stat_fd, buf, sizeof(buf), 0);
        parse_cpu_stat(buf, &metrics[i].cpu);
        
        pread(fds[i].memory_current_fd, buf, sizeof(buf), 0);
        metrics[i].memory = atol(buf);
    }
}

// Track CPU time per cgroup
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10240);
    __type(key, u64);  // cgroup id
    __type(value, u64);  // total CPU time ns
} cgroup_cpu SEC(".maps");

SEC("tracepoint/sched/sched_switch")
int trace_switch(struct trace_event_raw_sched_switch *ctx) {
    u64 cgroup_id = bpf_get_current_cgroup_id();
    u64 now = bpf_ktime_get_ns();
    
    // Record time for previous task's cgroup
    u64 *last_time = bpf_map_lookup_elem(&last_switch, &ctx->prev_pid);
    if (last_time) {
        u64 delta = now - *last_time;
        u64 *total = bpf_map_lookup_elem(&cgroup_cpu, &cgroup_id);
        if (total) {
            *total += delta;
        }
    }
    
    // Record switch time for next task
    bpf_map_update_elem(&last_switch, &ctx->next_pid, &now, BPF_ANY);
    return 0;
}

# Check seccomp profile -- seccomp filters run BEFORE any other
# security check in the syscall path. If seccomp blocks a syscall,
# you get EPERM/ENOSYS but no audit log entry from SELinux/AppArmor.
# This is the most common "invisible" denial.
docker inspect <container> | jq '.[0].HostConfig.SecurityOpt'

# Check AppArmor/SELinux -- these are LSM hooks that run AFTER
# DAC checks. A process can pass Unix permission checks but still
# be denied by mandatory access control policy.
docker inspect <container> | jq '.[0].AppArmorProfile'
getenforce  # SELinux status: Enforcing = blocking, Permissive = logging only

# Check capabilities -- the kernel checks capabilities in capable()
# which is called from hundreds of places. A missing CAP_SYS_ADMIN
# or CAP_NET_RAW is often the culprit for "works as root, fails as
# non-root" problems.
docker inspect <container> | jq '.[0].HostConfig.CapAdd'
docker inspect <container> | jq '.[0].HostConfig.CapDrop'

# Trace syscall failures -- strace intercepts via ptrace, so it sees
# the error return value from the kernel. Look for EPERM (permission
# denied) and EACCES (access denied) -- they come from different
# security layers (capabilities vs. DAC vs. LSM).
strace -f docker run --rm myimage 2>&1 | grep -i "denied\|EPERM\|EACCES"

# Check audit logs -- SELinux and AppArmor write AVC denial records
# to the audit subsystem. This is your definitive source of truth
# for MAC denials.
ausearch -m avc -ts recent

# Get detailed memory breakdown -- memory.stat is the kernel's
# authoritative accounting of every byte charged to this cgroup.
# The kernel maintains these counters in struct mem_cgroup, updating
# them on every page charge/uncharge event.
cat /sys/fs/cgroup/<path>/memory.stat

# Key fields (these map directly to kernel page types):
# anon    - Anonymous memory (heap via brk/mmap, stack, mmap(MAP_ANONYMOUS))
# file    - Page cache (file-backed pages read into memory)
# kernel  - Kernel memory charged to this cgroup (since cgroup v2)
# slab    - Slab allocations (dentry cache, inode cache, socket buffers)

# Check for file cache -- this is the #1 cause of "missing" memory.
# The kernel caches every file read into page cache. Your app reads
# a 1GB log file once? That 1GB stays in page cache (charged to your
# cgroup) until memory pressure forces reclaim.
grep -E "^(anon|file|kernel)" /sys/fs/cgroup/<path>/memory.stat

# Inside container -- /proc/meminfo shows the SYSTEM-WIDE view, not
# the cgroup view. This is a common source of confusion. The numbers
# here do not match memory.stat because /proc/meminfo is not cgroup-aware.
cat /proc/meminfo | grep -E "Cached|Buffers|Slab"

# Check for memory mapped files -- each line in /proc/<pid>/maps
# is a VMA (Virtual Memory Area). File-backed VMAs (those with a
# non-zero device and inode) contribute to file RSS.
cat /proc/<pid>/maps | grep -v "00000000 00:00" | wc -l

# 1. Confirm the leak exists and measure its rate
# /proc/<pid>/fd is a directory with a symlink for each open fd.
# Count them over time to establish the leak rate.
watch -n 5 'ls /proc/<pid>/fd | wc -l'

# 2. Identify WHAT is being leaked (files? sockets? pipes?)
# Each symlink points to the actual resource.
ls -la /proc/<pid>/fd | awk '{print $NF}' | sort | uniq -c | sort -rn | head -20
# If you see thousands of "socket:[xxxxx]" entries, it is socket leaks.
# If you see thousands of "/tmp/some-pattern", it is file leaks.

# 3. Find the code path that opens without closing -- use bpftrace
# to capture stack traces at every open() that is NOT followed by
# a close() of the same fd within a timeout window.
sudo bpftrace -e '
tracepoint:syscalls:sys_enter_openat /pid == TARGET/ {
    @open_stacks[tid] = ustack;
}'

# Check cgroup CPU stats -- nr_throttled and throttled_usec are the key
cat /sys/fs/cgroup/cpu.stat
# usage_usec 12345678
# nr_periods 5000        <- total scheduling periods
# nr_throttled 2500      <- periods where throttling occurred (50%!)
# throttled_usec 8000000 <- total time spent throttled (8 seconds)

points = process_RSS + process_swap_usage + child_RSS