A service's RSS grows but heap dumps and GC logs look clean: how do you investigate an off-heap leak?

package com.bank.diagnostic;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

public class DirectBufferLeak {
    public static void main(String[] args) {
        List<ByteBuffer> leakedBuffers = new ArrayList<>();

        // Simulates a service that allocates direct buffers
        // and forgets to release them
        for (int i = 0; i < 1000; i++) {
            ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 1024); // 1 MB
            leakedBuffers.add(buffer); // keeps the wrapper alive → never freed
        }

        System.out.println("Leaked " + leakedBuffers.size()
                + " direct buffers (" + leakedBuffers.size() + " MB off-heap)");
        // Heap dump shows only tiny wrapper objects
        // RSS shows 1 GB growth
    }
}

// Netty: WRONG - missing release()
ByteBuf buf = ctx.alloc().buffer(1024);
// ... use buf ...
// Forgot buf.release(); → pooled arena memory leak

// CORRECT
ByteBuf buf = ctx.alloc().buffer(1024);
try {
    // ... use buf ...
} finally {
    buf.release(); // always release
}

# Enable NMT (adds ~5-10% overhead)
java -XX:NativeMemoryTracking=summary -jar bank-service.jar

# Take a baseline
jcmd <pid> VM.native_memory baseline

# Wait for RSS to grow, then compare
jcmd <pid> VM.native_memory summary.diff

# Cap direct memory at 256 MB
java -XX:MaxDirectMemorySize=256m -jar bank-service.jar

# If exceeded:
# java.lang.OutOfMemoryError: Direct buffer memory
# This is better than silently eating all available RAM

package com.bank.diagnostic;

import java.lang.management.BufferPoolMXBean;
import java.lang.management.ManagementFactory;

public class MonitorDirectBuffers {
    public static void main(String[] args) {
        for (BufferPoolMXBean pool :
                ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) {
            System.out.println(pool.getName()
                    + ": count=" + pool.getCount()
                    + ", used=" + (pool.getMemoryUsed() / 1024 / 1024) + "MB"
                    + ", capacity=" + (pool.getTotalCapacity() / 1024 / 1024) + "MB");
        }
    }
}

direct: count=42, used=84MB, capacity=84MB
mapped: count=3, used=12MB, capacity=12MB

# pmap: show memory regions
pmap -x <pid> | sort -k3 -n | tail -20

# async-profiler: native allocation sampling
./profiler.sh -e malloc -d 60 -f malloc-profile.html <pid>
# Shows which native call sites are allocating
# Pinpoints the leaking library or JNI path

// Thread leak: each thread reserves a native stack
for (int i = 0; i < 10_000; i++) {
    new Thread(() -> {
        try { Thread.sleep(Long.MAX_VALUE); }
        catch (InterruptedException e) { Thread.currentThread().interrupt(); }
    }).start();
}
// 10,000 threads × 1 MB stack = 10 GB native memory
// Heap dump shows nothing

# Check thread count
jcmd <pid> Thread.print | grep -c "nid="

# Reduce glibc arena count to limit fragmentation
export MALLOC_ARENA_MAX=4
java -jar bank-service.jar

# Or switch to jemalloc with profiling
LD_PRELOAD=/usr/lib/libjemalloc.so MALLOC_CONF=prof:true \
  java -jar bank-service.jar

# Complete off-heap investigation checklist:
# 1. Confirm heap is clean: GC logs, jstat
# 2. Enable NMT and take baseline + diff
# 3. Check direct BufferPool MBean
# 4. Check thread count: jcmd Thread.print
# 5. Check Metaspace: jcmd VM.classloader_stats
# 6. pmap for growing anonymous regions
# 7. async-profiler -e malloc for allocation sites
# 8. Test MALLOC_ARENA_MAX for fragmentation