Memory alloc mismatch during db opening

Users Badger
1

When I open a store with existing path or new path, runtime Memstats show there is heap allocated of 80MB which is much more than size of value log files. And surprisingly Residential memory of system shows lower jump than 80MB. It jumps by around 20MB.

The first question is why a huge amount of memory is allocated?
The second question is why there is difference between runtime Alloc memory and residential memory usage of system.

The test code:

package main

import (
	"fmt"
	"runtime"
	"time"
	"github.com/dgraph-io/badger"
)

func main() {
	opened := false
	for {
		time.Sleep(5 * time.Second)
		PrintMemUsage()
		if !opened {
			opts := badger.DefaultOptions
			path := "some_path"
			opts.Dir = path
			opts.ValueDir = path
			opts.ValueLogFileSize = 1 << 26
			_, err := badger.Open(opts)
			if err != nil {
				panic(err)
			}
			opened = true
		}
	}
}

// PrintMemUsage outputs the current, total and OS memory being used. As well as the number
// of garage collection cycles completed.
func PrintMemUsage() {
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
	fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
	fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
	fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
	fmt.Printf("\tNumGC = %v\n", m.NumGC)
}

func bToMb(b uint64) uint64 {
	return b / 1024 / 1024
}

It prints this to stdOut:

Alloc = 0 MiB	TotalAlloc = 0 MiB	Sys = 2 MiB	NumGC = 0
Replaying from value pointer: {Fid:0 Len:0 Offset:0}
Iterating file id: 0
Iteration took: 19.2µs
Alloc = 83 MiB	TotalAlloc = 84 MiB	Sys = 90 MiB	NumGC = 1
Alloc = 83 MiB	TotalAlloc = 84 MiB	Sys = 90 MiB	NumGC = 1
Alloc = 83 MiB	TotalAlloc = 84 MiB	Sys = 90 MiB	NumGC = 1
Alloc = 83 MiB	TotalAlloc = 84 MiB	Sys = 90 MiB	NumGC = 1
Alloc = 83 MiB	TotalAlloc = 84 MiB	Sys = 90 MiB	NumGC = 1
Alloc = 83 MiB	TotalAlloc = 84 MiB	Sys = 90 MiB	NumGC = 1

value log files size is 16kb.
And system Residential memory usage jumps from 38MB to 58MB.

2 
package main

import (
    "fmt"
    "os"
    "runtime"
    "time"

    "github.com/dgraph-io/badger"
)

func main() {
    // Create or open Badger DB
    path := "./data"
    opts := badger.DefaultOptions(path).
        WithMaxTableSize(1 << 20).          // 1 MiB memtable (default was 64 MiB)
        WithValueLogFileSize(1 << 26).      // 64 MiB value log
        WithNumMemtables(1).                // Limit to 1 memtable
        WithValueLogLoadingMode(0).         // FileIO mode (not mmap)
        WithSyncWrites(false)               // Disable sync for testing

    db, err := badger.Open(opts)
    if err != nil {
        panic(err)
    }
    defer db.Close()

    // Monitor memory usage
    opened := false
    for i := 0; i < 5; i++ {
        time.Sleep(5 * time.Second)
        printMemUsage()
        if !opened {
            fmt.Println("Opening Badger DB...")
            opened = true
        }
    }
}

// printMemUsage outputs detailed memory stats
func printMemUsage() {
    var m runtime.MemStats
    runtime.ReadMemStats(&m)

    // Resident memory via /proc/self/stat (Linux only)
    rss := getRSS()

    fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
    fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
    fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
    fmt.Printf("\tRSS = %v MiB", bToMb(uint64(rss)))
    fmt.Printf("\tNumGC = %v\n", m.NumGC)
}

func bToMb(b uint64) uint64 {
    return b / 1024 / 1024
}

// getRSS retrieves resident memory size (Linux-specific)
func getRSS() int64 {
    if runtime.GOOS != "linux" {
        return 0 // Fallback for non-Linux systems
    }
    data, err := os.ReadFile("/proc/self/stat")
    if err != nil {
        return 0
    }
    var ignore int
    var rss int64
    // RSS is the 24th field in /proc/self/stat
    _, err = fmt.Sscanf(string(data), "%d %s %c %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
        &ignore, new(string), new(byte), &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore,
        &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore,
        &ignore, &ignore, &ignore, &rss)
    if err != nil {
        return 0
    }
    return rss * os.Getpagesize() // Convert pages to bytes
}