Implement Redis persistence with RDB snapshots and AOF logging for durability
Chapter 4: Persistence
A database that loses all data on restart isn’t very useful! In this chapter, we’ll implement both persistence strategies Redis uses: RDB snapshots and AOF (Append-Only File) logging.This chapter teaches one of the most fundamental trade-offs in all of systems engineering: durability vs. performance. Every write to disk costs time. The question is when you pay that cost and how much data you are willing to lose if the power goes out. RDB and AOF represent two different answers to this question, and understanding both will inform your thinking about databases, message queues, and any system that claims to “persist” data.
Redis forks to create snapshots without blocking the main process.
internal/persistence/bgsave.go
package persistenceimport ( "log" "sync" "time")// BackgroundSaver handles background RDB savestype BackgroundSaver struct { store *store.Store path string lastSave time.Time saving bool mu sync.Mutex // Auto-save conditions saveAfter map[int]int // changes -> seconds}// NewBackgroundSaver creates a new background saverfunc NewBackgroundSaver(store *store.Store, path string) *BackgroundSaver { return &BackgroundSaver{ store: store, path: path, lastSave: time.Now(), saveAfter: map[int]int{ 1: 900, // After 900 sec if at least 1 change 10: 300, // After 300 sec if at least 10 changes 10000: 60, // After 60 sec if at least 10000 changes }, }}// Save performs a background savefunc (bs *BackgroundSaver) Save() error { bs.mu.Lock() if bs.saving { bs.mu.Unlock() return fmt.Errorf("save already in progress") } bs.saving = true bs.mu.Unlock() defer func() { bs.mu.Lock() bs.saving = false bs.lastSave = time.Now() bs.mu.Unlock() }() // In Go, we can't fork like C, so we use a goroutine // For true copy-on-write, you'd need to use OS-specific calls return bs.saveSnapshot()}func (bs *BackgroundSaver) saveSnapshot() error { writer, err := NewRDBWriter(bs.path) if err != nil { return err } if err := writer.WriteHeader(); err != nil { return err } if err := writer.SelectDB(0); err != nil { return err } // Iterate over all keys bs.store.ForEach(func(key string, value interface{}) { switch v := value.(type) { case []byte: writer.WriteString(key, v) case *store.List: writer.WriteList(key, v.LRange(0, -1)) case *store.Set: writer.WriteSet(key, v.Members()) case *store.Hash: writer.WriteHash(key, v.GetAll()) case *store.ZSet: writer.WriteZSet(key, v.Range(0, -1, true)) } }) if err := writer.WriteFooter(); err != nil { return err } return writer.Close(bs.path)}// StartAutoSave starts the automatic save checkerfunc (bs *BackgroundSaver) StartAutoSave() { go func() { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() for range ticker.C { bs.checkAutoSave() } }()}func (bs *BackgroundSaver) checkAutoSave() { bs.mu.Lock() if bs.saving { bs.mu.Unlock() return } elapsed := int(time.Since(bs.lastSave).Seconds()) changes := bs.store.ChangesSinceLastSave() bs.mu.Unlock() for requiredChanges, requiredSeconds := range bs.saveAfter { if changes >= requiredChanges && elapsed >= requiredSeconds { log.Printf("Auto-save triggered: %d changes in %d seconds", changes, elapsed) bs.Save() return } }}
Part 3: Append-Only File (AOF)
AOF logs every write command for maximum durability.
AOF Writer
internal/persistence/aof.go
package persistenceimport ( "bufio" "fmt" "os" "sync" "time")// AOFSyncPolicy defines when to sync AOF to disktype AOFSyncPolicy intconst ( SyncAlways AOFSyncPolicy = iota // Sync after every write SyncEverySec // Sync every second SyncNo // Let OS decide)// AOFWriter handles append-only file operationstype AOFWriter struct { file *os.File writer *bufio.Writer policy AOFSyncPolicy mu sync.Mutex // For SyncEverySec lastSync time.Time}// NewAOFWriter opens or creates an AOF filefunc NewAOFWriter(path string, policy AOFSyncPolicy) (*AOFWriter, error) { file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { return nil, err } aof := &AOFWriter{ file: file, writer: bufio.NewWriter(file), policy: policy, lastSync: time.Now(), } if policy == SyncEverySec { go aof.syncLoop() } return aof, nil}// AppendCommand logs a command to AOF in RESP formatfunc (a *AOFWriter) AppendCommand(args []string) error { a.mu.Lock() defer a.mu.Unlock() // Write as RESP array fmt.Fprintf(a.writer, "*%d\r\n", len(args)) for _, arg := range args { fmt.Fprintf(a.writer, "$%d\r\n%s\r\n", len(arg), arg) } if a.policy == SyncAlways { if err := a.writer.Flush(); err != nil { return err } return a.file.Sync() } return nil}func (a *AOFWriter) syncLoop() { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() for range ticker.C { a.mu.Lock() if time.Since(a.lastSync) >= time.Second { a.writer.Flush() a.file.Sync() a.lastSync = time.Now() } a.mu.Unlock() }}// Close closes the AOF filefunc (a *AOFWriter) Close() error { a.mu.Lock() defer a.mu.Unlock() a.writer.Flush() a.file.Sync() return a.file.Close()}
AOF Loader
internal/persistence/aof_loader.go
package persistenceimport ( "bufio" "fmt" "io" "os" "strconv" "strings")// AOFLoader replays AOF commands to rebuild statetype AOFLoader struct { file *os.File reader *bufio.Reader}// NewAOFLoader opens an AOF file for readingfunc NewAOFLoader(path string) (*AOFLoader, error) { file, err := os.Open(path) if err != nil { if os.IsNotExist(err) { return nil, nil // No AOF file, that's okay } return nil, err } return &AOFLoader{ file: file, reader: bufio.NewReader(file), }, nil}// ReadCommand reads the next command from AOFfunc (l *AOFLoader) ReadCommand() ([]string, error) { // Read array marker line, err := l.reader.ReadString('\n') if err != nil { if err == io.EOF { return nil, io.EOF } return nil, err } line = strings.TrimSuffix(line, "\r\n") if !strings.HasPrefix(line, "*") { return nil, fmt.Errorf("expected array, got: %s", line) } count, err := strconv.Atoi(line[1:]) if err != nil { return nil, err } args := make([]string, count) for i := 0; i < count; i++ { // Read bulk string marker line, err := l.reader.ReadString('\n') if err != nil { return nil, err } line = strings.TrimSuffix(line, "\r\n") if !strings.HasPrefix(line, "$") { return nil, fmt.Errorf("expected bulk string, got: %s", line) } length, err := strconv.Atoi(line[1:]) if err != nil { return nil, err } // Read the actual string buf := make([]byte, length+2) // +2 for \r\n if _, err := io.ReadFull(l.reader, buf); err != nil { return nil, err } args[i] = string(buf[:length]) } return args, nil}// Close closes the AOF filefunc (l *AOFLoader) Close() error { return l.file.Close()}// ReplayAOF replays all commands from AOF to rebuild statefunc ReplayAOF(path string, handler func([]string) error) (int, error) { loader, err := NewAOFLoader(path) if err != nil { return 0, err } if loader == nil { return 0, nil // No AOF file } defer loader.Close() count := 0 for { cmd, err := loader.ReadCommand() if err == io.EOF { break } if err != nil { return count, err } if err := handler(cmd); err != nil { return count, err } count++ } return count, nil}
Part 4: AOF Rewriting
Over time, AOF files grow. Rewriting compacts them:
┌─────────────────────────────────────────────────────────────────────────────┐│ AOF REWRITING │├─────────────────────────────────────────────────────────────────────────────┤│ ││ BEFORE (AOF file): AFTER (Rewritten): ││ ───────────────── ───────────────── ││ SET counter 1 SET counter 100 ││ INCR counter SET name "final" ││ INCR counter RPUSH list "a" "b" "c" ││ SET name "foo" ││ INCR counter Much smaller! ││ ... Only final state. ││ SET name "final" ││ INCR counter (x97 more) ││ RPUSH list "a" ││ RPUSH list "b" ││ RPUSH list "c" ││ ││ Rewriting creates a new AOF from current database state, ││ not by reading the old AOF. ││ │└─────────────────────────────────────────────────────────────────────────────┘
internal/persistence/aof_rewrite.go
package persistenceimport ( "fmt" "os")// AOFRewriter handles AOF compactiontype AOFRewriter struct { store *store.Store path string}// NewAOFRewriter creates a new AOF rewriterfunc NewAOFRewriter(store *store.Store, path string) *AOFRewriter { return &AOFRewriter{store: store, path: path}}// Rewrite creates a new compacted AOF filefunc (r *AOFRewriter) Rewrite() error { tmpPath := r.path + ".rewrite" // Create new AOF with current state writer, err := NewAOFWriter(tmpPath, SyncNo) if err != nil { return err } // Write current state as commands r.store.ForEach(func(key string, value interface{}) { switch v := value.(type) { case []byte: writer.AppendCommand([]string{"SET", key, string(v)}) case *store.List: elements := v.LRange(0, -1) if len(elements) > 0 { args := []string{"RPUSH", key} for _, e := range elements { args = append(args, string(e)) } writer.AppendCommand(args) } case *store.Set: members := v.Members() if len(members) > 0 { args := []string{"SADD", key} args = append(args, members...) writer.AppendCommand(args) } case *store.Hash: fields := v.GetAll() if len(fields) > 0 { args := []string{"HSET", key} for field, val := range fields { args = append(args, field, string(val)) } writer.AppendCommand(args) } case *store.ZSet: members := v.Range(0, -1, true) if len(members) > 0 { args := []string{"ZADD", key} for _, m := range members { args = append(args, fmt.Sprintf("%g", m.Score), m.Member) } writer.AppendCommand(args) } } }) writer.Close() // Atomic rename return os.Rename(tmpPath, r.path)}
Part 5: Integrating Persistence
internal/server/server.go
package serverimport ( "log")type Server struct { store *store.Store aof *persistence.AOFWriter rdb *persistence.BackgroundSaver}func NewServer(config Config) (*Server, error) { store := store.NewStore() // Load existing data if config.AOFEnabled { count, err := persistence.ReplayAOF(config.AOFPath, func(cmd []string) error { // Execute command on store return store.ExecuteCommand(cmd) }) if err != nil { return nil, err } log.Printf("Loaded %d commands from AOF", count) } else if config.RDBEnabled { // Load from RDB if err := loadRDB(config.RDBPath, store); err != nil { log.Printf("Warning: could not load RDB: %v", err) } } s := &Server{store: store} // Initialize AOF if config.AOFEnabled { aof, err := persistence.NewAOFWriter(config.AOFPath, config.AOFSyncPolicy) if err != nil { return nil, err } s.aof = aof } // Initialize RDB background saver if config.RDBEnabled { s.rdb = persistence.NewBackgroundSaver(store, config.RDBPath) s.rdb.StartAutoSave() } return s, nil}// HandleCommand processes a command and logs to AOF if neededfunc (s *Server) HandleCommand(cmd []string) (interface{}, error) { result, err := s.store.ExecuteCommand(cmd) if err != nil { return nil, err } // Log write commands to AOF if s.aof != nil && isWriteCommand(cmd[0]) { s.aof.AppendCommand(cmd) } return result, nil}func isWriteCommand(cmd string) bool { switch strings.ToUpper(cmd) { case "SET", "DEL", "INCR", "DECR", "LPUSH", "RPUSH", "LPOP", "RPOP", "SADD", "SREM", "HSET", "HDEL", "ZADD", "ZREM": return true } return false}
Exercises
Exercise 1: Implement Expiration Persistence
Persist key expiration times:
// In RDB: Store expire time before key-value// OpCodeExpire + timestamp + key + value// In AOF: Log EXPIRE commands// Or use SETEX format
Exercise 2: Implement BGREWRITEAOF
Add the BGREWRITEAOF command:
// 1. Create new AOF in background// 2. Buffer new commands during rewrite// 3. Append buffer to new AOF// 4. Atomic swap
Exercise 3: Implement RDB Compression
Add LZF compression to RDB:
// Compress string values > 64 bytes// Use special type marker for compressed strings// TypeCompressedString = 0xC0
Key Takeaways
RDB Snapshots
Point-in-time backups, compact but may lose recent data