From 3891bb9212bba5ea9fa3e0d9c423de8e3f15fef4 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 26 Apr 2026 19:06:26 +0800 Subject: [PATCH] docs: add database backup and snapshot implementation plan --- .../2026-04-26-database-backup-snapshot.md | 1591 +++++++++++++++++ 1 file changed, 1591 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-26-database-backup-snapshot.md diff --git a/docs/superpowers/plans/2026-04-26-database-backup-snapshot.md b/docs/superpowers/plans/2026-04-26-database-backup-snapshot.md new file mode 100644 index 00000000..38101a02 --- /dev/null +++ b/docs/superpowers/plans/2026-04-26-database-backup-snapshot.md @@ -0,0 +1,1591 @@ +# Database Backup & Snapshot Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add backup, scheduled snapshot, export, and restore functionality for both SQLite and MariaDB, operable via panel UI and x-ui.sh CLI. + +**Architecture:** New `BackupService` in web/service handles mysqldump/sqlite3 dump execution, tar.gz packaging, and file management. `BackupController` exposes REST API endpoints. `BackupJob` is a cron-scheduled job that delegates to BackupService. Settings stored via existing `SettingService`/`AllSetting` reflection-based persistence. CLI subcommands in main.go call the same backup/restore logic directly. x-ui.sh wraps CLI commands. + +**Tech Stack:** Go (Gin, GORM, exec.Command), bash (x-ui.sh), Vue.js + Ant Design Vue (panel UI), robfig/cron v3 (scheduling) + +--- + +### Task 1: Add backup settings to entity and setting service + +**Files:** +- Modify: `web/entity/entity.go:113-124` +- Modify: `web/service/setting.go:113-130` (add defaults) +- Modify: `web/service/setting.go:229-250` (add to `systemIntegration` group or new `backup` group) + +- [ ] **Step 1: Add backup fields to AllSetting struct** + +In `web/entity/entity.go`, add before the closing of the struct (line 124, after `TurnstileSecretKey`): + +```go + // Backup settings + BackupEnabled bool `json:"backupEnabled" form:"backupEnabled"` // Enable scheduled backups + BackupFrequency string `json:"backupFrequency" form:"backupFrequency"` // hourly, every12h, daily, weekly + BackupHour int `json:"backupHour" form:"backupHour"` // hour of day (0-23) + BackupMaxCount int `json:"backupMaxCount" form:"backupMaxCount"` // max backups to retain (1-100) +``` + +- [ ] **Step 2: Add default values in setting.go** + +In `web/service/setting.go`, add inside the `defaultValueMap` block (after line 129 `"trafficFlushInterval": "10",`): + +```go + // Backup settings + "backupEnabled": "false", + "backupFrequency": "daily", + "backupHour": "3", + "backupMaxCount": "10", +``` + +- [ ] **Step 3: Add backup group to settingGroups** + +In `web/service/setting.go`, add a new `"backup"` group to `settingGroups` (after the `"node"` group, line 248): + +```go + "backup": { + "enabled": "backupEnabled", + "frequency": "backupFrequency", + "hour": "backupHour", + "maxCount": "backupMaxCount", + }, +``` + +- [ ] **Step 4: Add getter/setter methods to SettingService** + +In `web/service/setting.go`, add after the existing setting methods (e.g., after line 1089 `GetLdapSyncCron`): + +```go +func (s *SettingService) GetBackupEnabled() (bool, error) { return s.getBool("backupEnabled") } +func (s *SettingService) GetBackupFrequency() (string, error) { return s.getString("backupFrequency") } +func (s *SettingService) GetBackupHour() (int, error) { return s.getInt("backupHour") } +func (s *SettingService) GetBackupMaxCount() (int, error) { return s.getInt("backupMaxCount") } +``` + +- [ ] **Step 5: Verify compilation** + +Run: `go build ./...` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add web/entity/entity.go web/service/setting.go +git commit -m "feat: add backup config fields to entity and setting service" +``` + +--- + +### Task 2: Create BackupService + +**Files:** +- Create: `web/service/backup.go` + +- [ ] **Step 1: Create `web/service/backup.go`** + +```go +package service + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/mhsanaei/3x-ui/v2/config" + "github.com/mhsanaei/3x-ui/v2/database" + "github.com/mhsanaei/3x-ui/v2/logger" +) + +const ( + backupDir = "/etc/x-ui/backups" + minDiskFreeMB = 100 + maxBackupDirSizeMB = 500 +) + +type BackupMeta struct { + DBType string `json:"dbType"` + Timestamp string `json:"timestamp"` + Version string `json:"version"` +} + +type BackupEntry struct { + Filename string `json:"filename"` + Timestamp string `json:"timestamp"` + Size int64 `json:"size"` +} + +// BackupService handles database backup, restore, and file management operations. +type BackupService struct { +} + +// ensureBackupDir creates the backup directory if it does not exist. +func ensureBackupDir() error { + if err := os.MkdirAll(backupDir, 0755); err != nil { + return fmt.Errorf("create backup directory: %w", err) + } + return nil +} + +// checkNodeRole verifies the current node is a master node. +func checkNodeRole() error { + nodeCfg := config.GetNodeConfigFromJSON() + if nodeCfg.Role == config.NodeRoleWorker { + return fmt.Errorf("backup and restore can only be performed on the master node") + } + return nil +} + +// CreateBackup creates an immediate backup of the current database. +func (s *BackupService) CreateBackup() (string, error) { + if err := checkNodeRole(); err != nil { + return "", err + } + if err := ensureBackupDir(); err != nil { + return "", err + } + + dbCfg := config.GetDBConfigFromJSON() + timestamp := time.Now().Format("2006-01-02-150405") + filename := fmt.Sprintf("backup-%s.tar.gz", timestamp) + filePath := filepath.Join(backupDir, filename) + + var dumpSQL string + var err error + + switch dbCfg.Type { + case "mariadb": + dumpSQL, err = dumpMariaDB(dbCfg) + case "sqlite", "": + dumpSQL, err = dumpSQLite(config.GetDBPath()) + default: + return "", fmt.Errorf("unsupported database type: %s", dbCfg.Type) + } + if err != nil { + return "", fmt.Errorf("database dump failed: %w", err) + } + + meta := BackupMeta{ + DBType: defaultDBType(dbCfg.Type), + Timestamp: time.Now().UTC().Format(time.RFC3339), + Version: config.GetVersion(), + } + + if err := createTarGz(filePath, meta, dumpSQL); err != nil { + return "", fmt.Errorf("create archive failed: %w", err) + } + + return filePath, nil +} + +// ListBackups returns all backup files sorted by time (newest first). +func (s *BackupService) ListBackups() ([]BackupEntry, error) { + entries, err := os.ReadDir(backupDir) + if err != nil { + if os.IsNotExist(err) { + return []BackupEntry{}, nil + } + return nil, fmt.Errorf("read backup directory: %w", err) + } + + var result []BackupEntry + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), "backup-") || !strings.HasSuffix(entry.Name(), ".tar.gz") { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + ts := extractTimestamp(entry.Name()) + result = append(result, BackupEntry{ + Filename: entry.Name(), + Timestamp: ts, + Size: info.Size(), + }) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].Timestamp > result[j].Timestamp + }) + + return result, nil +} + +// RestoreBackup restores the database from a backup file. +func (s *BackupService) RestoreBackup(filename string) error { + if err := checkNodeRole(); err != nil { + return err + } + + filePath := filepath.Join(backupDir, filename) + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return fmt.Errorf("backup file not found: %s", filePath) + } + + meta, dumpSQL, err := extractTarGz(filePath) + if err != nil { + return fmt.Errorf("invalid backup file: %w", err) + } + + dbCfg := config.GetDBConfigFromJSON() + currentDBType := defaultDBType(dbCfg.Type) + + if meta.DBType != currentDBType { + return fmt.Errorf("backup type (%s) does not match current database (%s)", meta.DBType, currentDBType) + } + + // Create safety backup before restoring + safetyFile, err := s.createSafetyBackup(dbCfg) + if err != nil { + logger.Warning("failed to create safety backup before restore:", err) + } else { + logger.Info("safety backup created:", safetyFile) + } + + if err := restoreDB(dbCfg, dumpSQL); err != nil { + return fmt.Errorf("restore failed: %w", err) + } + + return nil +} + +// DeleteBackup deletes a backup file. +func (s *BackupService) DeleteBackup(filename string) error { + filePath := filepath.Join(backupDir, filename) + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return fmt.Errorf("backup file not found: %s", filePath) + } + return os.Remove(filePath) +} + +// GetBackupPath returns the full path to a backup file. +func (s *BackupService) GetBackupPath(filename string) string { + return filepath.Join(backupDir, filename) +} + +// ApplyRetention applies retention policy, keeping maxCount newest backups. +func (s *BackupService) ApplyRetention(maxCount int) error { + entries, err := s.ListBackups() + if err != nil { + return err + } + + // Also count pre-restore safety backups for total size check, but exclude from retention deletion + if len(entries) <= maxCount { + return nil + } + + for i := maxCount; i < len(entries); i++ { + filePath := s.GetBackupPath(entries[i].Filename) + if err := os.Remove(filePath); err != nil { + logger.Warning("failed to delete old backup:", entries[i].Filename, err) + } else { + logger.Info("deleted old backup:", entries[i].Filename) + } + } + + return nil +} + +// CreateSnapshot creates a scheduled backup and applies retention. +func (s *BackupService) CreateSnapshot(settingService SettingService) error { + enabled, err := settingService.GetBackupEnabled() + if err != nil || !enabled { + return nil + } + + filePath, err := s.CreateBackup() + if err != nil { + return fmt.Errorf("snapshot backup failed: %w", err) + } + logger.Info("scheduled backup created:", filePath) + + maxCount := 10 + if mc, err := settingService.GetBackupMaxCount(); err == nil && mc > 0 { + maxCount = mc + } + if err := s.ApplyRetention(maxCount); err != nil { + logger.Warning("retention apply failed:", err) + } + + return nil +} + +// createSafetyBackup creates an emergency backup before restore. +func (s *BackupService) createSafetyBackup(dbCfg config.DBConfig) (string, error) { + timestamp := time.Now().Format("2006-01-02-150405") + filename := fmt.Sprintf("pre-restore-%s.tar.gz", timestamp) + filePath := filepath.Join(backupDir, filename) + + var dumpSQL string + var err error + switch dbCfg.Type { + case "mariadb": + dumpSQL, err = dumpMariaDB(dbCfg) + default: + dumpSQL, err = dumpSQLite(config.GetDBPath()) + } + if err != nil { + return "", err + } + + meta := BackupMeta{ + DBType: defaultDBType(dbCfg.Type), + Timestamp: time.Now().UTC().Format(time.RFC3339), + Version: config.GetVersion(), + } + + if err := createTarGz(filePath, meta, dumpSQL); err != nil { + return "", err + } + + return filePath, nil +} + +// defaultDBType normalizes empty string to "sqlite". +func defaultDBType(t string) string { + if t == "" { + return "sqlite" + } + return t +} + +// extractTimestamp extracts the timestamp string from backup filename. +func extractTimestamp(filename string) string { + // backup-2026-04-26-030000.tar.gz + name := strings.TrimPrefix(filename, "backup-") + name = strings.TrimSuffix(name, ".tar.gz") + return name +} + +// dumpMariaDB runs mysqldump and returns the SQL output. +func dumpMariaDB(dbCfg config.DBConfig) (string, error) { + args := []string{ + "--single-transaction", + "--routines", + "--triggers", + "--no-tablespaces", + fmt.Sprintf("-h%s", dbCfg.Host), + fmt.Sprintf("-P%s", dbCfg.Port), + } + if dbCfg.User != "" { + args = append(args, fmt.Sprintf("-u%s", dbCfg.User)) + } + if dbCfg.Password != "" { + args = append(args, fmt.Sprintf("-p%s", dbCfg.Password)) + } + args = append(args, dbCfg.Name) + + cmd := exec.Command("mysqldump", args...) + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("mysqldump: %w (output: %s)", err, out.String()) + } + return out.String(), nil +} + +// dumpSQLite runs sqlite3 .dump and returns the SQL output. +func dumpSQLite(dbPath string) (string, error) { + database.Checkpoint() + cmd := exec.Command("sqlite3", dbPath, ".dump") + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("sqlite3 dump: %w (output: %s)", err, out.String()) + } + return out.String(), nil +} + +// restoreDB restores the database from SQL dump. +func restoreDB(dbCfg config.DBConfig, dumpSQL string) error { + switch dbCfg.Type { + case "mariadb": + args := []string{ + fmt.Sprintf("-h%s", dbCfg.Host), + fmt.Sprintf("-P%s", dbCfg.Port), + } + if dbCfg.User != "" { + args = append(args, fmt.Sprintf("-u%s", dbCfg.User)) + } + if dbCfg.Password != "" { + args = append(args, fmt.Sprintf("-p%s", dbCfg.Password)) + } + args = append(args, dbCfg.Name) + + cmd := exec.Command("mysql", args...) + cmd.Stdin = strings.NewReader(dumpSQL) + var stderr strings.Builder + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("mysql restore: %w (stderr: %s)", err, stderr.String()) + } + default: + cmd := exec.Command("sqlite3", config.GetDBPath()) + cmd.Stdin = strings.NewReader(dumpSQL) + var stderr strings.Builder + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("sqlite3 restore: %w (stderr: %s)", err, stderr.String()) + } + } + return nil +} + +// createTarGz creates a tar.gz archive containing metadata.json and dump.sql. +func createTarGz(filePath string, meta BackupMeta, dumpSQL string) error { + f, err := os.Create(filePath) + if err != nil { + return err + } + defer f.Close() + + gw := gzip.NewWriter(f) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + // metadata.json + metaBytes, _ := json.MarshalIndent(meta, "", " ") + if err := tw.WriteHeader(&tar.Header{ + Name: "metadata.json", + Size: int64(len(metaBytes)), + Mode: 0644, + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + if _, err := tw.Write(metaBytes); err != nil { + return err + } + + // dump.sql + dumpBytes := []byte(dumpSQL) + if err := tw.WriteHeader(&tar.Header{ + Name: "dump.sql", + Size: int64(len(dumpBytes)), + Mode: 0644, + Typeflag: tar.TypeReg, + }); err != nil { + return err + } + if _, err := tw.Write(dumpBytes); err != nil { + return err + } + + return nil +} + +// extractTarGz reads a tar.gz archive and returns metadata and SQL dump. +func extractTarGz(filePath string) (*BackupMeta, string, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, "", err + } + defer f.Close() + + gr, err := gzip.NewReader(f) + if err != nil { + return nil, "", err + } + defer gr.Close() + + tr := tar.NewReader(gr) + + var meta BackupMeta + var dumpSQL string + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, "", err + } + + var buf strings.Builder + if _, err := io.Copy(&buf, tr); err != nil { + return nil, "", err + } + + switch hdr.Name { + case "metadata.json": + if err := json.Unmarshal([]byte(buf.String()), &meta); err != nil { + return nil, "", fmt.Errorf("invalid metadata.json: %w", err) + } + case "dump.sql": + dumpSQL = buf.String() + } + } + + if dumpSQL == "" { + return nil, "", fmt.Errorf("dump.sql not found in archive") + } + if meta.DBType == "" { + return nil, "", fmt.Errorf("metadata.json missing dbType") + } + + return &meta, dumpSQL, nil +} +``` + +- [ ] **Step 2: Verify compilation** + +Run: `go build ./...` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add web/service/backup.go +git commit -m "feat: add BackupService with dump, archive, restore logic" +``` + +--- + +### Task 3: Create BackupController + +**Files:** +- Create: `web/controller/backup.go` + +- [ ] **Step 1: Create `web/controller/backup.go`** + +```go +package controller + +import ( + "fmt" + "net/http" + "os" + + "github.com/gin-gonic/gin" + "github.com/mhsanaei/3x-ui/v2/web/service" +) + +// BackupController handles database backup and restore API endpoints. +type BackupController struct { + BaseController + backupService service.BackupService +} + +// initRouter registers backup API routes. +func (a *BackupController) initRouter(g *gin.RouterGroup) { + g.POST("/backup", a.createBackup) + g.POST("/restore/:filename", a.restoreBackup) + g.POST("/deleteBackup/:filename", a.deleteBackup) + g.GET("/listBackups", a.listBackups) + g.GET("/downloadBackup/:filename", a.downloadBackup) +} + +// createBackup creates an immediate manual backup. +func (a *BackupController) createBackup(c *gin.Context) { + filePath, err := a.backupService.CreateBackup() + if err != nil { + jsonMsg(c, I18nWeb(c, "pages.settings.backup.createError"), err) + return + } + jsonObj(c, filePath, nil) +} + +// restoreBackup restores the database from a backup file. +func (a *BackupController) restoreBackup(c *gin.Context) { + filename := c.Param("filename") + if !isValidFilename(filename) { + jsonMsg(c, "Invalid filename", fmt.Errorf("invalid filename")) + return + } + if err := a.backupService.RestoreBackup(filename); err != nil { + jsonMsg(c, I18nWeb(c, "pages.settings.backup.restoreError"), err) + return + } + jsonObj(c, "restore completed", nil) +} + +// deleteBackup deletes a backup file. +func (a *BackupController) deleteBackup(c *gin.Context) { + filename := c.Param("filename") + if !isValidFilename(filename) { + jsonMsg(c, "Invalid filename", fmt.Errorf("invalid filename")) + return + } + if err := a.backupService.DeleteBackup(filename); err != nil { + jsonMsg(c, "delete failed", err) + return + } + jsonObj(c, "deleted", nil) +} + +// listBackups lists all backup files. +func (a *BackupController) listBackups(c *gin.Context) { + entries, err := a.backupService.ListBackups() + if err != nil { + jsonMsg(c, "list backups failed", err) + return + } + jsonObj(c, entries, nil) +} + +// downloadBackup downloads a backup file. +func (a *BackupController) downloadBackup(c *gin.Context) { + filename := c.Param("filename") + if !isValidFilename(filename) { + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("invalid filename")) + return + } + filePath := a.backupService.GetBackupPath(filename) + data, err := os.ReadFile(filePath) + if err != nil { + jsonMsg(c, "read backup file failed", err) + return + } + c.Header("Content-Type", "application/gzip") + c.Header("Content-Disposition", "attachment; filename="+filename) + c.Writer.Write(data) +} +``` + +- [ ] **Step 2: Verify compilation** + +Run: `go build ./...` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add web/controller/backup.go +git commit -m "feat: add BackupController API endpoints" +``` + +--- + +### Task 4: Create BackupJob for scheduled snapshots + +**Files:** +- Create: `web/job/backup_job.go` + +- [ ] **Step 1: Create `web/job/backup_job.go`** + +```go +package job + +import ( + "time" + + "github.com/mhsanaei/3x-ui/v2/logger" + "github.com/mhsanaei/3x-ui/v2/web/service" +) + +// BackupJob handles scheduled database backups. +type BackupJob struct { + settingService service.SettingService + backupService service.BackupService + lastRun string // stores the frequency of the last run to detect change +} + +// NewBackupJob creates a new BackupJob instance. +func NewBackupJob() *BackupJob { + return &BackupJob{} +} + +// Run executes the scheduled backup if enabled and time matches the configured frequency. +func (j *BackupJob) Run() { + enabled, err := j.settingService.GetBackupEnabled() + if err != nil || !enabled { + return + } + + frequency, err := j.settingService.GetBackupFrequency() + if err != nil { + return + } + + if !j.shouldRun(frequency) { + return + } + + if err := j.backupService.CreateSnapshot(j.settingService); err != nil { + logger.Warning("scheduled backup failed:", err) + } +} + +// shouldRun checks if the backup should run based on the configured frequency. +func (j *BackupJob) shouldRun(frequency string) bool { + now := time.Now() + + switch frequency { + case "hourly": + return now.Minute() == 0 + case "every12h": + return (now.Hour() == 0 || now.Hour() == 12) && now.Minute() == 0 + case "daily": + hour, err := j.settingService.GetBackupHour() + if err != nil { + hour = 3 + } + return now.Hour() == hour && now.Minute() == 0 + case "weekly": + if now.Weekday() != time.Sunday { + return false + } + hour, err := j.settingService.GetBackupHour() + if err != nil { + hour = 3 + } + return now.Hour() == hour && now.Minute() == 0 + default: + return false + } +} +``` + +- [ ] **Step 2: Verify compilation** + +Run: `go build ./...` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add web/job/backup_job.go +git commit -m "feat: add BackupJob for scheduled snapshots" +``` + +--- + +### Task 5: Wire up routes and job scheduling + +**Files:** +- Modify: `web/controller/server.go:41-63` (add backup routes to initRouter) +- Modify: `web/web.go:360-413` (add backup job to cron) + +- [ ] **Step 1: Register backup routes in ServerController** + +In `web/controller/server.go`, inside `initRouter()` after line 62, add: + +```go + // Backup routes + backupCtrl := BackupController{} + backupCtrl.initRouter(g) +``` + +- [ ] **Step 2: Register backup job in web.go** + +In `web/web.go`, after `s.cron.AddJob("@daily", job.NewClearLogsJob())` (around line 365), add: + +```go + // Schedule database backup job (runs every minute, checks schedule internally) + s.cron.AddJob("@every 1m", job.NewBackupJob()) +``` + +- [ ] **Step 3: Verify compilation** + +Run: `go build ./...` +Expected: PASS + +- [ ] **Step 4: Run vet** + +Run: `go vet ./...` +Expected: no errors + +- [ ] **Step 5: Commit** + +```bash +git add web/controller/server.go web/web.go +git commit -m "feat: wire backup routes and scheduling job" +``` + +--- + +### Task 6: Create backup UI page + +**Files:** +- Create: `web/html/settings/backup.html` +- Modify: `web/html/settings.html:67-109` (add backup tab) + +- [ ] **Step 1: Read current settings.html tabs section** + +Read `web/html/settings.html:67-109` to understand the current tab structure before editing. + +- [ ] **Step 2: Create `web/html/settings/backup.html`** + +```html +
+ + + + + + + + + + + + + + Every Hour + Every 12 Hours + Every Day + Every Week + + + + + + + + + + + + + + + + + + + + + + Create Backup Now + + + + + + + Backup List + + + + + + + + + + +
+``` + +- [ ] **Step 3: Add backup tab to settings.html** + +In `web/html/settings.html`, after the clash tab (closing `` on line 109), add: + +```html + + + {{ template "settings/backup" . }} + +``` + +- [ ] **Step 4: Add backup Vue.js methods and data to settings.html script** + +In `web/html/settings.html`, in the `