diff --git a/.dockerignore b/.dockerignore index 7cfc7f8d..07544676 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ db cert pgdata *.db +*.dump diff --git a/.gitignore b/.gitignore index d343f43b..6ea14172 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ Thumbs.db x-ui.db x-ui.db-shm x-ui.db-wal +*.dump # Ignore Docker specific files docker-compose.override.yml diff --git a/database/dump_sqlite.go b/database/dump_sqlite.go new file mode 100644 index 00000000..8b71b48d --- /dev/null +++ b/database/dump_sqlite.go @@ -0,0 +1,218 @@ +package database + +import ( + "database/sql" + "fmt" + "os" + "strconv" + "strings" + "unicode/utf8" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// DumpSQLite writes a portable SQL text dump of the SQLite database at srcPath +// to outPath. The output mirrors the `sqlite3 .dump` format (schema + data + +// indexes wrapped in a transaction), so it can be rebuilt with RestoreSQLite or +// loaded by the sqlite3 CLI. The source database is opened read-only in effect +// and left untouched. +func DumpSQLite(srcPath, outPath string) error { + data, err := DumpSQLiteToBytes(srcPath) + if err != nil { + return err + } + return os.WriteFile(outPath, data, 0o644) +} + +// DumpSQLiteToBytes builds the same `sqlite3 .dump`-style SQL text as DumpSQLite +// but returns it in memory, which the panel uses to stream a migration download. +func DumpSQLiteToBytes(srcPath string) ([]byte, error) { + if _, err := os.Stat(srcPath); err != nil { + return nil, fmt.Errorf("source sqlite not found at %s: %w", srcPath, err) + } + + gdb, err := gorm.Open(sqlite.Open(srcPath), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return nil, err + } + sqlDB, err := gdb.DB() + if err != nil { + return nil, err + } + defer sqlDB.Close() + + var b strings.Builder + b.WriteString("PRAGMA foreign_keys=OFF;\n") + b.WriteString("BEGIN TRANSACTION;\n") + + // Tables in creation order, each followed by its data. + type object struct{ name, ddl string } + var tables []object + rows, err := sqlDB.Query(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL ORDER BY rowid`) + if err != nil { + return nil, err + } + for rows.Next() { + var o object + if err := rows.Scan(&o.name, &o.ddl); err != nil { + rows.Close() + return nil, err + } + tables = append(tables, o) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + + for _, t := range tables { + b.WriteString(t.ddl) + b.WriteString(";\n") + if err := dumpTableData(sqlDB, t.name, &b); err != nil { + return nil, err + } + } + + // AUTOINCREMENT bookkeeping, restored verbatim like the sqlite3 CLI does. + if sqliteTableExists(sqlDB, "sqlite_sequence") { + b.WriteString("DELETE FROM sqlite_sequence;\n") + if err := dumpTableData(sqlDB, "sqlite_sequence", &b); err != nil { + return nil, err + } + } + + // Indexes, triggers and views after the data is in place. + rows2, err := sqlDB.Query(`SELECT sql FROM sqlite_master WHERE type IN ('index','trigger','view') AND sql IS NOT NULL ORDER BY rowid`) + if err != nil { + return nil, err + } + for rows2.Next() { + var ddl string + if err := rows2.Scan(&ddl); err != nil { + rows2.Close() + return nil, err + } + b.WriteString(ddl) + b.WriteString(";\n") + } + if err := rows2.Err(); err != nil { + rows2.Close() + return nil, err + } + rows2.Close() + + b.WriteString("COMMIT;\n") + + return []byte(b.String()), nil +} + +// RestoreSQLite rebuilds a SQLite database at dstPath from a SQL text dump +// produced by DumpSQLite (or `sqlite3 .dump`). dstPath must not already exist so +// an existing database is never clobbered silently. +func RestoreSQLite(dumpPath, dstPath string) error { + script, err := os.ReadFile(dumpPath) + if err != nil { + return err + } + if _, err := os.Stat(dstPath); err == nil { + return fmt.Errorf("destination already exists: %s", dstPath) + } + + gdb, err := gorm.Open(sqlite.Open(dstPath), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return err + } + sqlDB, err := gdb.DB() + if err != nil { + return err + } + + // mattn/go-sqlite3 executes every statement in a multi-statement string. + if _, err := sqlDB.Exec(string(script)); err != nil { + sqlDB.Close() + os.Remove(dstPath) + return fmt.Errorf("restore failed: %w", err) + } + return sqlDB.Close() +} + +// dumpTableData appends one INSERT statement per row of table to b. +func dumpTableData(db *sql.DB, table string, b *strings.Builder) error { + rows, err := db.Query(`SELECT * FROM "` + table + `"`) + if err != nil { + return err + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return err + } + n := len(cols) + prefix := `INSERT INTO "` + table + `" VALUES(` + + for rows.Next() { + vals := make([]any, n) + ptrs := make([]any, n) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return err + } + b.WriteString(prefix) + for i, v := range vals { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(sqliteLiteral(v)) + } + b.WriteString(");\n") + } + return rows.Err() +} + +// sqliteLiteral renders a scanned column value as a SQLite SQL literal. +func sqliteLiteral(v any) string { + switch x := v.(type) { + case nil: + return "NULL" + case int64: + return strconv.FormatInt(x, 10) + case float64: + return strconv.FormatFloat(x, 'g', -1, 64) + case bool: + if x { + return "1" + } + return "0" + case string: + return quoteSQLiteText(x) + case []byte: + if utf8.Valid(x) { + return quoteSQLiteText(string(x)) + } + var sb strings.Builder + sb.WriteString("X'") + for _, c := range x { + fmt.Fprintf(&sb, "%02x", c) + } + sb.WriteByte('\'') + return sb.String() + default: + return quoteSQLiteText(fmt.Sprintf("%v", x)) + } +} + +func quoteSQLiteText(s string) string { + return "'" + strings.ReplaceAll(s, "'", "''") + "'" +} + +func sqliteTableExists(db *sql.DB, name string) bool { + var found string + err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&found) + return err == nil +} diff --git a/database/dump_sqlite_test.go b/database/dump_sqlite_test.go new file mode 100644 index 00000000..59508d2e --- /dev/null +++ b/database/dump_sqlite_test.go @@ -0,0 +1,137 @@ +package database + +import ( + "os" + "path/filepath" + "testing" + + "github.com/mhsanaei/3x-ui/v3/database/model" + "github.com/mhsanaei/3x-ui/v3/xray" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// TestCopyAllModelsIntoSQLite exercises the same AutoMigrate + copyTable +// machinery that ExportPostgresToSQLite relies on, but with a SQLite source so +// it needs no external database. The Postgres source path uses identical gorm +// reads (see MigrateData), so this validates the destination-side copy. +func TestCopyAllModelsIntoSQLite(t *testing.T) { + dir := t.TempDir() + srcPath := filepath.Join(dir, "src.db") + dstPath := filepath.Join(dir, "dst.db") + + src, err := gorm.Open(sqlite.Open(srcPath), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatalf("open src: %v", err) + } + defer closeGorm(src) + for _, m := range migrationModels() { + if err := src.AutoMigrate(m); err != nil { + t.Fatalf("automigrate src %T: %v", m, err) + } + } + + // Seed a few rows across parent/child tables and a composite-PK table. + if err := src.Create(&model.User{Username: "admin", Password: "x"}).Error; err != nil { + t.Fatalf("seed user: %v", err) + } + if err := src.Create(&model.Inbound{UserId: 1, Remark: "in", Port: 443, Protocol: "vless", Tag: "inbound-443"}).Error; err != nil { + t.Fatalf("seed inbound: %v", err) + } + if err := src.Create(&xray.ClientTraffic{InboundId: 1, Email: "a@b.c", Enable: true, Up: 10, Down: 20}).Error; err != nil { + t.Fatalf("seed traffic: %v", err) + } + + dst, err := gorm.Open(sqlite.Open(dstPath), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatalf("open dst: %v", err) + } + defer closeGorm(dst) + if err := copyAllModels(src, dst); err != nil { + t.Fatalf("copyAllModels: %v", err) + } + + for _, tc := range []struct { + model any + want int64 + }{ + {&model.User{}, 1}, + {&model.Inbound{}, 1}, + {&xray.ClientTraffic{}, 1}, + } { + var got int64 + if err := dst.Model(tc.model).Count(&got).Error; err != nil { + t.Fatalf("count %T: %v", tc.model, err) + } + if got != tc.want { + t.Errorf("%T: got %d rows, want %d", tc.model, got, tc.want) + } + } + + // Spot-check a copied value survived the round-trip. + var ct xray.ClientTraffic + if err := dst.Where("email = ?", "a@b.c").First(&ct).Error; err != nil { + t.Fatalf("read back traffic: %v", err) + } + if ct.Up != 10 || ct.Down != 20 || !ct.Enable { + t.Errorf("traffic mismatch: %+v", ct) + } +} + +// TestDumpAndRestoreSQLiteRoundTrip dumps a seeded SQLite db to .dump text and +// rebuilds it, asserting the row survives. +func TestDumpAndRestoreSQLiteRoundTrip(t *testing.T) { + dir := t.TempDir() + srcPath := filepath.Join(dir, "src.db") + dumpPath := filepath.Join(dir, "out.dump") + dstPath := filepath.Join(dir, "rebuilt.db") + + src, err := gorm.Open(sqlite.Open(srcPath), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatalf("open src: %v", err) + } + if err := src.AutoMigrate(&model.Setting{}); err != nil { + t.Fatalf("automigrate: %v", err) + } + if err := src.Create(&model.Setting{Key: "secret", Value: "o'brien \"quote\""}).Error; err != nil { + t.Fatalf("seed: %v", err) + } + if sqlDB, _ := src.DB(); sqlDB != nil { + sqlDB.Close() + } + + if err := DumpSQLite(srcPath, dumpPath); err != nil { + t.Fatalf("DumpSQLite: %v", err) + } + if fi, err := os.Stat(dumpPath); err != nil || fi.Size() == 0 { + t.Fatalf("dump missing/empty: %v", err) + } + if err := RestoreSQLite(dumpPath, dstPath); err != nil { + t.Fatalf("RestoreSQLite: %v", err) + } + + dst, err := gorm.Open(sqlite.Open(dstPath), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatalf("open dst: %v", err) + } + defer closeGorm(dst) + var s model.Setting + if err := dst.Where("key = ?", "secret").First(&s).Error; err != nil { + t.Fatalf("read back: %v", err) + } + if s.Value != "o'brien \"quote\"" { + t.Errorf("value mismatch after round-trip: %q", s.Value) + } +} + +// closeGorm closes the underlying *sql.DB so Windows can delete the temp file. +func closeGorm(db *gorm.DB) { + if db == nil { + return + } + if s, err := db.DB(); err == nil { + s.Close() + } +} diff --git a/database/migrate_data.go b/database/migrate_data.go index 49918c5b..d4e6cdec 100644 --- a/database/migrate_data.go +++ b/database/migrate_data.go @@ -86,6 +86,15 @@ func MigrateData(srcPath, dstDSN string) error { } } + // Empty the destination tables so the migration is idempotent: a fresh + // PostgreSQL DB already holds an auto-seeded admin (id=1) from any prior + // panel start, and a partially-failed earlier run leaves rows behind. Either + // way a plain INSERT with explicit ids would collide on users_pkey, so clear + // our tables (only) before copying. + if err := truncatePostgresTables(dst, migrationModels()); err != nil { + return fmt.Errorf("clear destination tables: %w", err) + } + totalRows := 0 for _, m := range migrationModels() { n, err := copyTable(src, dst, m) @@ -105,6 +114,62 @@ func MigrateData(srcPath, dstDSN string) error { return nil } +// ExportPostgresToSQLite copies every row from the PostgreSQL database described +// by srcDSN into a fresh SQLite file at dstPath. It is the reverse of +// MigrateData and is used to hand a PostgreSQL-backed panel a portable .db file. +// dstPath is created/overwritten; the PostgreSQL source is left untouched. +func ExportPostgresToSQLite(srcDSN, dstPath string) error { + if srcDSN == "" { + return errors.New("source DSN is required") + } + if err := os.MkdirAll(path.Dir(dstPath), 0755); err != nil { + return err + } + // Start from an empty file so AutoMigrate creates the canonical schema. + if err := os.Remove(dstPath); err != nil && !os.IsNotExist(err) { + return err + } + + src, err := gorm.Open(postgres.Open(srcDSN), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return fmt.Errorf("open postgres source: %w", err) + } + srcSQL, err := src.DB() + if err != nil { + return err + } + defer srcSQL.Close() + + // No WAL: keep all data in the main file so it is complete once closed. + dst, err := gorm.Open(sqlite.Open(dstPath+"?_busy_timeout=10000"), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return fmt.Errorf("open sqlite destination: %w", err) + } + dstSQL, err := dst.DB() + if err != nil { + return err + } + defer dstSQL.Close() + + return copyAllModels(src, dst) +} + +// copyAllModels (re)creates the schema on dst and copies every migrated table +// from src to dst in FK-safe order. src/dst may be any gorm backend. +func copyAllModels(src, dst *gorm.DB) error { + for _, m := range migrationModels() { + if err := dst.AutoMigrate(m); err != nil { + return fmt.Errorf("AutoMigrate %T: %w", m, err) + } + } + for _, m := range migrationModels() { + if _, err := copyTable(src, dst, m); err != nil { + return fmt.Errorf("copy %T: %w", m, err) + } + } + return nil +} + func copyTable(src, dst *gorm.DB, mdl any) (int, error) { const batchSize = 500 @@ -157,6 +222,26 @@ func copyTable(src, dst *gorm.DB, mdl any) (int, error) { return total, nil } +// truncatePostgresTables empties every migrated table on dst in a single +// statement, resetting identity sequences. CASCADE covers the inbound/client +// foreign keys regardless of insertion order. Only the panel's own tables are +// touched, never the rest of the schema. +func truncatePostgresTables(dst *gorm.DB, models []any) error { + tables := make([]string, 0, len(models)) + for _, m := range models { + stmt := &gorm.Statement{DB: dst} + if err := stmt.Parse(m); err != nil { + return err + } + tables = append(tables, `"`+stmt.Schema.Table+`"`) + } + if len(tables) == 0 { + return nil + } + log.Println("Clearing destination tables...") + return dst.Exec("TRUNCATE TABLE " + strings.Join(tables, ", ") + " RESTART IDENTITY CASCADE").Error +} + // resetPostgresSequences advances each migrated table's id sequence past MAX(id), // otherwise the next INSERT-without-id would clash with copied rows. func resetPostgresSequences(dst *gorm.DB) error { diff --git a/frontend/src/pages/api-docs/endpoints.ts b/frontend/src/pages/api-docs/endpoints.ts index 6e9c7e8e..2233e211 100644 --- a/frontend/src/pages/api-docs/endpoints.ts +++ b/frontend/src/pages/api-docs/endpoints.ts @@ -307,6 +307,11 @@ export const sections: readonly Section[] = [ path: '/panel/api/server/getDb', summary: 'Stream the SQLite database file as an attachment. Use as a manual backup.', }, + { + method: 'GET', + path: '/panel/api/server/getMigration', + summary: 'Stream a cross-engine migration file as an attachment: a .dump (SQL text) on SQLite, or a .db SQLite database built from the live data on PostgreSQL.', + }, { method: 'GET', path: '/panel/api/server/getNewUUID', diff --git a/frontend/src/pages/index/BackupModal.tsx b/frontend/src/pages/index/BackupModal.tsx index 3935b103..bf6eb294 100644 --- a/frontend/src/pages/index/BackupModal.tsx +++ b/frontend/src/pages/index/BackupModal.tsx @@ -25,6 +25,10 @@ export default function BackupModal({ open, basePath: _basePath, onClose, onBusy window.location.href = (window.X_UI_BASE_PATH || '') + 'panel/api/server/getDb'; } + function exportMigration() { + window.location.href = (window.X_UI_BASE_PATH || '') + 'panel/api/server/getMigration'; + } + function importDb() { const fileInput = document.createElement('input'); fileInput.type = 'file'; @@ -82,6 +86,16 @@ export default function BackupModal({ open, basePath: _basePath, onClose, onBusy