diff --git a/Dockerfile b/Dockerfile index 06ddc638..3383bc6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,10 @@ RUN chmod +x \ /usr/bin/x-ui ENV XUI_ENABLE_FAIL2BAN="true" +# Database backend: set XUI_DB_TYPE=postgres and XUI_DB_DSN=postgres://... to use PostgreSQL. +# Default (unset) is SQLite stored under /etc/x-ui. +ENV XUI_DB_TYPE="" +ENV XUI_DB_DSN="" EXPOSE 2053 VOLUME [ "/etc/x-ui" ] CMD [ "./x-ui" ] diff --git a/README.md b/README.md index c678ad92..310a9b72 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,38 @@ bash <(curl -Ls https://raw.githubusercontent.com/mhsanaei/3x-ui/master/install. For full documentation, please visit the [project Wiki](https://github.com/MHSanaei/3x-ui/wiki). +## Database Options + +3X-UI supports two backends, chosen during the install: + +- **SQLite** (default) — a single file at `/etc/x-ui/x-ui.db`. Zero setup, ideal for small/medium deployments. +- **PostgreSQL** — recommended for high client counts or multi-node setups. The installer can install PostgreSQL locally for you, or accept a DSN to an existing server. + +At runtime the backend is selected via env vars (the installer writes these to `/etc/default/x-ui` for you): + +``` +XUI_DB_TYPE=postgres +XUI_DB_DSN=postgres://xui:password@127.0.0.1:5432/xui?sslmode=disable +``` + +### Migrating an existing SQLite install to PostgreSQL + +```bash +x-ui migrate-db --dsn "postgres://xui:password@127.0.0.1:5432/xui?sslmode=disable" +# then set XUI_DB_TYPE and XUI_DB_DSN in /etc/default/x-ui and restart: +systemctl restart x-ui +``` + +The source SQLite file is left untouched; remove it manually once you have verified the new backend. + +### Docker + +The default `docker compose up -d` keeps using SQLite. To run with the bundled PostgreSQL service, uncomment the two `XUI_DB_*` env lines in `docker-compose.yml` and start with the profile: + +```bash +docker compose --profile postgres up -d +``` + ## A Special Thanks to - [alireza0](https://github.com/alireza0/) diff --git a/config/config.go b/config/config.go index e5b43a29..31c285d2 100644 --- a/config/config.go +++ b/config/config.go @@ -100,6 +100,22 @@ func GetDBPath() string { return fmt.Sprintf("%s/%s.db", GetDBFolderPath(), GetName()) } +// GetDBKind returns the configured database backend: "sqlite" (default) or "postgres". +func GetDBKind() string { + v := strings.ToLower(strings.TrimSpace(os.Getenv("XUI_DB_TYPE"))) + switch v { + case "postgres", "postgresql", "pg": + return "postgres" + default: + return "sqlite" + } +} + +// GetDBDSN returns the PostgreSQL DSN from XUI_DB_DSN. Empty for sqlite. +func GetDBDSN() string { + return strings.TrimSpace(os.Getenv("XUI_DB_DSN")) +} + // GetLogFolder returns the path to the log folder based on environment variables or platform defaults. func GetLogFolder() string { logFolderPath := os.Getenv("XUI_LOG_FOLDER") diff --git a/database/db.go b/database/db.go index 58556e80..785a4e96 100644 --- a/database/db.go +++ b/database/db.go @@ -1,5 +1,5 @@ // Package database provides database initialization, migration, and management utilities -// for the 3x-ui panel using GORM with SQLite. +// for the 3x-ui panel using GORM with SQLite or PostgreSQL. package database import ( @@ -19,6 +19,7 @@ import ( "github.com/mhsanaei/3x-ui/v3/util/crypto" "github.com/mhsanaei/3x-ui/v3/xray" + "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" @@ -26,6 +27,27 @@ import ( var db *gorm.DB +const ( + DialectSQLite = "sqlite" + DialectPostgres = "postgres" +) + +// IsPostgres reports whether the active connection is a PostgreSQL backend. +func IsPostgres() bool { + if db == nil { + return config.GetDBKind() == "postgres" + } + return db.Dialector.Name() == "postgres" +} + +// Dialect returns the active GORM dialect name, or "" if the DB is not open. +func Dialect() string { + if db == nil { + return "" + } + return db.Dialector.Name() +} + const ( defaultUsername = "admin" defaultPassword = "admin" @@ -65,20 +87,25 @@ func isIgnorableDuplicateColumnErr(err error, mdl any) bool { return false } errMsg := strings.ToLower(err.Error()) - const dupPrefix = "duplicate column name:" - if !strings.Contains(errMsg, dupPrefix) { - return false + // SQLite: "duplicate column name: foo" + // Postgres: `pq: column "foo" of relation "bar" already exists` / `sqlstate 42701` + const sqlitePrefix = "duplicate column name:" + if _, after, ok := strings.Cut(errMsg, sqlitePrefix); ok { + col := strings.TrimSpace(after) + col = strings.Trim(col, "`\"[]") + return col != "" && db != nil && db.Migrator().HasColumn(mdl, col) } - _, after, ok := strings.Cut(errMsg, dupPrefix) - if !ok { - return false + if strings.Contains(errMsg, "already exists") && strings.Contains(errMsg, "column ") { + // Best effort: extract the column name between the first pair of double quotes. + if _, after, ok := strings.Cut(errMsg, "column \""); ok { + rest := after + if e := strings.Index(rest, "\""); e > 0 { + col := rest[:e] + return col != "" && db != nil && db.Migrator().HasColumn(mdl, col) + } + } } - col := strings.TrimSpace(after) - col = strings.Trim(col, "`\"[]") - if col == "" { - return false - } - return db != nil && db.Migrator().HasColumn(mdl, col) + return false } // initUser creates a default admin user if the users table is empty. @@ -281,43 +308,56 @@ func isTableEmpty(tableName string) (bool, error) { } // InitDB sets up the database connection, migrates models, and runs seeders. +// When XUI_DB_TYPE=postgres, dbPath is ignored and XUI_DB_DSN is used instead. func InitDB(dbPath string) error { - dir := path.Dir(dbPath) - err := os.MkdirAll(dir, 0755) - if err != nil { - return err - } - var gormLogger logger.Interface - if config.IsDebug() { gormLogger = logger.Default } else { gormLogger = logger.Discard } + c := &gorm.Config{Logger: gormLogger} - c := &gorm.Config{ - Logger: gormLogger, - } - dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=10000&_synchronous=NORMAL&_txlock=immediate" - db, err = gorm.Open(sqlite.Open(dsn), c) - if err != nil { - return err + var err error + switch config.GetDBKind() { + case "postgres": + dsn := config.GetDBDSN() + if dsn == "" { + return errors.New("XUI_DB_TYPE=postgres but XUI_DB_DSN is empty") + } + db, err = gorm.Open(postgres.Open(dsn), c) + if err != nil { + return err + } + default: + dir := path.Dir(dbPath) + if err = os.MkdirAll(dir, 0755); err != nil { + return err + } + dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=10000&_synchronous=NORMAL&_txlock=immediate" + db, err = gorm.Open(sqlite.Open(dsn), c) + if err != nil { + return err + } + sqlDB, err := db.DB() + if err != nil { + return err + } + if _, err := sqlDB.Exec("PRAGMA journal_mode=WAL"); err != nil { + return err + } + if _, err := sqlDB.Exec("PRAGMA busy_timeout=10000"); err != nil { + return err + } + if _, err := sqlDB.Exec("PRAGMA synchronous=NORMAL"); err != nil { + return err + } } sqlDB, err := db.DB() if err != nil { return err } - if _, err := sqlDB.Exec("PRAGMA journal_mode=WAL"); err != nil { - return err - } - if _, err := sqlDB.Exec("PRAGMA busy_timeout=10000"); err != nil { - return err - } - if _, err := sqlDB.Exec("PRAGMA synchronous=NORMAL"); err != nil { - return err - } sqlDB.SetMaxOpenConns(8) sqlDB.SetMaxIdleConns(4) sqlDB.SetConnMaxLifetime(time.Hour) @@ -370,13 +410,12 @@ func IsSQLiteDB(file io.ReaderAt) (bool, error) { } // Checkpoint performs a WAL checkpoint on the SQLite database to ensure data consistency. +// No-op on PostgreSQL (WAL there is managed by the server). func Checkpoint() error { - // Update WAL - err := db.Exec("PRAGMA wal_checkpoint;").Error - if err != nil { - return err + if IsPostgres() { + return nil } - return nil + return db.Exec("PRAGMA wal_checkpoint;").Error } // ValidateSQLiteDB opens the provided sqlite DB path with a throw-away connection diff --git a/database/dialect.go b/database/dialect.go new file mode 100644 index 00000000..b33a1dbe --- /dev/null +++ b/database/dialect.go @@ -0,0 +1,26 @@ +package database + +import "fmt" + +// JSONClientsFromInbound returns the FROM clause that yields one row per element +// of inbounds.settings -> clients, with a column named `client.value` whose text +// fields can be read with JSONFieldText("client.value", ""). +func JSONClientsFromInbound() string { + if IsPostgres() { + return "FROM inbounds, jsonb_array_elements(inbounds.settings::jsonb -> 'clients') AS client(value)" + } + return "FROM inbounds, JSON_EACH(JSON_EXTRACT(inbounds.settings, '$.clients')) AS client" +} + +// JSONFieldText returns a SQL expression that extracts the textual value of +// from a JSON expression. On both backends the result is the raw (unquoted) string, +// so callers do NOT need to trim surrounding quotes. +func JSONFieldText(expr, key string) string { + if IsPostgres() { + return fmt.Sprintf("(%s ->> '%s')", expr, key) + } + // SQLite's JSON_EXTRACT on a text value returns the JSON-encoded form + // (with surrounding quotes). Wrap it in json_extract(json_quote(...)) trick + // is fragile; simpler: unwrap quotes with TRIM(BOTH '"'). + return fmt.Sprintf("TRIM(JSON_EXTRACT(%s, '$.%s'), '\"')", expr, key) +} diff --git a/database/migrate_data.go b/database/migrate_data.go new file mode 100644 index 00000000..afa6f8ed --- /dev/null +++ b/database/migrate_data.go @@ -0,0 +1,143 @@ +package database + +import ( + "errors" + "fmt" + "log" + "os" + "path" + "reflect" + "time" + + "github.com/mhsanaei/3x-ui/v3/database/model" + "github.com/mhsanaei/3x-ui/v3/xray" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// migrationModels is the FK-aware order in which tables are created and copied. +// Parents come before their children so foreign-key constraints stay satisfied +// even when checks are not explicitly disabled. +func migrationModels() []any { + return []any{ + &model.User{}, + &model.Setting{}, + &model.HistoryOfSeeders{}, + &model.CustomGeoResource{}, + &model.Node{}, + &model.ApiToken{}, + &model.Inbound{}, + &xray.ClientTraffic{}, + &model.OutboundTraffics{}, + &model.InboundClientIps{}, + &model.ClientRecord{}, + &model.ClientInbound{}, + &model.InboundFallbackChild{}, + } +} + +// MigrateData copies every row from the configured SQLite file at srcPath into +// a fresh PostgreSQL database described by dstDSN. The destination tables are +// (re)created with AutoMigrate before the copy. Source data is left untouched. +func MigrateData(srcPath, dstDSN string) error { + if _, err := os.Stat(srcPath); err != nil { + return fmt.Errorf("source sqlite not found at %s: %w", srcPath, err) + } + if dstDSN == "" { + return errors.New("destination DSN is required") + } + + if err := os.MkdirAll(path.Dir(srcPath), 0755); err != nil { + return err + } + + srcDSN := srcPath + "?_journal_mode=WAL&_busy_timeout=10000" + src, err := gorm.Open(sqlite.Open(srcDSN), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return fmt.Errorf("open sqlite source: %w", err) + } + srcSQL, err := src.DB() + if err != nil { + return err + } + defer srcSQL.Close() + + dst, err := gorm.Open(postgres.Open(dstDSN), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return fmt.Errorf("open postgres destination: %w", err) + } + dstSQL, err := dst.DB() + if err != nil { + return err + } + defer dstSQL.Close() + dstSQL.SetConnMaxLifetime(time.Hour) + + log.Println("Creating destination schema...") + for _, m := range migrationModels() { + if err := dst.AutoMigrate(m); err != nil { + return fmt.Errorf("AutoMigrate %T: %w", m, err) + } + } + + totalRows := 0 + for _, m := range migrationModels() { + n, err := copyTable(src, dst, m) + if err != nil { + return fmt.Errorf("copy %T: %w", m, err) + } + totalRows += n + log.Printf(" %-32s %d rows", reflect.TypeOf(m).Elem().Name(), n) + } + + if err := resetPostgresSequences(dst); err != nil { + log.Printf("warning: failed to reset some postgres sequences: %v", err) + } + + log.Printf("Migration complete: %d rows across %d tables.", totalRows, len(migrationModels())) + log.Println("Set XUI_DB_TYPE=postgres and XUI_DB_DSN=... in /etc/default/x-ui, then restart x-ui.") + return nil +} + +// copyTable streams every row of `mdl` from src to dst in batches. +func copyTable(src, dst *gorm.DB, mdl any) (int, error) { + sliceType := reflect.SliceOf(reflect.PointerTo(reflect.TypeOf(mdl).Elem())) + batchPtr := reflect.New(sliceType) + batchPtr.Elem().Set(reflect.MakeSlice(sliceType, 0, 0)) + + total := 0 + err := src.Model(mdl).FindInBatches(batchPtr.Interface(), 500, func(tx *gorm.DB, _ int) error { + batch := batchPtr.Elem() + if batch.Len() == 0 { + return nil + } + if err := dst.CreateInBatches(batchPtr.Interface(), 200).Error; err != nil { + return err + } + total += batch.Len() + return nil + }).Error + return total, err +} + +// resetPostgresSequences advances each table's id sequence past MAX(id), +// otherwise the next INSERT-without-id would clash with copied rows. +func resetPostgresSequences(dst *gorm.DB) error { + tables := []string{ + "users", "inbounds", "outbound_traffics", "settings", "inbound_client_ips", + "client_traffics", "history_of_seeders", "custom_geo_resources", "nodes", + "api_tokens", "client_records", "client_inbounds", "inbound_fallback_children", + } + for _, t := range tables { + // setval is a no-op if the table or its id sequence doesn't exist; we ignore errors per-table. + _ = dst.Exec(fmt.Sprintf( + `SELECT setval(pg_get_serial_sequence('%s','id'), COALESCE((SELECT MAX(id) FROM "%s"), 1), true) + WHERE pg_get_serial_sequence('%s','id') IS NOT NULL`, + t, t, t, + )).Error + } + return nil +} diff --git a/docker-compose.yml b/docker-compose.yml index d05fc4b8..ddb9493a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,24 @@ services: environment: XRAY_VMESS_AEAD_FORCED: "false" XUI_ENABLE_FAIL2BAN: "true" + # To use PostgreSQL instead of the default SQLite, run: + # docker compose --profile postgres up -d + # and uncomment the two lines below. + # XUI_DB_TYPE: "postgres" + # XUI_DB_DSN: "postgres://xui:xui@postgres:5432/xui?sslmode=disable" tty: true ports: - "2053:2053" + restart: unless-stopped + + postgres: + image: postgres:16-alpine + container_name: 3xui_postgres + profiles: ["postgres"] + environment: + POSTGRES_USER: xui + POSTGRES_PASSWORD: xui + POSTGRES_DB: xui + volumes: + - $PWD/pgdata/:/var/lib/postgresql/data restart: unless-stopped \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index af331712..af20a3a0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -471,61 +471,61 @@ } }, "node_modules/@intlify/core-base": { - "version": "11.4.2", - "resolved": "https://registry.npmjs.org/@intlify/core-base/-/core-base-11.4.2.tgz", - "integrity": "sha512-7fpuCcVmeLv2T9qHsARqGvh8xt+sV2fH+Q+gMHFwB/rPXzo85DpbJFKn7dBH1L5p0c2cSh2DW+2h/64EKrISmA==", + "version": "11.4.4", + "resolved": "https://registry.npmjs.org/@intlify/core-base/-/core-base-11.4.4.tgz", + "integrity": "sha512-w/vItlylrAmhebkIbVl5YY8XMCtj8Mb2g70ttxktMYuf5AuRahgEHL2iLgLIsZBIbTSgs4hkUo7ucCL0uTJvOg==", "license": "MIT", "dependencies": { - "@intlify/devtools-types": "11.4.2", - "@intlify/message-compiler": "11.4.2", - "@intlify/shared": "11.4.2" + "@intlify/devtools-types": "11.4.4", + "@intlify/message-compiler": "11.4.4", + "@intlify/shared": "11.4.4" }, "engines": { - "node": ">= 16" + "node": ">= 22" }, "funding": { "url": "https://github.com/sponsors/kazupon" } }, "node_modules/@intlify/devtools-types": { - "version": "11.4.2", - "resolved": "https://registry.npmjs.org/@intlify/devtools-types/-/devtools-types-11.4.2.tgz", - "integrity": "sha512-3u8EN1kB6EMSi96KXs5k7a8y2X2g4+h3X6iwVZU47cP4n+mTuq//WMjG588BzSp/2XQ/dTXo2BLUXX+XS+PNfA==", + "version": "11.4.4", + "resolved": "https://registry.npmjs.org/@intlify/devtools-types/-/devtools-types-11.4.4.tgz", + "integrity": "sha512-PcBLmGmDQsTSVV911P8upzpcLJO1CNVYi/IH6bGnLR2nA+0L963+kXN1ZrisTEnbtw2ewN6HMMSldqzjronA0Q==", "license": "MIT", "dependencies": { - "@intlify/core-base": "11.4.2", - "@intlify/shared": "11.4.2" + "@intlify/core-base": "11.4.4", + "@intlify/shared": "11.4.4" }, "engines": { - "node": ">= 16" + "node": ">= 22" }, "funding": { "url": "https://github.com/sponsors/kazupon" } }, "node_modules/@intlify/message-compiler": { - "version": "11.4.2", - "resolved": "https://registry.npmjs.org/@intlify/message-compiler/-/message-compiler-11.4.2.tgz", - "integrity": "sha512-a6CDSGSMTGrg0BjD97x8TBYPf7qQMDlZipJ6UDfv/pd4OIym8TMlHu3MsH0bTNnRdAG2D6EFEykIgiQPqvtTkA==", + "version": "11.4.4", + "resolved": "https://registry.npmjs.org/@intlify/message-compiler/-/message-compiler-11.4.4.tgz", + "integrity": "sha512-vn0OAV9pYkJlPPmgnsSm5eAG3mL0+9C/oaded2JY9jmxBbhmUXT3TcAUY8WRgLY9Hte7lkUJKpXrVlYjMXBD2w==", "license": "MIT", "dependencies": { - "@intlify/shared": "11.4.2", + "@intlify/shared": "11.4.4", "source-map-js": "^1.0.2" }, "engines": { - "node": ">= 16" + "node": ">= 22" }, "funding": { "url": "https://github.com/sponsors/kazupon" } }, "node_modules/@intlify/shared": { - "version": "11.4.2", - "resolved": "https://registry.npmjs.org/@intlify/shared/-/shared-11.4.2.tgz", - "integrity": "sha512-NzpHbguRCsOHDwxmlBa9qu/imc+/QWgsYUaK6FZeNC0wK8QfAbhqrktEp/haVzxU1aikH8IX4ytD+mfFEMi/9A==", + "version": "11.4.4", + "resolved": "https://registry.npmjs.org/@intlify/shared/-/shared-11.4.4.tgz", + "integrity": "sha512-QRUCHqda1U6aR14FR0vvXD4+4gj6+fm0AhAozvSuRCw0fCvrmCugWpgiR4xH2NI6s8am6N9p5OhirplsX8ZS3g==", "license": "MIT", "engines": { - "node": ">= 16" + "node": ">= 22" }, "funding": { "url": "https://github.com/sponsors/kazupon" @@ -3080,18 +3080,18 @@ } }, "node_modules/vue-i18n": { - "version": "11.4.2", - "resolved": "https://registry.npmjs.org/vue-i18n/-/vue-i18n-11.4.2.tgz", - "integrity": "sha512-sADDeKXqAGsPX6tK3t3y2ZiMpbVWN12tG+MhTiJ06rVoh58eGtM4wFyw3uWGbVkXByVp9Ne/AP+nSSzI+J9OAQ==", + "version": "11.4.4", + "resolved": "https://registry.npmjs.org/vue-i18n/-/vue-i18n-11.4.4.tgz", + "integrity": "sha512-gIbXVSFQV4jcSJxfwdZ5zSZmZ+12CnX0K3vBkRSd6Zn+HSzCp+QwUgPwpD/uN0oKNKI9RzlUXPKVedEuMgNG0A==", "license": "MIT", "dependencies": { - "@intlify/core-base": "11.4.2", - "@intlify/devtools-types": "11.4.2", - "@intlify/shared": "11.4.2", + "@intlify/core-base": "11.4.4", + "@intlify/devtools-types": "11.4.4", + "@intlify/shared": "11.4.4", "@vue/devtools-api": "^6.5.0" }, "engines": { - "node": ">= 16" + "node": ">= 22" }, "funding": { "url": "https://github.com/sponsors/kazupon" diff --git a/frontend/src/pages/inbounds/InboundList.vue b/frontend/src/pages/inbounds/InboundList.vue index cfb559b0..68687e6d 100644 --- a/frontend/src/pages/inbounds/InboundList.vue +++ b/frontend/src/pages/inbounds/InboundList.vue @@ -18,7 +18,6 @@ import { } from '@ant-design/icons-vue'; import { HttpUtil, SizeFormatter, IntlUtil, ColorUtils } from '@/utils'; -import { DBInbound } from '@/models/dbinbound.js'; import InfinityIcon from '@/components/InfinityIcon.vue'; import { useDatepicker } from '@/composables/useDatepicker.js'; diff --git a/go.mod b/go.mod index 6eb1e5c7..886634ae 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( golang.org/x/text v0.37.0 google.golang.org/grpc v1.81.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 ) @@ -53,6 +54,10 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.4.0 // indirect github.com/grbit/go-json v0.11.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 7d03ead2..6bf86420 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,14 @@ github.com/grbit/go-json v0.11.0 h1:bAbyMdYrYl/OjYsSqLH99N2DyQ291mHy726Mx+sYrnc= github.com/grbit/go-json v0.11.0/go.mod h1:IYpHsdybQ386+6g3VE6AXQ3uTGa5mquBme5/ZWmtzek= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -169,6 +177,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -272,6 +281,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= diff --git a/main.go b/main.go index c5ce40b9..f080d573 100644 --- a/main.go +++ b/main.go @@ -439,6 +439,12 @@ func main() { runCmd := flag.NewFlagSet("run", flag.ExitOnError) + migrateDbCmd := flag.NewFlagSet("migrate-db", flag.ExitOnError) + var migrateDsn string + var migrateSrc string + migrateDbCmd.StringVar(&migrateDsn, "dsn", "", "Destination PostgreSQL DSN (postgres://user:pass@host:port/db?sslmode=disable)") + migrateDbCmd.StringVar(&migrateSrc, "src", "", "Source SQLite file (defaults to the configured x-ui.db)") + settingCmd := flag.NewFlagSet("setting", flag.ExitOnError) var port int var username string @@ -482,6 +488,7 @@ func main() { fmt.Println("Commands:") fmt.Println(" run run web panel") fmt.Println(" migrate migrate form other/old x-ui") + fmt.Println(" migrate-db copy data from the SQLite file into a PostgreSQL database") fmt.Println(" setting set settings") } @@ -501,6 +508,23 @@ func main() { runWebServer() case "migrate": migrateDb() + case "migrate-db": + if err := migrateDbCmd.Parse(os.Args[2:]); err != nil { + fmt.Println(err) + return + } + src := migrateSrc + if src == "" { + src = config.GetDBPath() + } + if migrateDsn == "" { + fmt.Println("--dsn is required: postgres://user:pass@host:port/dbname?sslmode=disable") + return + } + if err := database.MigrateData(src, migrateDsn); err != nil { + fmt.Println("migration failed:", err) + os.Exit(1) + } case "setting": err := settingCmd.Parse(os.Args[2:]) if err != nil { diff --git a/web/job/node_traffic_sync_job.go b/web/job/node_traffic_sync_job.go index 542fe969..63de5018 100644 --- a/web/job/node_traffic_sync_job.go +++ b/web/job/node_traffic_sync_job.go @@ -18,12 +18,12 @@ const ( ) type NodeTrafficSyncJob struct { - nodeService service.NodeService - inboundService service.InboundService - settingService service.SettingService - xrayService service.XrayService - running sync.Mutex - structural atomicBool + nodeService service.NodeService + inboundService service.InboundService + settingService service.SettingService + xrayService service.XrayService + running sync.Mutex + structural atomicBool } type atomicBool struct { diff --git a/web/service/inbound.go b/web/service/inbound.go index 17e2d191..15ff3af0 100644 --- a/web/service/inbound.go +++ b/web/service/inbound.go @@ -242,12 +242,12 @@ func (s *InboundService) GetClients(inbound *model.Inbound) ([]model.Client, err func (s *InboundService) getAllEmails() ([]string, error) { db := database.GetDB() var emails []string - err := db.Raw(` - SELECT DISTINCT JSON_EXTRACT(client.value, '$.email') - FROM inbounds, - JSON_EACH(JSON_EXTRACT(inbounds.settings, '$.clients')) AS client - `).Scan(&emails).Error - if err != nil { + query := fmt.Sprintf( + "SELECT DISTINCT %s %s", + database.JSONFieldText("client.value", "email"), + database.JSONClientsFromInbound(), + ) + if err := db.Raw(query).Scan(&emails).Error; err != nil { return nil, err } return emails, nil @@ -261,22 +261,22 @@ func (s *InboundService) getAllEmailSubIDs() (map[string]string, error) { Email string SubID string } - err := db.Raw(` - SELECT JSON_EXTRACT(client.value, '$.email') AS email, - JSON_EXTRACT(client.value, '$.subId') AS sub_id - FROM inbounds, - JSON_EACH(JSON_EXTRACT(inbounds.settings, '$.clients')) AS client - `).Scan(&rows).Error - if err != nil { + query := fmt.Sprintf( + "SELECT %s AS email, %s AS sub_id %s", + database.JSONFieldText("client.value", "email"), + database.JSONFieldText("client.value", "subId"), + database.JSONClientsFromInbound(), + ) + if err := db.Raw(query).Scan(&rows).Error; err != nil { return nil, err } result := make(map[string]string, len(rows)) for _, r := range rows { - email := strings.ToLower(strings.Trim(r.Email, "\"")) + email := strings.ToLower(r.Email) if email == "" { continue } - subID := strings.Trim(r.SubID, "\"") + subID := r.SubID if existing, ok := result[email]; ok { if existing != subID { result[email] = "" @@ -296,14 +296,12 @@ func (s *InboundService) emailUsedByOtherInbounds(email string, exceptInboundId } db := database.GetDB() var count int64 - err := db.Raw(` - SELECT COUNT(*) - FROM inbounds, - JSON_EACH(JSON_EXTRACT(inbounds.settings, '$.clients')) AS client - WHERE inbounds.id != ? - AND LOWER(JSON_EXTRACT(client.value, '$.email')) = LOWER(?) - `, exceptInboundId, email).Scan(&count).Error - if err != nil { + query := fmt.Sprintf( + "SELECT COUNT(*) %s WHERE inbounds.id != ? AND LOWER(%s) = LOWER(?)", + database.JSONClientsFromInbound(), + database.JSONFieldText("client.value", "email"), + ) + if err := db.Raw(query, exceptInboundId, email).Scan(&count).Error; err != nil { return false, err } return count > 0, nil @@ -2043,14 +2041,12 @@ func (s *InboundService) GetClientReverseTags() (string, error) { func (s *InboundService) MigrationRemoveOrphanedTraffics() { db := database.GetDB() - db.Exec(` - DELETE FROM client_traffics - WHERE email NOT IN ( - SELECT JSON_EXTRACT(client.value, '$.email') - FROM inbounds, - JSON_EACH(JSON_EXTRACT(inbounds.settings, '$.clients')) AS client - ) - `) + query := fmt.Sprintf( + "DELETE FROM client_traffics WHERE email NOT IN (SELECT %s %s)", + database.JSONFieldText("client.value", "email"), + database.JSONClientsFromInbound(), + ) + db.Exec(query) } // AddClientStat inserts a per-client accounting row, no-op on email @@ -2390,12 +2386,14 @@ func (s *InboundService) DelDepletedClients(id int) (err error) { emails = append(emails, e) } var stillReferenced []string - if err = tx.Raw(` - SELECT DISTINCT LOWER(JSON_EXTRACT(client.value, '$.email')) - FROM inbounds, - JSON_EACH(JSON_EXTRACT(inbounds.settings, '$.clients')) AS client - WHERE LOWER(JSON_EXTRACT(client.value, '$.email')) IN ? - `, emails).Scan(&stillReferenced).Error; err != nil { + emailExpr := database.JSONFieldText("client.value", "email") + stillQuery := fmt.Sprintf( + "SELECT DISTINCT LOWER(%s) %s WHERE LOWER(%s) IN ?", + emailExpr, + database.JSONClientsFromInbound(), + emailExpr, + ) + if err = tx.Raw(stillQuery, emails).Scan(&stillReferenced).Error; err != nil { return err } stillSet := make(map[string]struct{}, len(stillReferenced)) @@ -2756,8 +2754,10 @@ func (s *InboundService) MigrationRequirements() { defer func() { if err == nil { tx.Commit() - if dbErr := db.Exec(`VACUUM "main"`).Error; dbErr != nil { - logger.Warningf("VACUUM failed: %v", dbErr) + if !database.IsPostgres() { + if dbErr := db.Exec(`VACUUM "main"`).Error; dbErr != nil { + logger.Warningf("VACUUM failed: %v", dbErr) + } } } else { tx.Rollback() diff --git a/web/translation/en-US.json b/web/translation/en-US.json index 6d440694..b20e209e 100644 --- a/web/translation/en-US.json +++ b/web/translation/en-US.json @@ -114,7 +114,7 @@ "emptyUsername": "Username is required", "emptyPassword": "Password is required", "wrongUsernameOrPassword": "Invalid username or password or two-factor code.", - "successLogin": " You have successfully logged into your account." + "successLogin": "You have successfully logged into your account." } }, "index": { @@ -725,7 +725,7 @@ "generalConfigs": "General", "generalConfigsDesc": "These options will determine general adjustments.", "logConfigs": "Log", - "logConfigsDesc": "Logs may affect your server's efficiency. It is recommended to enable it wisely only in case of your needs", + "logConfigsDesc": "Logs may affect your server's efficiency. It is recommended to enable them wisely only when needed.", "blockConfigsDesc": "These options will block traffic based on specific requested protocols and websites.", "basicRouting": "Basic Routing", "blockConnectionsConfigsDesc": "These options will block traffic based on the specific requested country.", @@ -874,7 +874,7 @@ "expectIPs": "Expect IPs", "unexpectIPs": "Unexpected IPs", "useSystemHosts": "Use System Hosts", - "useSystemHostsDesc": "Use the hosts file from an installed system", + "useSystemHostsDesc": "Use the operating system's hosts file", "serveStale": "Serve Stale", "serveStaleDesc": "Return expired cached results while refreshing in the background", "serveExpiredTTL": "Serve Expired TTL",