On PostgreSQL the logs table is a RANGE-partitioned table on created_at owned by pg_partman; GORM AutoMigrate would alter its composite primary key (id, created_at) and create redundant per-partition indexes, breaking startup. Extract ensureLogTable() to skip AutoMigrate on PostgreSQL (existence check only) while keeping the original behavior on SQLite/MySQL. The dedicated LOG_SQL_DSN PostgreSQL branch follows the same rule. Ship the matching postgres-partman image (Dockerfile + 01-partition.sql) that creates and maintains the partitioned table via pg_partman + pg_cron. Co-Authored-By: ZCode <noreply@anthropic.com>master
| @@ -0,0 +1,39 @@ | |||||
| # Custom PostgreSQL 18.4 image with pg_partman + pg_cron extensions. | |||||
| # | |||||
| # The logs table of new-api is a RANGE-partitioned table on created_at, created | |||||
| # and maintained entirely on the database side by the pg_partman extension. | |||||
| # pg_cron drives periodic maintenance (premake future weekly partitions). | |||||
| # | |||||
| # Both extensions are installed via Debian packages. pg_cron must be loaded via | |||||
| # shared_preload_libraries, so we append it to postgresql.conf.sample: initdb | |||||
| # generates PGDATA/postgresql.conf from this sample, meaning both the temporary | |||||
| # server (during docker-entrypoint-initdb.d) and the real server load pg_cron, | |||||
| # allowing CREATE EXTENSION pg_cron to succeed at init time. | |||||
| # | |||||
| # Build/push: | |||||
| # docker build -t registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 . | |||||
| # docker push registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 | |||||
| FROM postgres:18 | |||||
| # Use Aliyun Debian mirror for faster/reliable apt downloads from within China. | |||||
| # postgres:18 is based on Debian trixie which uses the deb822-format | |||||
| # /etc/apt/sources.list.d/debian.sources file instead of sources.list. | |||||
| RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' \ | |||||
| /etc/apt/sources.list.d/debian.sources | |||||
| RUN apt-get update \ | |||||
| && apt-get install -y --no-install-recommends \ | |||||
| postgresql-18-partman \ | |||||
| postgresql-18-cron \ | |||||
| && rm -rf /var/lib/apt/lists/* | |||||
| # Preload pg_cron so it is available during first-run init scripts. | |||||
| # pg_partman is a plain SQL extension and does not need preloading. | |||||
| RUN echo "shared_preload_libraries = 'pg_cron'" >> /usr/share/postgresql/18/postgresql.conf.sample | |||||
| RUN echo "cron.database_name = 'new-api'" >> /usr/share/postgresql/18/postgresql.conf.sample | |||||
| # First-run init: create extensions, the partitioned logs parent table, register | |||||
| # it with pg_partman, build core indexes, and schedule maintenance via pg_cron. | |||||
| # Runs against the database named by POSTGRES_DB (new-api). | |||||
| COPY docker-entrypoint-initdb.d/01-partition.sql /docker-entrypoint-initdb.d/01-partition.sql | |||||
| @@ -0,0 +1,102 @@ | |||||
| -- 01-partition.sql | |||||
| -- First-run initialization for new-api's partitioned logs table. | |||||
| -- Executed by the postgres official entrypoint against the database named by | |||||
| -- POSTGRES_DB (new-api), as a superuser. | |||||
| -- | |||||
| -- This script is the SINGLE source of truth for the logs partitioning setup. | |||||
| -- The new-api Go application does NOT create or maintain partitions; it only | |||||
| -- skips GORM AutoMigrate for the logs table on PostgreSQL (see | |||||
| -- model/main.go::ensureLogTable). pg_partman owns partition lifecycle, pg_cron | |||||
| -- drives periodic maintenance. | |||||
| -- 1. Extensions -------------------------------------------------------------- | |||||
| -- Install pg_partman into a dedicated schema named "partman" so its functions | |||||
| -- are referenced as partman.xxx (pg_partman 5.x does not create the schema | |||||
| -- automatically; without this CREATE EXTENSION lands in public and | |||||
| -- partman.create_parent fails with "schema partman does not exist"). | |||||
| CREATE SCHEMA IF NOT EXISTS partman; | |||||
| CREATE EXTENSION IF NOT EXISTS pg_partman WITH SCHEMA partman; | |||||
| -- pg_cron must be created in the database set by cron.database_name (new-api), | |||||
| -- which requires shared_preload_libraries='pg_cron' (configured in the image). | |||||
| -- Include partman in the default search_path so its functions resolve without | |||||
| -- schema-qualifying every call below. | |||||
| SET search_path = partman, public; | |||||
| CREATE EXTENSION IF NOT EXISTS pg_cron; | |||||
| -- 2. Partitioned parent table ----------------------------------------------- | |||||
| -- Columns mirror model.Log exactly. PG requires the partition key (created_at) | |||||
| -- to be part of the primary key, so it is a composite (id, created_at). The app | |||||
| -- never queries logs by id alone, so this does not affect business logic. | |||||
| CREATE TABLE IF NOT EXISTS public.logs ( | |||||
| id BIGSERIAL, | |||||
| user_id INTEGER, | |||||
| created_at BIGINT NOT NULL, | |||||
| type INTEGER, | |||||
| content TEXT, | |||||
| username VARCHAR(64) DEFAULT '', | |||||
| token_name VARCHAR(255) DEFAULT '', | |||||
| model_name VARCHAR(255) DEFAULT '', | |||||
| quota INTEGER DEFAULT 0, | |||||
| prompt_tokens INTEGER DEFAULT 0, | |||||
| completion_tokens INTEGER DEFAULT 0, | |||||
| use_time INTEGER DEFAULT 0, | |||||
| is_stream BOOLEAN, | |||||
| channel_id INTEGER, | |||||
| token_id INTEGER DEFAULT 0, | |||||
| "group" VARCHAR(255), | |||||
| ip VARCHAR(64) DEFAULT '', | |||||
| request_id VARCHAR(64) DEFAULT '', | |||||
| chat_id VARCHAR(128) DEFAULT '', | |||||
| upstream_id VARCHAR(128) DEFAULT '', | |||||
| other TEXT, | |||||
| PRIMARY KEY (id, created_at) | |||||
| ) PARTITION BY RANGE (created_at); | |||||
| -- 3. Hand off to pg_partman -------------------------------------------------- | |||||
| -- Weekly native range partitioning on the bigint epoch (seconds) column. | |||||
| -- p_epoch='seconds' -> created_at is a unix-seconds bigint | |||||
| -- p_type='range' -> pg_partman 5.x uses PG-native partitioning with | |||||
| -- p_type values 'range'/'list' (the old 'native' | |||||
| -- alias from 4.x was removed) | |||||
| -- p_interval='1 week' -> one partition per week (pg_partman 5.x dropped | |||||
| -- the 'weekly' preset in favor of native PG | |||||
| -- interval values) | |||||
| -- p_date_trunc_interval='week' -> align partition boundaries to ISO weeks (Monday) | |||||
| -- p_premake=8 -> always keep 8 future weeks pre-created | |||||
| -- p_default_table=true -> create a DEFAULT partition catching out-of-range | |||||
| -- inserts so they never fail silently (RecordConsumeLog | |||||
| -- only logs errors without retrying) | |||||
| -- No retention is set: per current requirement we do NOT auto-drop old partitions. | |||||
| SELECT partman.create_parent( | |||||
| p_parent_table => 'public.logs', | |||||
| p_control => 'created_at', | |||||
| p_type => 'range', | |||||
| p_interval => '1 week', | |||||
| p_epoch => 'seconds', | |||||
| p_date_trunc_interval => 'week', | |||||
| p_premake => 8, | |||||
| p_default_table => true | |||||
| ); | |||||
| -- 4. Core indexes on the parent table --------------------------------------- | |||||
| -- PG 11+ propagates indexes created on a partitioned parent to all child | |||||
| -- partitions automatically. These cover the hot query paths in model/log.go | |||||
| -- (GetAllLogs / GetUserLogs ordering by created_at desc, id desc; lookups by | |||||
| -- user/model/channel/request/token). Trimmed from the model's 16 index tags to | |||||
| -- the 6 actually used, cutting index write overhead. | |||||
| CREATE INDEX IF NOT EXISTS idx_logs_created_at_id ON public.logs (created_at DESC, id DESC); | |||||
| CREATE INDEX IF NOT EXISTS idx_logs_user_id_created ON public.logs (user_id, created_at DESC); | |||||
| CREATE INDEX IF NOT EXISTS idx_logs_model_name ON public.logs (model_name); | |||||
| CREATE INDEX IF NOT EXISTS idx_logs_channel_id ON public.logs (channel_id); | |||||
| CREATE INDEX IF NOT EXISTS idx_logs_request_id ON public.logs (request_id); | |||||
| CREATE INDEX IF NOT EXISTS idx_logs_token_id ON public.logs (token_id); | |||||
| -- 5. Schedule periodic maintenance ------------------------------------------ | |||||
| -- run_maintenance_proc() inspects partman.part_config and premakes the next | |||||
| -- partitions when needed. Every 30 minutes is more than enough; weekly partitions | |||||
| -- only need creation roughly once a week. No retention => no drops. | |||||
| SELECT cron.schedule( | |||||
| 'log-partition-maint', | |||||
| '*/30 * * * *', | |||||
| $$CALL partman.run_maintenance_proc()$$ | |||||
| ); | |||||
| @@ -266,7 +266,12 @@ func migrateDB() error { | |||||
| &Option{}, | &Option{}, | ||||
| &Redemption{}, | &Redemption{}, | ||||
| &Ability{}, | &Ability{}, | ||||
| &Log{}, | |||||
| // &Log{} is intentionally omitted here. On PostgreSQL the logs table is a | |||||
| // partitioned table managed by the pg_partman extension; GORM's AutoMigrate | |||||
| // would try to alter its composite primary key (id, created_at) and create | |||||
| // redundant per-partition indexes, breaking startup. The logs table is | |||||
| // handled separately by ensureLogTable() below. On SQLite/MySQL it falls | |||||
| // back to the original AutoMigrate behavior. | |||||
| &Midjourney{}, | &Midjourney{}, | ||||
| &TopUp{}, | &TopUp{}, | ||||
| &QuotaData{}, | &QuotaData{}, | ||||
| @@ -296,6 +301,9 @@ func migrateDB() error { | |||||
| if err != nil { | if err != nil { | ||||
| return err | return err | ||||
| } | } | ||||
| if err := ensureLogTable(); err != nil { | |||||
| return err | |||||
| } | |||||
| if err := DB.Exec("DROP TABLE IF EXISTS channel_pricings").Error; err != nil { | if err := DB.Exec("DROP TABLE IF EXISTS channel_pricings").Error; err != nil { | ||||
| return err | return err | ||||
| } | } | ||||
| @@ -337,7 +345,8 @@ func migrateDBFast() error { | |||||
| {&Option{}, "Option"}, | {&Option{}, "Option"}, | ||||
| {&Redemption{}, "Redemption"}, | {&Redemption{}, "Redemption"}, | ||||
| {&Ability{}, "Ability"}, | {&Ability{}, "Ability"}, | ||||
| {&Log{}, "Log"}, | |||||
| // &Log{} omitted: see ensureLogTable() / migrateDB() for rationale | |||||
| // (pg_partman-managed partitioned table on PostgreSQL). | |||||
| {&Midjourney{}, "Midjourney"}, | {&Midjourney{}, "Midjourney"}, | ||||
| {&TopUp{}, "TopUp"}, | {&TopUp{}, "TopUp"}, | ||||
| {&QuotaData{}, "QuotaData"}, | {&QuotaData{}, "QuotaData"}, | ||||
| @@ -386,6 +395,9 @@ func migrateDBFast() error { | |||||
| return err | return err | ||||
| } | } | ||||
| } | } | ||||
| if err := ensureLogTable(); err != nil { | |||||
| return err | |||||
| } | |||||
| if common.UsingSQLite { | if common.UsingSQLite { | ||||
| if err := ensureSubscriptionPlanTableSQLite(); err != nil { | if err := ensureSubscriptionPlanTableSQLite(); err != nil { | ||||
| return err | return err | ||||
| @@ -399,7 +411,61 @@ func migrateDBFast() error { | |||||
| return nil | return nil | ||||
| } | } | ||||
| // ensureLogTable handles the logs table, which behaves differently depending on | |||||
| // the database backend: | |||||
| // | |||||
| // - SQLite / MySQL: behaves as before, GORM AutoMigrate creates/maintains it. | |||||
| // - PostgreSQL: the logs table is a RANGE-partitioned table on created_at, | |||||
| // created and maintained by the pg_partman extension (initdb script). GORM's | |||||
| // AutoMigrate must NOT touch it, otherwise it would try to alter the composite | |||||
| // primary key (id, created_at) - required by the partition key - and create | |||||
| // redundant per-partition indexes from the model's index tags, breaking | |||||
| // startup. Here we only verify the table exists and warn if it does not. | |||||
| // | |||||
| // No partition creation/maintenance logic lives in application code; pg_partman | |||||
| // plus pg_cron own that responsibility entirely on the database side. | |||||
| func ensureLogTable() error { | |||||
| if !common.UsingPostgreSQL { | |||||
| return DB.AutoMigrate(&Log{}) | |||||
| } | |||||
| var exists bool | |||||
| if err := DB.Raw(`SELECT EXISTS ( | |||||
| SELECT 1 FROM information_schema.tables | |||||
| WHERE table_schema = current_schema() AND table_name = 'logs' | |||||
| )`).Scan(&exists).Error; err != nil { | |||||
| return err | |||||
| } | |||||
| if exists { | |||||
| common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate") | |||||
| return nil | |||||
| } | |||||
| common.SysLog("WARNING: PostgreSQL detected but 'logs' table not found. " + | |||||
| "Ensure the pg_partman init script created the partitioned logs table before starting new-api.") | |||||
| return nil | |||||
| } | |||||
| func migrateLOGDB() error { | func migrateLOGDB() error { | ||||
| // When LOG_SQL_DSN is empty, LOG_DB == DB and InitLogDB returns early without | |||||
| // calling this function, so the logs table is already handled by ensureLogTable() | |||||
| // during migrateDB(). This branch only runs for a dedicated PostgreSQL log | |||||
| // database: keep pg_partman-managed behavior (skip AutoMigrate) consistent with | |||||
| // the main database. | |||||
| if common.LogSqlType == common.DatabaseTypePostgreSQL { | |||||
| var exists bool | |||||
| if err := LOG_DB.Raw(`SELECT EXISTS ( | |||||
| SELECT 1 FROM information_schema.tables | |||||
| WHERE table_schema = current_schema() AND table_name = 'logs' | |||||
| )`).Scan(&exists).Error; err != nil { | |||||
| return err | |||||
| } | |||||
| if exists { | |||||
| common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate") | |||||
| return nil | |||||
| } | |||||
| common.SysLog("WARNING: PostgreSQL log database detected but 'logs' table not found. " + | |||||
| "Ensure the pg_partman init script created the partitioned logs table before starting new-api.") | |||||
| return nil | |||||
| } | |||||
| var err error | var err error | ||||
| if err = LOG_DB.AutoMigrate(&Log{}); err != nil { | if err = LOG_DB.AutoMigrate(&Log{}); err != nil { | ||||
| return err | return err | ||||