On PostgreSQL the logs table is a RANGE-partitioned table on created_at owned by pg_partman; GORM AutoMigrate would alter its composite primary key (id, created_at) and create redundant per-partition indexes, breaking startup. Extract ensureLogTable() to skip AutoMigrate on PostgreSQL (existence check only) while keeping the original behavior on SQLite/MySQL. The dedicated LOG_SQL_DSN PostgreSQL branch follows the same rule. Ship the matching postgres-partman image (Dockerfile + 01-partition.sql) that creates and maintains the partitioned table via pg_partman + pg_cron. Co-Authored-By: ZCode <noreply@anthropic.com>master
| @@ -0,0 +1,39 @@ | |||
| # Custom PostgreSQL 18.4 image with pg_partman + pg_cron extensions. | |||
| # | |||
| # The logs table of new-api is a RANGE-partitioned table on created_at, created | |||
| # and maintained entirely on the database side by the pg_partman extension. | |||
| # pg_cron drives periodic maintenance (premake future weekly partitions). | |||
| # | |||
| # Both extensions are installed via Debian packages. pg_cron must be loaded via | |||
| # shared_preload_libraries, so we append it to postgresql.conf.sample: initdb | |||
| # generates PGDATA/postgresql.conf from this sample, meaning both the temporary | |||
| # server (during docker-entrypoint-initdb.d) and the real server load pg_cron, | |||
| # allowing CREATE EXTENSION pg_cron to succeed at init time. | |||
| # | |||
| # Build/push: | |||
| # docker build -t registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 . | |||
| # docker push registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 | |||
| FROM postgres:18 | |||
| # Use Aliyun Debian mirror for faster/reliable apt downloads from within China. | |||
| # postgres:18 is based on Debian trixie which uses the deb822-format | |||
| # /etc/apt/sources.list.d/debian.sources file instead of sources.list. | |||
| RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' \ | |||
| /etc/apt/sources.list.d/debian.sources | |||
| RUN apt-get update \ | |||
| && apt-get install -y --no-install-recommends \ | |||
| postgresql-18-partman \ | |||
| postgresql-18-cron \ | |||
| && rm -rf /var/lib/apt/lists/* | |||
| # Preload pg_cron so it is available during first-run init scripts. | |||
| # pg_partman is a plain SQL extension and does not need preloading. | |||
| RUN echo "shared_preload_libraries = 'pg_cron'" >> /usr/share/postgresql/18/postgresql.conf.sample | |||
| RUN echo "cron.database_name = 'new-api'" >> /usr/share/postgresql/18/postgresql.conf.sample | |||
| # First-run init: create extensions, the partitioned logs parent table, register | |||
| # it with pg_partman, build core indexes, and schedule maintenance via pg_cron. | |||
| # Runs against the database named by POSTGRES_DB (new-api). | |||
| COPY docker-entrypoint-initdb.d/01-partition.sql /docker-entrypoint-initdb.d/01-partition.sql | |||
| @@ -0,0 +1,102 @@ | |||
| -- 01-partition.sql | |||
| -- First-run initialization for new-api's partitioned logs table. | |||
| -- Executed by the postgres official entrypoint against the database named by | |||
| -- POSTGRES_DB (new-api), as a superuser. | |||
| -- | |||
| -- This script is the SINGLE source of truth for the logs partitioning setup. | |||
| -- The new-api Go application does NOT create or maintain partitions; it only | |||
| -- skips GORM AutoMigrate for the logs table on PostgreSQL (see | |||
| -- model/main.go::ensureLogTable). pg_partman owns partition lifecycle, pg_cron | |||
| -- drives periodic maintenance. | |||
| -- 1. Extensions -------------------------------------------------------------- | |||
| -- Install pg_partman into a dedicated schema named "partman" so its functions | |||
| -- are referenced as partman.xxx (pg_partman 5.x does not create the schema | |||
| -- automatically; without this CREATE EXTENSION lands in public and | |||
| -- partman.create_parent fails with "schema partman does not exist"). | |||
| CREATE SCHEMA IF NOT EXISTS partman; | |||
| CREATE EXTENSION IF NOT EXISTS pg_partman WITH SCHEMA partman; | |||
| -- pg_cron must be created in the database set by cron.database_name (new-api), | |||
| -- which requires shared_preload_libraries='pg_cron' (configured in the image). | |||
| -- Include partman in the default search_path so its functions resolve without | |||
| -- schema-qualifying every call below. | |||
| SET search_path = partman, public; | |||
| CREATE EXTENSION IF NOT EXISTS pg_cron; | |||
| -- 2. Partitioned parent table ----------------------------------------------- | |||
| -- Columns mirror model.Log exactly. PG requires the partition key (created_at) | |||
| -- to be part of the primary key, so it is a composite (id, created_at). The app | |||
| -- never queries logs by id alone, so this does not affect business logic. | |||
| CREATE TABLE IF NOT EXISTS public.logs ( | |||
| id BIGSERIAL, | |||
| user_id INTEGER, | |||
| created_at BIGINT NOT NULL, | |||
| type INTEGER, | |||
| content TEXT, | |||
| username VARCHAR(64) DEFAULT '', | |||
| token_name VARCHAR(255) DEFAULT '', | |||
| model_name VARCHAR(255) DEFAULT '', | |||
| quota INTEGER DEFAULT 0, | |||
| prompt_tokens INTEGER DEFAULT 0, | |||
| completion_tokens INTEGER DEFAULT 0, | |||
| use_time INTEGER DEFAULT 0, | |||
| is_stream BOOLEAN, | |||
| channel_id INTEGER, | |||
| token_id INTEGER DEFAULT 0, | |||
| "group" VARCHAR(255), | |||
| ip VARCHAR(64) DEFAULT '', | |||
| request_id VARCHAR(64) DEFAULT '', | |||
| chat_id VARCHAR(128) DEFAULT '', | |||
| upstream_id VARCHAR(128) DEFAULT '', | |||
| other TEXT, | |||
| PRIMARY KEY (id, created_at) | |||
| ) PARTITION BY RANGE (created_at); | |||
| -- 3. Hand off to pg_partman -------------------------------------------------- | |||
| -- Weekly native range partitioning on the bigint epoch (seconds) column. | |||
| -- p_epoch='seconds' -> created_at is a unix-seconds bigint | |||
| -- p_type='range' -> pg_partman 5.x uses PG-native partitioning with | |||
| -- p_type values 'range'/'list' (the old 'native' | |||
| -- alias from 4.x was removed) | |||
| -- p_interval='1 week' -> one partition per week (pg_partman 5.x dropped | |||
| -- the 'weekly' preset in favor of native PG | |||
| -- interval values) | |||
| -- p_date_trunc_interval='week' -> align partition boundaries to ISO weeks (Monday) | |||
| -- p_premake=8 -> always keep 8 future weeks pre-created | |||
| -- p_default_table=true -> create a DEFAULT partition catching out-of-range | |||
| -- inserts so they never fail silently (RecordConsumeLog | |||
| -- only logs errors without retrying) | |||
| -- No retention is set: per current requirement we do NOT auto-drop old partitions. | |||
| SELECT partman.create_parent( | |||
| p_parent_table => 'public.logs', | |||
| p_control => 'created_at', | |||
| p_type => 'range', | |||
| p_interval => '1 week', | |||
| p_epoch => 'seconds', | |||
| p_date_trunc_interval => 'week', | |||
| p_premake => 8, | |||
| p_default_table => true | |||
| ); | |||
| -- 4. Core indexes on the parent table --------------------------------------- | |||
| -- PG 11+ propagates indexes created on a partitioned parent to all child | |||
| -- partitions automatically. These cover the hot query paths in model/log.go | |||
| -- (GetAllLogs / GetUserLogs ordering by created_at desc, id desc; lookups by | |||
| -- user/model/channel/request/token). Trimmed from the model's 16 index tags to | |||
| -- the 6 actually used, cutting index write overhead. | |||
| CREATE INDEX IF NOT EXISTS idx_logs_created_at_id ON public.logs (created_at DESC, id DESC); | |||
| CREATE INDEX IF NOT EXISTS idx_logs_user_id_created ON public.logs (user_id, created_at DESC); | |||
| CREATE INDEX IF NOT EXISTS idx_logs_model_name ON public.logs (model_name); | |||
| CREATE INDEX IF NOT EXISTS idx_logs_channel_id ON public.logs (channel_id); | |||
| CREATE INDEX IF NOT EXISTS idx_logs_request_id ON public.logs (request_id); | |||
| CREATE INDEX IF NOT EXISTS idx_logs_token_id ON public.logs (token_id); | |||
| -- 5. Schedule periodic maintenance ------------------------------------------ | |||
| -- run_maintenance_proc() inspects partman.part_config and premakes the next | |||
| -- partitions when needed. Every 30 minutes is more than enough; weekly partitions | |||
| -- only need creation roughly once a week. No retention => no drops. | |||
| SELECT cron.schedule( | |||
| 'log-partition-maint', | |||
| '*/30 * * * *', | |||
| $$CALL partman.run_maintenance_proc()$$ | |||
| ); | |||
| @@ -266,7 +266,12 @@ func migrateDB() error { | |||
| &Option{}, | |||
| &Redemption{}, | |||
| &Ability{}, | |||
| &Log{}, | |||
| // &Log{} is intentionally omitted here. On PostgreSQL the logs table is a | |||
| // partitioned table managed by the pg_partman extension; GORM's AutoMigrate | |||
| // would try to alter its composite primary key (id, created_at) and create | |||
| // redundant per-partition indexes, breaking startup. The logs table is | |||
| // handled separately by ensureLogTable() below. On SQLite/MySQL it falls | |||
| // back to the original AutoMigrate behavior. | |||
| &Midjourney{}, | |||
| &TopUp{}, | |||
| &QuotaData{}, | |||
| @@ -296,6 +301,9 @@ func migrateDB() error { | |||
| if err != nil { | |||
| return err | |||
| } | |||
| if err := ensureLogTable(); err != nil { | |||
| return err | |||
| } | |||
| if err := DB.Exec("DROP TABLE IF EXISTS channel_pricings").Error; err != nil { | |||
| return err | |||
| } | |||
| @@ -337,7 +345,8 @@ func migrateDBFast() error { | |||
| {&Option{}, "Option"}, | |||
| {&Redemption{}, "Redemption"}, | |||
| {&Ability{}, "Ability"}, | |||
| {&Log{}, "Log"}, | |||
| // &Log{} omitted: see ensureLogTable() / migrateDB() for rationale | |||
| // (pg_partman-managed partitioned table on PostgreSQL). | |||
| {&Midjourney{}, "Midjourney"}, | |||
| {&TopUp{}, "TopUp"}, | |||
| {&QuotaData{}, "QuotaData"}, | |||
| @@ -386,6 +395,9 @@ func migrateDBFast() error { | |||
| return err | |||
| } | |||
| } | |||
| if err := ensureLogTable(); err != nil { | |||
| return err | |||
| } | |||
| if common.UsingSQLite { | |||
| if err := ensureSubscriptionPlanTableSQLite(); err != nil { | |||
| return err | |||
| @@ -399,7 +411,61 @@ func migrateDBFast() error { | |||
| return nil | |||
| } | |||
| // ensureLogTable handles the logs table, which behaves differently depending on | |||
| // the database backend: | |||
| // | |||
| // - SQLite / MySQL: behaves as before, GORM AutoMigrate creates/maintains it. | |||
| // - PostgreSQL: the logs table is a RANGE-partitioned table on created_at, | |||
| // created and maintained by the pg_partman extension (initdb script). GORM's | |||
| // AutoMigrate must NOT touch it, otherwise it would try to alter the composite | |||
| // primary key (id, created_at) - required by the partition key - and create | |||
| // redundant per-partition indexes from the model's index tags, breaking | |||
| // startup. Here we only verify the table exists and warn if it does not. | |||
| // | |||
| // No partition creation/maintenance logic lives in application code; pg_partman | |||
| // plus pg_cron own that responsibility entirely on the database side. | |||
| func ensureLogTable() error { | |||
| if !common.UsingPostgreSQL { | |||
| return DB.AutoMigrate(&Log{}) | |||
| } | |||
| var exists bool | |||
| if err := DB.Raw(`SELECT EXISTS ( | |||
| SELECT 1 FROM information_schema.tables | |||
| WHERE table_schema = current_schema() AND table_name = 'logs' | |||
| )`).Scan(&exists).Error; err != nil { | |||
| return err | |||
| } | |||
| if exists { | |||
| common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate") | |||
| return nil | |||
| } | |||
| common.SysLog("WARNING: PostgreSQL detected but 'logs' table not found. " + | |||
| "Ensure the pg_partman init script created the partitioned logs table before starting new-api.") | |||
| return nil | |||
| } | |||
| func migrateLOGDB() error { | |||
| // When LOG_SQL_DSN is empty, LOG_DB == DB and InitLogDB returns early without | |||
| // calling this function, so the logs table is already handled by ensureLogTable() | |||
| // during migrateDB(). This branch only runs for a dedicated PostgreSQL log | |||
| // database: keep pg_partman-managed behavior (skip AutoMigrate) consistent with | |||
| // the main database. | |||
| if common.LogSqlType == common.DatabaseTypePostgreSQL { | |||
| var exists bool | |||
| if err := LOG_DB.Raw(`SELECT EXISTS ( | |||
| SELECT 1 FROM information_schema.tables | |||
| WHERE table_schema = current_schema() AND table_name = 'logs' | |||
| )`).Scan(&exists).Error; err != nil { | |||
| return err | |||
| } | |||
| if exists { | |||
| common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate") | |||
| return nil | |||
| } | |||
| common.SysLog("WARNING: PostgreSQL log database detected but 'logs' table not found. " + | |||
| "Ensure the pg_partman init script created the partitioned logs table before starting new-api.") | |||
| return nil | |||
| } | |||
| var err error | |||
| if err = LOG_DB.AutoMigrate(&Log{}); err != nil { | |||
| return err | |||