From 0b79e4ee0719d52ce16c7b9014d4991dbb63f8a2 Mon Sep 17 00:00:00 2001 From: fengsilin Date: Wed, 19 Aug 2026 14:25:35 +0800 Subject: [PATCH] fix(db): skip logs AutoMigrate for pg_partman-managed PostgreSQL tables On PostgreSQL the logs table is a RANGE-partitioned table on created_at owned by pg_partman; GORM AutoMigrate would alter its composite primary key (id, created_at) and create redundant per-partition indexes, breaking startup. Extract ensureLogTable() to skip AutoMigrate on PostgreSQL (existence check only) while keeping the original behavior on SQLite/MySQL. The dedicated LOG_SQL_DSN PostgreSQL branch follows the same rule. Ship the matching postgres-partman image (Dockerfile + 01-partition.sql) that creates and maintains the partitioned table via pg_partman + pg_cron. Co-Authored-By: ZCode --- deploy/postgres-partman/Dockerfile | 39 +++++++ .../01-partition.sql | 102 ++++++++++++++++++ model/main.go | 70 +++++++++++- 3 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 deploy/postgres-partman/Dockerfile create mode 100644 deploy/postgres-partman/docker-entrypoint-initdb.d/01-partition.sql diff --git a/deploy/postgres-partman/Dockerfile b/deploy/postgres-partman/Dockerfile new file mode 100644 index 0000000..5dd34f6 --- /dev/null +++ b/deploy/postgres-partman/Dockerfile @@ -0,0 +1,39 @@ +# Custom PostgreSQL 18.4 image with pg_partman + pg_cron extensions. +# +# The logs table of new-api is a RANGE-partitioned table on created_at, created +# and maintained entirely on the database side by the pg_partman extension. +# pg_cron drives periodic maintenance (premake future weekly partitions). +# +# Both extensions are installed via Debian packages. pg_cron must be loaded via +# shared_preload_libraries, so we append it to postgresql.conf.sample: initdb +# generates PGDATA/postgresql.conf from this sample, meaning both the temporary +# server (during docker-entrypoint-initdb.d) and the real server load pg_cron, +# allowing CREATE EXTENSION pg_cron to succeed at init time. +# +# Build/push: +# docker build -t registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 . +# docker push registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 + +FROM postgres:18 + +# Use Aliyun Debian mirror for faster/reliable apt downloads from within China. +# postgres:18 is based on Debian trixie which uses the deb822-format +# /etc/apt/sources.list.d/debian.sources file instead of sources.list. +RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' \ + /etc/apt/sources.list.d/debian.sources + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-18-partman \ + postgresql-18-cron \ + && rm -rf /var/lib/apt/lists/* + +# Preload pg_cron so it is available during first-run init scripts. +# pg_partman is a plain SQL extension and does not need preloading. +RUN echo "shared_preload_libraries = 'pg_cron'" >> /usr/share/postgresql/18/postgresql.conf.sample +RUN echo "cron.database_name = 'new-api'" >> /usr/share/postgresql/18/postgresql.conf.sample + +# First-run init: create extensions, the partitioned logs parent table, register +# it with pg_partman, build core indexes, and schedule maintenance via pg_cron. +# Runs against the database named by POSTGRES_DB (new-api). +COPY docker-entrypoint-initdb.d/01-partition.sql /docker-entrypoint-initdb.d/01-partition.sql diff --git a/deploy/postgres-partman/docker-entrypoint-initdb.d/01-partition.sql b/deploy/postgres-partman/docker-entrypoint-initdb.d/01-partition.sql new file mode 100644 index 0000000..0a0984a --- /dev/null +++ b/deploy/postgres-partman/docker-entrypoint-initdb.d/01-partition.sql @@ -0,0 +1,102 @@ +-- 01-partition.sql +-- First-run initialization for new-api's partitioned logs table. +-- Executed by the postgres official entrypoint against the database named by +-- POSTGRES_DB (new-api), as a superuser. +-- +-- This script is the SINGLE source of truth for the logs partitioning setup. +-- The new-api Go application does NOT create or maintain partitions; it only +-- skips GORM AutoMigrate for the logs table on PostgreSQL (see +-- model/main.go::ensureLogTable). pg_partman owns partition lifecycle, pg_cron +-- drives periodic maintenance. + +-- 1. Extensions -------------------------------------------------------------- +-- Install pg_partman into a dedicated schema named "partman" so its functions +-- are referenced as partman.xxx (pg_partman 5.x does not create the schema +-- automatically; without this CREATE EXTENSION lands in public and +-- partman.create_parent fails with "schema partman does not exist"). +CREATE SCHEMA IF NOT EXISTS partman; +CREATE EXTENSION IF NOT EXISTS pg_partman WITH SCHEMA partman; +-- pg_cron must be created in the database set by cron.database_name (new-api), +-- which requires shared_preload_libraries='pg_cron' (configured in the image). +-- Include partman in the default search_path so its functions resolve without +-- schema-qualifying every call below. +SET search_path = partman, public; +CREATE EXTENSION IF NOT EXISTS pg_cron; + +-- 2. Partitioned parent table ----------------------------------------------- +-- Columns mirror model.Log exactly. PG requires the partition key (created_at) +-- to be part of the primary key, so it is a composite (id, created_at). The app +-- never queries logs by id alone, so this does not affect business logic. +CREATE TABLE IF NOT EXISTS public.logs ( + id BIGSERIAL, + user_id INTEGER, + created_at BIGINT NOT NULL, + type INTEGER, + content TEXT, + username VARCHAR(64) DEFAULT '', + token_name VARCHAR(255) DEFAULT '', + model_name VARCHAR(255) DEFAULT '', + quota INTEGER DEFAULT 0, + prompt_tokens INTEGER DEFAULT 0, + completion_tokens INTEGER DEFAULT 0, + use_time INTEGER DEFAULT 0, + is_stream BOOLEAN, + channel_id INTEGER, + token_id INTEGER DEFAULT 0, + "group" VARCHAR(255), + ip VARCHAR(64) DEFAULT '', + request_id VARCHAR(64) DEFAULT '', + chat_id VARCHAR(128) DEFAULT '', + upstream_id VARCHAR(128) DEFAULT '', + other TEXT, + PRIMARY KEY (id, created_at) +) PARTITION BY RANGE (created_at); + +-- 3. Hand off to pg_partman -------------------------------------------------- +-- Weekly native range partitioning on the bigint epoch (seconds) column. +-- p_epoch='seconds' -> created_at is a unix-seconds bigint +-- p_type='range' -> pg_partman 5.x uses PG-native partitioning with +-- p_type values 'range'/'list' (the old 'native' +-- alias from 4.x was removed) +-- p_interval='1 week' -> one partition per week (pg_partman 5.x dropped +-- the 'weekly' preset in favor of native PG +-- interval values) +-- p_date_trunc_interval='week' -> align partition boundaries to ISO weeks (Monday) +-- p_premake=8 -> always keep 8 future weeks pre-created +-- p_default_table=true -> create a DEFAULT partition catching out-of-range +-- inserts so they never fail silently (RecordConsumeLog +-- only logs errors without retrying) +-- No retention is set: per current requirement we do NOT auto-drop old partitions. +SELECT partman.create_parent( + p_parent_table => 'public.logs', + p_control => 'created_at', + p_type => 'range', + p_interval => '1 week', + p_epoch => 'seconds', + p_date_trunc_interval => 'week', + p_premake => 8, + p_default_table => true +); + +-- 4. Core indexes on the parent table --------------------------------------- +-- PG 11+ propagates indexes created on a partitioned parent to all child +-- partitions automatically. These cover the hot query paths in model/log.go +-- (GetAllLogs / GetUserLogs ordering by created_at desc, id desc; lookups by +-- user/model/channel/request/token). Trimmed from the model's 16 index tags to +-- the 6 actually used, cutting index write overhead. +CREATE INDEX IF NOT EXISTS idx_logs_created_at_id ON public.logs (created_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_logs_user_id_created ON public.logs (user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_logs_model_name ON public.logs (model_name); +CREATE INDEX IF NOT EXISTS idx_logs_channel_id ON public.logs (channel_id); +CREATE INDEX IF NOT EXISTS idx_logs_request_id ON public.logs (request_id); +CREATE INDEX IF NOT EXISTS idx_logs_token_id ON public.logs (token_id); + +-- 5. Schedule periodic maintenance ------------------------------------------ +-- run_maintenance_proc() inspects partman.part_config and premakes the next +-- partitions when needed. Every 30 minutes is more than enough; weekly partitions +-- only need creation roughly once a week. No retention => no drops. +SELECT cron.schedule( + 'log-partition-maint', + '*/30 * * * *', + $$CALL partman.run_maintenance_proc()$$ +); diff --git a/model/main.go b/model/main.go index b60fa97..024b827 100644 --- a/model/main.go +++ b/model/main.go @@ -266,7 +266,12 @@ func migrateDB() error { &Option{}, &Redemption{}, &Ability{}, - &Log{}, + // &Log{} is intentionally omitted here. On PostgreSQL the logs table is a + // partitioned table managed by the pg_partman extension; GORM's AutoMigrate + // would try to alter its composite primary key (id, created_at) and create + // redundant per-partition indexes, breaking startup. The logs table is + // handled separately by ensureLogTable() below. On SQLite/MySQL it falls + // back to the original AutoMigrate behavior. &Midjourney{}, &TopUp{}, &QuotaData{}, @@ -296,6 +301,9 @@ func migrateDB() error { if err != nil { return err } + if err := ensureLogTable(); err != nil { + return err + } if err := DB.Exec("DROP TABLE IF EXISTS channel_pricings").Error; err != nil { return err } @@ -337,7 +345,8 @@ func migrateDBFast() error { {&Option{}, "Option"}, {&Redemption{}, "Redemption"}, {&Ability{}, "Ability"}, - {&Log{}, "Log"}, + // &Log{} omitted: see ensureLogTable() / migrateDB() for rationale + // (pg_partman-managed partitioned table on PostgreSQL). {&Midjourney{}, "Midjourney"}, {&TopUp{}, "TopUp"}, {&QuotaData{}, "QuotaData"}, @@ -386,6 +395,9 @@ func migrateDBFast() error { return err } } + if err := ensureLogTable(); err != nil { + return err + } if common.UsingSQLite { if err := ensureSubscriptionPlanTableSQLite(); err != nil { return err @@ -399,7 +411,61 @@ func migrateDBFast() error { return nil } +// ensureLogTable handles the logs table, which behaves differently depending on +// the database backend: +// +// - SQLite / MySQL: behaves as before, GORM AutoMigrate creates/maintains it. +// - PostgreSQL: the logs table is a RANGE-partitioned table on created_at, +// created and maintained by the pg_partman extension (initdb script). GORM's +// AutoMigrate must NOT touch it, otherwise it would try to alter the composite +// primary key (id, created_at) - required by the partition key - and create +// redundant per-partition indexes from the model's index tags, breaking +// startup. Here we only verify the table exists and warn if it does not. +// +// No partition creation/maintenance logic lives in application code; pg_partman +// plus pg_cron own that responsibility entirely on the database side. +func ensureLogTable() error { + if !common.UsingPostgreSQL { + return DB.AutoMigrate(&Log{}) + } + var exists bool + if err := DB.Raw(`SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = 'logs' + )`).Scan(&exists).Error; err != nil { + return err + } + if exists { + common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate") + return nil + } + common.SysLog("WARNING: PostgreSQL detected but 'logs' table not found. " + + "Ensure the pg_partman init script created the partitioned logs table before starting new-api.") + return nil +} + func migrateLOGDB() error { + // When LOG_SQL_DSN is empty, LOG_DB == DB and InitLogDB returns early without + // calling this function, so the logs table is already handled by ensureLogTable() + // during migrateDB(). This branch only runs for a dedicated PostgreSQL log + // database: keep pg_partman-managed behavior (skip AutoMigrate) consistent with + // the main database. + if common.LogSqlType == common.DatabaseTypePostgreSQL { + var exists bool + if err := LOG_DB.Raw(`SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = 'logs' + )`).Scan(&exists).Error; err != nil { + return err + } + if exists { + common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate") + return nil + } + common.SysLog("WARNING: PostgreSQL log database detected but 'logs' table not found. " + + "Ensure the pg_partman init script created the partitioned logs table before starting new-api.") + return nil + } var err error if err = LOG_DB.AutoMigrate(&Log{}); err != nil { return err