Parcourir la source

fix(db): skip logs AutoMigrate for pg_partman-managed PostgreSQL tables

On PostgreSQL the logs table is a RANGE-partitioned table on created_at
owned by pg_partman; GORM AutoMigrate would alter its composite primary
key (id, created_at) and create redundant per-partition indexes,
breaking startup. Extract ensureLogTable() to skip AutoMigrate on
PostgreSQL (existence check only) while keeping the original behavior
on SQLite/MySQL. The dedicated LOG_SQL_DSN PostgreSQL branch follows
the same rule. Ship the matching postgres-partman image (Dockerfile +
01-partition.sql) that creates and maintains the partitioned table via
pg_partman + pg_cron.

Co-Authored-By: ZCode <noreply@anthropic.com>
master
fengsilin il y a 14 heures
Parent
révision
0b79e4ee07
3 fichiers modifiés avec 209 ajouts et 2 suppressions
  1. +39
    -0
      deploy/postgres-partman/Dockerfile
  2. +102
    -0
      deploy/postgres-partman/docker-entrypoint-initdb.d/01-partition.sql
  3. +68
    -2
      model/main.go

+ 39
- 0
deploy/postgres-partman/Dockerfile Voir le fichier

@@ -0,0 +1,39 @@
# Custom PostgreSQL 18.4 image with pg_partman + pg_cron extensions.
#
# The logs table of new-api is a RANGE-partitioned table on created_at, created
# and maintained entirely on the database side by the pg_partman extension.
# pg_cron drives periodic maintenance (premake future weekly partitions).
#
# Both extensions are installed via Debian packages. pg_cron must be loaded via
# shared_preload_libraries, so we append it to postgresql.conf.sample: initdb
# generates PGDATA/postgresql.conf from this sample, meaning both the temporary
# server (during docker-entrypoint-initdb.d) and the real server load pg_cron,
# allowing CREATE EXTENSION pg_cron to succeed at init time.
#
# Build/push:
# docker build -t registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18 .
# docker push registry.cn-hangzhou.aliyuncs.com/fengsilin/postgres-partman:18

FROM postgres:18

# Use Aliyun Debian mirror for faster/reliable apt downloads from within China.
# postgres:18 is based on Debian trixie which uses the deb822-format
# /etc/apt/sources.list.d/debian.sources file instead of sources.list.
RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' \
/etc/apt/sources.list.d/debian.sources

RUN apt-get update \
&& apt-get install -y --no-install-recommends \
postgresql-18-partman \
postgresql-18-cron \
&& rm -rf /var/lib/apt/lists/*

# Preload pg_cron so it is available during first-run init scripts.
# pg_partman is a plain SQL extension and does not need preloading.
RUN echo "shared_preload_libraries = 'pg_cron'" >> /usr/share/postgresql/18/postgresql.conf.sample
RUN echo "cron.database_name = 'new-api'" >> /usr/share/postgresql/18/postgresql.conf.sample

# First-run init: create extensions, the partitioned logs parent table, register
# it with pg_partman, build core indexes, and schedule maintenance via pg_cron.
# Runs against the database named by POSTGRES_DB (new-api).
COPY docker-entrypoint-initdb.d/01-partition.sql /docker-entrypoint-initdb.d/01-partition.sql

+ 102
- 0
deploy/postgres-partman/docker-entrypoint-initdb.d/01-partition.sql Voir le fichier

@@ -0,0 +1,102 @@
-- 01-partition.sql
-- First-run initialization for new-api's partitioned logs table.
-- Executed by the postgres official entrypoint against the database named by
-- POSTGRES_DB (new-api), as a superuser.
--
-- This script is the SINGLE source of truth for the logs partitioning setup.
-- The new-api Go application does NOT create or maintain partitions; it only
-- skips GORM AutoMigrate for the logs table on PostgreSQL (see
-- model/main.go::ensureLogTable). pg_partman owns partition lifecycle, pg_cron
-- drives periodic maintenance.

-- 1. Extensions --------------------------------------------------------------
-- Install pg_partman into a dedicated schema named "partman" so its functions
-- are referenced as partman.xxx (pg_partman 5.x does not create the schema
-- automatically; without this CREATE EXTENSION lands in public and
-- partman.create_parent fails with "schema partman does not exist").
CREATE SCHEMA IF NOT EXISTS partman;
CREATE EXTENSION IF NOT EXISTS pg_partman WITH SCHEMA partman;
-- pg_cron must be created in the database set by cron.database_name (new-api),
-- which requires shared_preload_libraries='pg_cron' (configured in the image).
-- Include partman in the default search_path so its functions resolve without
-- schema-qualifying every call below.
SET search_path = partman, public;
CREATE EXTENSION IF NOT EXISTS pg_cron;

-- 2. Partitioned parent table -----------------------------------------------
-- Columns mirror model.Log exactly. PG requires the partition key (created_at)
-- to be part of the primary key, so it is a composite (id, created_at). The app
-- never queries logs by id alone, so this does not affect business logic.
CREATE TABLE IF NOT EXISTS public.logs (
id BIGSERIAL,
user_id INTEGER,
created_at BIGINT NOT NULL,
type INTEGER,
content TEXT,
username VARCHAR(64) DEFAULT '',
token_name VARCHAR(255) DEFAULT '',
model_name VARCHAR(255) DEFAULT '',
quota INTEGER DEFAULT 0,
prompt_tokens INTEGER DEFAULT 0,
completion_tokens INTEGER DEFAULT 0,
use_time INTEGER DEFAULT 0,
is_stream BOOLEAN,
channel_id INTEGER,
token_id INTEGER DEFAULT 0,
"group" VARCHAR(255),
ip VARCHAR(64) DEFAULT '',
request_id VARCHAR(64) DEFAULT '',
chat_id VARCHAR(128) DEFAULT '',
upstream_id VARCHAR(128) DEFAULT '',
other TEXT,
PRIMARY KEY (id, created_at)
) PARTITION BY RANGE (created_at);

-- 3. Hand off to pg_partman --------------------------------------------------
-- Weekly native range partitioning on the bigint epoch (seconds) column.
-- p_epoch='seconds' -> created_at is a unix-seconds bigint
-- p_type='range' -> pg_partman 5.x uses PG-native partitioning with
-- p_type values 'range'/'list' (the old 'native'
-- alias from 4.x was removed)
-- p_interval='1 week' -> one partition per week (pg_partman 5.x dropped
-- the 'weekly' preset in favor of native PG
-- interval values)
-- p_date_trunc_interval='week' -> align partition boundaries to ISO weeks (Monday)
-- p_premake=8 -> always keep 8 future weeks pre-created
-- p_default_table=true -> create a DEFAULT partition catching out-of-range
-- inserts so they never fail silently (RecordConsumeLog
-- only logs errors without retrying)
-- No retention is set: per current requirement we do NOT auto-drop old partitions.
SELECT partman.create_parent(
p_parent_table => 'public.logs',
p_control => 'created_at',
p_type => 'range',
p_interval => '1 week',
p_epoch => 'seconds',
p_date_trunc_interval => 'week',
p_premake => 8,
p_default_table => true
);

-- 4. Core indexes on the parent table ---------------------------------------
-- PG 11+ propagates indexes created on a partitioned parent to all child
-- partitions automatically. These cover the hot query paths in model/log.go
-- (GetAllLogs / GetUserLogs ordering by created_at desc, id desc; lookups by
-- user/model/channel/request/token). Trimmed from the model's 16 index tags to
-- the 6 actually used, cutting index write overhead.
CREATE INDEX IF NOT EXISTS idx_logs_created_at_id ON public.logs (created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_logs_user_id_created ON public.logs (user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_logs_model_name ON public.logs (model_name);
CREATE INDEX IF NOT EXISTS idx_logs_channel_id ON public.logs (channel_id);
CREATE INDEX IF NOT EXISTS idx_logs_request_id ON public.logs (request_id);
CREATE INDEX IF NOT EXISTS idx_logs_token_id ON public.logs (token_id);

-- 5. Schedule periodic maintenance ------------------------------------------
-- run_maintenance_proc() inspects partman.part_config and premakes the next
-- partitions when needed. Every 30 minutes is more than enough; weekly partitions
-- only need creation roughly once a week. No retention => no drops.
SELECT cron.schedule(
'log-partition-maint',
'*/30 * * * *',
$$CALL partman.run_maintenance_proc()$$
);

+ 68
- 2
model/main.go Voir le fichier

@@ -266,7 +266,12 @@ func migrateDB() error {
&Option{},
&Redemption{},
&Ability{},
&Log{},
// &Log{} is intentionally omitted here. On PostgreSQL the logs table is a
// partitioned table managed by the pg_partman extension; GORM's AutoMigrate
// would try to alter its composite primary key (id, created_at) and create
// redundant per-partition indexes, breaking startup. The logs table is
// handled separately by ensureLogTable() below. On SQLite/MySQL it falls
// back to the original AutoMigrate behavior.
&Midjourney{},
&TopUp{},
&QuotaData{},
@@ -296,6 +301,9 @@ func migrateDB() error {
if err != nil {
return err
}
if err := ensureLogTable(); err != nil {
return err
}
if err := DB.Exec("DROP TABLE IF EXISTS channel_pricings").Error; err != nil {
return err
}
@@ -337,7 +345,8 @@ func migrateDBFast() error {
{&Option{}, "Option"},
{&Redemption{}, "Redemption"},
{&Ability{}, "Ability"},
{&Log{}, "Log"},
// &Log{} omitted: see ensureLogTable() / migrateDB() for rationale
// (pg_partman-managed partitioned table on PostgreSQL).
{&Midjourney{}, "Midjourney"},
{&TopUp{}, "TopUp"},
{&QuotaData{}, "QuotaData"},
@@ -386,6 +395,9 @@ func migrateDBFast() error {
return err
}
}
if err := ensureLogTable(); err != nil {
return err
}
if common.UsingSQLite {
if err := ensureSubscriptionPlanTableSQLite(); err != nil {
return err
@@ -399,7 +411,61 @@ func migrateDBFast() error {
return nil
}

// ensureLogTable handles the logs table, which behaves differently depending on
// the database backend:
//
// - SQLite / MySQL: behaves as before, GORM AutoMigrate creates/maintains it.
// - PostgreSQL: the logs table is a RANGE-partitioned table on created_at,
// created and maintained by the pg_partman extension (initdb script). GORM's
// AutoMigrate must NOT touch it, otherwise it would try to alter the composite
// primary key (id, created_at) - required by the partition key - and create
// redundant per-partition indexes from the model's index tags, breaking
// startup. Here we only verify the table exists and warn if it does not.
//
// No partition creation/maintenance logic lives in application code; pg_partman
// plus pg_cron own that responsibility entirely on the database side.
func ensureLogTable() error {
if !common.UsingPostgreSQL {
return DB.AutoMigrate(&Log{})
}
var exists bool
if err := DB.Raw(`SELECT EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_schema = current_schema() AND table_name = 'logs'
)`).Scan(&exists).Error; err != nil {
return err
}
if exists {
common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate")
return nil
}
common.SysLog("WARNING: PostgreSQL detected but 'logs' table not found. " +
"Ensure the pg_partman init script created the partitioned logs table before starting new-api.")
return nil
}

func migrateLOGDB() error {
// When LOG_SQL_DSN is empty, LOG_DB == DB and InitLogDB returns early without
// calling this function, so the logs table is already handled by ensureLogTable()
// during migrateDB(). This branch only runs for a dedicated PostgreSQL log
// database: keep pg_partman-managed behavior (skip AutoMigrate) consistent with
// the main database.
if common.LogSqlType == common.DatabaseTypePostgreSQL {
var exists bool
if err := LOG_DB.Raw(`SELECT EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_schema = current_schema() AND table_name = 'logs'
)`).Scan(&exists).Error; err != nil {
return err
}
if exists {
common.SysLog("logs table is a pg_partman-managed partitioned table, skipping GORM AutoMigrate")
return nil
}
common.SysLog("WARNING: PostgreSQL log database detected but 'logs' table not found. " +
"Ensure the pg_partman init script created the partitioned logs table before starting new-api.")
return nil
}
var err error
if err = LOG_DB.AutoMigrate(&Log{}); err != nil {
return err


Chargement…
Annuler
Enregistrer