Files
geo/server/internal/tenant/app/kol_generation_service.go
T
root db95b8e4ee feat(kol): store KOL prompt content in database with object storage fallback
Add prompt_content columns to kol_prompts and kol_prompt_revisions so
prompt bodies live alongside their metadata, eliminating an extra round
trip to object storage for the common read path while keeping the old
asset key as a fallback for legacy rows.

Reads go through a singleflight-deduped, cache-backed loader that
prefers the database column, falls back to the asset key, and tolerates
missing objects via a new objectstorage.ErrObjectNotFound returned by
both the Aliyun OSS and MinIO clients on 404/NoSuchKey responses.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 01:47:51 +08:00

629 lines
22 KiB
Go

package app
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/sync/singleflight"
"github.com/geo-platform/tenant-api/internal/shared/auth"
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
"github.com/geo-platform/tenant-api/internal/shared/response"
"github.com/geo-platform/tenant-api/internal/tenant/repository"
)
const (
kolGenerationTaskType = "kol"
kolGenerationTaskRecordType = "article_generation"
kolGenerationTaskRecordScope = "generation_task"
)
var ErrKolGenerationForbidden = response.ErrForbidden(40373, "kol_generation_forbidden", "subscription prompt is not available")
type KolGenerationSubmitRequest struct {
Variables map[string]any `json:"variables"`
EnableWebSearch bool `json:"enable_web_search"`
KnowledgeGroupIDs []int64 `json:"knowledge_group_ids"`
}
type KolGenerationSubmitResponse struct {
ArticleID int64 `json:"article_id"`
GenerationTaskID int64 `json:"generation_task_id"`
UsageLogID int64 `json:"usage_log_id"`
}
type ScheduleKolGenerationInput struct {
ScheduleTaskID int64
OperatorID *int64
TenantID int64
WorkspaceID int64
BrandID *int64
SubscriptionPromptID int64
Name string
Variables map[string]any
EnableWebSearch bool
KnowledgeGroupIDs []int64
AutoPublish bool
PublishAccountIDs []string
CoverAssetURL *string
CoverImageAssetID *int64
GenerateCount int
ScheduledFor time.Time
}
type KolSubscriptionPromptSchemaResponse struct {
SubscriptionPromptID int64 `json:"subscription_prompt_id"`
PackageID int64 `json:"package_id"`
PackageName string `json:"package_name"`
PromptID int64 `json:"prompt_id"`
PromptName string `json:"prompt_name"`
PlatformHint *string `json:"platform_hint"`
SchemaJSON *KolSchemaJSON `json:"schema_json"`
CardConfigJSON map[string]interface{} `json:"card_config_json"`
}
type kolGenerationTaskInput struct {
SubscriptionPromptID int64 `json:"subscription_prompt_id"`
SubscriptionID int64 `json:"subscription_id"`
PackageID int64 `json:"package_id"`
PromptID int64 `json:"prompt_id"`
PromptName string `json:"prompt_name"`
PlatformHint string `json:"platform_hint"`
PromptRevisionNo int `json:"prompt_revision_no"`
Variables map[string]any `json:"variables"`
EnableWebSearch bool `json:"enable_web_search"`
KnowledgeGroupIDs []int64 `json:"knowledge_group_ids"`
SchemaSnapshot json.RawMessage `json:"schema_snapshot"`
CardConfigSnapshot json.RawMessage `json:"card_config_snapshot"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
RenderedPrompt string `json:"rendered_prompt"`
}
type KolGenerationService struct {
pool *pgxpool.Pool
subRepo repository.KolSubscriptionRepository
promptRepo repository.KolPromptRepository
usageRepo repository.KolUsageRepository
articleRepo repository.ArticleRepository
taskRepo repository.AuditRepository
quotaRepo repository.QuotaRepository
asset *KolPromptAsset
rabbitMQ *rabbitmq.Client
cache sharedcache.Cache
cacheGroup singleflight.Group
}
func NewKolGenerationService(
pool *pgxpool.Pool,
subRepo repository.KolSubscriptionRepository,
promptRepo repository.KolPromptRepository,
usageRepo repository.KolUsageRepository,
articleRepo repository.ArticleRepository,
taskRepo repository.AuditRepository,
quotaRepo repository.QuotaRepository,
asset *KolPromptAsset,
rabbitMQ *rabbitmq.Client,
) *KolGenerationService {
return &KolGenerationService{
pool: pool,
subRepo: subRepo,
promptRepo: promptRepo,
usageRepo: usageRepo,
articleRepo: articleRepo,
taskRepo: taskRepo,
quotaRepo: quotaRepo,
asset: asset,
rabbitMQ: rabbitMQ,
}
}
func (s *KolGenerationService) WithCache(c sharedcache.Cache) *KolGenerationService {
s.cache = c
return s
}
func (s *KolGenerationService) GetSchema(ctx context.Context, actor auth.Actor, subPromptID int64) (*KolSubscriptionPromptSchemaResponse, error) {
row, prompt, err := s.loadAuthorizedPrompt(ctx, actor, subPromptID)
if err != nil {
return nil, err
}
schema, err := decodeKolSchema(prompt.SchemaJSON)
if err != nil {
return nil, response.ErrInternal(50096, "kol_generation_schema_decode_failed", err.Error())
}
cardConfig, err := decodeKolCardConfig(prompt.CardConfigJSON)
if err != nil {
return nil, response.ErrInternal(50097, "kol_generation_card_config_decode_failed", err.Error())
}
return &KolSubscriptionPromptSchemaResponse{
SubscriptionPromptID: row.ID,
PackageID: row.PackageID,
PackageName: row.PackageName,
PromptID: row.PromptID,
PromptName: row.PromptName,
PlatformHint: row.PlatformHint,
SchemaJSON: schema,
CardConfigJSON: cardConfig,
}, nil
}
func (s *KolGenerationService) Submit(ctx context.Context, actor auth.Actor, subPromptID int64, req KolGenerationSubmitRequest) (*KolGenerationSubmitResponse, error) {
brandID, err := requireCurrentBrandID(ctx)
if err != nil {
return nil, err
}
return s.enqueue(ctx, kolGenerationEnqueueInput{
Actor: actor,
BrandID: brandID,
SubscriptionPromptID: subPromptID,
Variables: req.Variables,
EnableWebSearch: req.EnableWebSearch,
KnowledgeGroupIDs: req.KnowledgeGroupIDs,
})
}
func (s *KolGenerationService) EnqueueScheduledGeneration(ctx context.Context, input ScheduleKolGenerationInput) (*KolGenerationSubmitResponse, error) {
if input.BrandID == nil || *input.BrandID <= 0 {
return nil, response.ErrBadRequest(40033, "brand_context_required", "current brand is required")
}
operatorID := int64(0)
if input.OperatorID != nil {
operatorID = *input.OperatorID
}
actor := auth.Actor{
TenantID: input.TenantID,
UserID: operatorID,
PrimaryWorkspaceID: input.WorkspaceID,
}
return s.enqueue(ctx, kolGenerationEnqueueInput{
Actor: actor,
BrandID: *input.BrandID,
SubscriptionPromptID: input.SubscriptionPromptID,
Variables: input.Variables,
EnableWebSearch: input.EnableWebSearch,
KnowledgeGroupIDs: input.KnowledgeGroupIDs,
Schedule: &input,
})
}
type kolGenerationEnqueueInput struct {
Actor auth.Actor
BrandID int64
SubscriptionPromptID int64
Variables map[string]any
EnableWebSearch bool
KnowledgeGroupIDs []int64
Schedule *ScheduleKolGenerationInput
}
func (s *KolGenerationService) enqueue(ctx context.Context, input kolGenerationEnqueueInput) (*KolGenerationSubmitResponse, error) {
actor := input.Actor
row, prompt, err := s.loadAuthorizedPrompt(ctx, actor, input.SubscriptionPromptID)
if err != nil {
return nil, err
}
variables := input.Variables
if variables == nil {
variables = map[string]any{}
}
schema, err := decodeKolSchema(prompt.SchemaJSON)
if err != nil {
return nil, response.ErrInternal(50096, "kol_generation_schema_decode_failed", err.Error())
}
cardConfig, err := decodeKolCardConfig(prompt.CardConfigJSON)
if err != nil {
return nil, response.ErrInternal(50097, "kol_generation_card_config_decode_failed", err.Error())
}
content, err := s.loadPromptContent(ctx, prompt)
if err != nil {
return nil, response.ErrInternal(50098, "kol_generation_prompt_asset_load_failed", err.Error())
}
if strings.TrimSpace(content) == "" {
return nil, response.ErrInternal(50098, "kol_generation_prompt_content_missing", "published prompt content not found")
}
renderedPrompt, err := RenderPrompt(content, *schema, variables)
if err != nil {
return nil, response.ErrBadRequest(40071, "kol_generation_variables_invalid", err.Error())
}
renderedPromptHash := HashPrompt(renderedPrompt)
if input.EnableWebSearch && !kolCardConfigAllowsWebSearch(cardConfig) {
return nil, response.ErrBadRequest(40073, "kol_generation_web_search_disabled", "web search is disabled for this prompt")
}
knowledgeGroupIDs := normalizeInt64IDs(input.KnowledgeGroupIDs)
if len(knowledgeGroupIDs) > 0 && !kolCardConfigAllowsUserKnowledge(cardConfig) {
return nil, response.ErrBadRequest(40074, "kol_generation_knowledge_disabled", "knowledge selection is disabled for this prompt")
}
enableWebSearch := kolCardConfigAllowsWebSearch(cardConfig) && input.EnableWebSearch
promptRevisionNo := 1
if row.PublishedRevisionNo != nil && *row.PublishedRevisionNo > 0 {
promptRevisionNo = *row.PublishedRevisionNo
}
balance, err := s.quotaRepo.GetCurrentBalance(ctx, actor.TenantID, "article_generation")
if err != nil || balance < 1 {
return nil, response.ErrForbidden(40301, "quota_insufficient", "generation quota insufficient, please upgrade your plan")
}
taskInput := map[string]any{
"subscription_prompt_id": input.SubscriptionPromptID,
"subscription_id": row.SubscriptionID,
"package_id": row.PackageID,
"prompt_id": row.PromptID,
"prompt_name": row.PromptName,
"platform_hint": derefString(row.PlatformHint),
"prompt_revision_no": promptRevisionNo,
"variables": variables,
"enable_web_search": enableWebSearch,
"knowledge_group_ids": knowledgeGroupIDs,
"schema_snapshot": json.RawMessage(prompt.SchemaJSON),
"card_config_snapshot": json.RawMessage(prompt.CardConfigJSON),
"rendered_prompt_hash": renderedPromptHash,
"rendered_prompt": renderedPrompt,
}
if input.Schedule != nil {
attachKolScheduleTaskInput(taskInput, *input.Schedule)
}
inputJSON, err := json.Marshal(taskInput)
if err != nil {
return nil, response.ErrInternal(50099, "kol_generation_input_encode_failed", err.Error())
}
variablesJSON, err := json.Marshal(variables)
if err != nil {
return nil, response.ErrBadRequest(40072, "kol_generation_variables_invalid", "variables must be valid json")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50100, "kol_generation_tx_failed", "failed to begin generation transaction")
}
defer rollbackGenerationTx(tx)
quotaTx := repository.NewQuotaRepository(tx)
articleTx := repository.NewArticleRepository(tx)
auditTx := repository.NewAuditRepository(tx)
usageTx := repository.NewKolUsageRepository(tx)
newBalance := balance - 1
reason := "generation_reserve"
referenceType := "generation_task"
if _, err := quotaTx.InsertQuotaLedger(ctx, repository.QuotaLedgerInput{
TenantID: actor.TenantID,
OperatorID: actor.UserID,
QuotaType: "article_generation",
Delta: -1,
BalanceAfter: newBalance,
Reason: &reason,
ReferenceType: &referenceType,
}); err != nil {
return nil, response.ErrInternal(50101, "kol_generation_quota_reserve_failed", err.Error())
}
reservationID, err := quotaTx.CreateQuotaReservation(ctx, repository.QuotaReservationInput{
TenantID: actor.TenantID,
OperatorID: actor.UserID,
QuotaType: "article_generation",
ResourceType: "article",
ResourceID: nil,
ReservedAmount: 1,
ExpireAt: time.Now().Add(time.Hour),
})
if err != nil {
return nil, response.ErrInternal(50102, "kol_generation_reservation_create_failed", err.Error())
}
articleID, err := articleTx.CreateArticle(ctx, repository.CreateArticleInput{
TenantID: actor.TenantID,
BrandID: input.BrandID,
SourceType: kolGenerationTaskType,
KolPromptID: &row.PromptID,
})
if err != nil {
return nil, response.ErrInternal(50103, "kol_generation_article_create_failed", err.Error())
}
if err := articleTx.UpdateArticleGenerateStatus(ctx, articleID, actor.TenantID, "generating"); err != nil {
return nil, response.ErrInternal(50112, "kol_generation_article_state_update_failed", err.Error())
}
if input.Schedule != nil {
coverAssetURL, coverImageAssetID := scheduleCoverState(input.Schedule.CoverAssetURL, input.Schedule.CoverImageAssetID)
title := strings.TrimSpace(input.Schedule.Name)
if title == "" {
title = row.PromptName
}
wizardStateJSON, _ := mergeArticleWizardState(nil, title, coverAssetURL, coverImageAssetID)
if _, err := tx.Exec(ctx, `
UPDATE articles
SET wizard_state_json = $1,
updated_at = NOW()
WHERE id = $2 AND tenant_id = $3
`, wizardStateJSON, articleID, actor.TenantID); err != nil {
return nil, response.ErrInternal(50112, "kol_generation_article_state_update_failed", err.Error())
}
}
if err := quotaTx.UpdateQuotaReservationResource(ctx, reservationID, actor.TenantID, articleID); err != nil {
return nil, response.ErrInternal(50104, "kol_generation_reservation_update_failed", err.Error())
}
taskID, err := auditTx.CreateGenerationTask(ctx, repository.GenerationTaskInput{
OperatorID: actor.UserID,
TenantID: actor.TenantID,
ArticleID: &articleID,
QuotaReservationID: &reservationID,
TaskType: kolGenerationTaskType,
RequestHash: &renderedPromptHash,
InputParamsJSON: inputJSON,
})
if err != nil {
return nil, response.ErrInternal(50105, "kol_generation_task_create_failed", err.Error())
}
if _, err := auditTx.CreateTaskRecord(ctx, repository.TaskRecordInput{
TenantID: actor.TenantID,
OperatorID: actor.UserID,
TaskType: kolGenerationTaskRecordType,
ResourceType: kolGenerationTaskRecordScope,
ResourceID: taskID,
Status: "queued",
}); err != nil {
return nil, response.ErrInternal(50106, "kol_generation_task_record_create_failed", err.Error())
}
usage, err := usageTx.Create(ctx, repository.CreateKolUsageLogInput{
TenantID: actor.TenantID,
SubscriptionID: &row.SubscriptionID,
SubscriptionPromptID: &row.ID,
PackageID: row.PackageID,
PromptID: row.PromptID,
PromptRevisionNo: promptRevisionNo,
GenerationTaskID: &taskID,
VariablesSnapshot: variablesJSON,
SchemaSnapshot: prompt.SchemaJSON,
RenderedPromptHash: &renderedPromptHash,
})
if err != nil {
return nil, response.ErrInternal(50107, "kol_generation_usage_create_failed", err.Error())
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50108, "kol_generation_commit_failed", err.Error())
}
InvalidateArticleCaches(ctx, s.cache, actor.TenantID, &articleID)
task := ClaimedGenerationTask{
ID: taskID,
TenantID: actor.TenantID,
OperatorID: actor.UserID,
ArticleID: articleID,
QuotaReservationID: reservationID,
TaskType: kolGenerationTaskType,
}
if err := publishArticleGenerationTask(ctx, s.rabbitMQ, articleGenerationTaskEnvelope{
TaskID: taskID,
TaskType: kolGenerationTaskType,
TenantID: actor.TenantID,
ArticleID: articleID,
}); err != nil {
logCtx := GenerationTaskLogContext{
TaskID: task.ID,
TenantID: task.TenantID,
OperatorID: task.OperatorID,
ArticleID: task.ArticleID,
ReservationID: task.QuotaReservationID,
TaskType: kolGenerationTaskType,
Stage: "queue_enqueue",
StatusBefore: "queued",
StatusAfter: "failed",
Result: GenerationTaskResultFailure,
}
RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, logCtx)
_ = HandleKolGenerationTaskFailure(context.Background(), s.pool, s.cache, task, "queue_enqueue", err)
return nil, response.ErrServiceUnavailable(50343, "generation_queue_unavailable", "generation queue is busy, please retry")
}
return &KolGenerationSubmitResponse{
ArticleID: articleID,
GenerationTaskID: taskID,
UsageLogID: usage.ID,
}, nil
}
func attachKolScheduleTaskInput(input map[string]any, schedule ScheduleKolGenerationInput) {
input["generation_mode"] = "schedule"
input["schedule_task_id"] = schedule.ScheduleTaskID
input["target_type"] = ScheduleTargetKolSubscriptionPrompt
input["task_name"] = strings.TrimSpace(schedule.Name)
if schedule.WorkspaceID > 0 {
input["workspace_id"] = schedule.WorkspaceID
}
if schedule.GenerateCount > 0 {
input["generate_count"] = schedule.GenerateCount
}
if schedule.BrandID != nil && *schedule.BrandID > 0 {
input["brand_id"] = *schedule.BrandID
}
if !schedule.ScheduledFor.IsZero() {
input["scheduled_for"] = schedule.ScheduledFor.UTC().Format(time.RFC3339)
}
if schedule.AutoPublish {
input["schedule_auto_publish"] = true
input["schedule_publish_account_ids"] = schedule.PublishAccountIDs
if schedule.CoverAssetURL != nil {
input["schedule_cover_asset_url"] = strings.TrimSpace(*schedule.CoverAssetURL)
}
if schedule.CoverImageAssetID != nil {
input["schedule_cover_image_asset_id"] = *schedule.CoverImageAssetID
}
}
}
func scheduleCoverState(coverURL *string, coverImageAssetID *int64) (*string, NullableInt64Input) {
var normalizedURL *string
var normalizedID NullableInt64Input
if coverURL != nil {
trimmed := strings.TrimSpace(*coverURL)
if trimmed != "" {
normalizedURL = &trimmed
normalizedID.Set = true
normalizedID.Value = coverImageAssetID
}
}
return normalizedURL, normalizedID
}
func (s *KolGenerationService) loadAuthorizedPrompt(ctx context.Context, actor auth.Actor, subPromptID int64) (*repository.KolSubscriptionPrompt, *repository.KolPrompt, error) {
row, err := s.subRepo.GetSubscriptionPromptByID(ctx, actor.TenantID, subPromptID)
if err != nil {
return nil, nil, response.ErrInternal(50109, "kol_subscription_prompt_query_failed", err.Error())
}
if row == nil {
return nil, nil, ErrKolGenerationForbidden
}
now := time.Now()
switch {
case row.Status != "active":
return nil, nil, ErrKolGenerationForbidden
case row.SubscriptionStatus != "active":
return nil, nil, ErrKolGenerationForbidden
case row.SubscriptionEndAt != nil && row.SubscriptionEndAt.Before(now):
return nil, nil, ErrKolGenerationForbidden
case row.PackageStatus != "published":
return nil, nil, ErrKolGenerationForbidden
case row.PromptStatus != "active":
return nil, nil, ErrKolGenerationForbidden
}
prompt, err := s.promptRepo.GetByID(ctx, row.CreatorTenantID, row.PromptID)
if err != nil {
return nil, nil, response.ErrInternal(50110, "kol_prompt_query_failed", err.Error())
}
if prompt == nil || !s.promptHasContent(ctx, prompt) {
return nil, nil, response.ErrInternal(50111, "kol_prompt_content_missing", "published prompt content not found")
}
return row, prompt, nil
}
func (s *KolGenerationService) loadPromptContent(ctx context.Context, prompt *repository.KolPrompt) (string, error) {
return loadKolPromptContent(ctx, s.cache, &s.cacheGroup, s.promptRepo, s.asset, prompt)
}
func (s *KolGenerationService) promptHasContent(ctx context.Context, prompt *repository.KolPrompt) bool {
content, err := s.loadPromptContent(ctx, prompt)
return err == nil && strings.TrimSpace(content) != ""
}
func HandleKolGenerationTaskFailure(
ctx context.Context,
pool *pgxpool.Pool,
cache sharedcache.Cache,
task ClaimedGenerationTask,
stage string,
taskErr error,
) error {
cleanupCtx, cancel := newGenerationCleanupContext()
defer cancel()
errMsg := formatGenerationFailure(stage, taskErr)
now := time.Now()
logCtx := GenerationTaskLogContext{
TaskID: task.ID,
TenantID: task.TenantID,
OperatorID: task.OperatorID,
ArticleID: task.ArticleID,
ReservationID: task.QuotaReservationID,
TaskType: kolGenerationTaskType,
Stage: stage,
StatusAfter: "failed",
Result: GenerationTaskResultFailure,
}
RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
auditRepo := repository.NewAuditRepository(pool)
articleRepo := repository.NewArticleRepository(pool)
quotaRepo := repository.NewQuotaRepository(pool)
usageRepo := repository.NewKolUsageRepository(pool)
var failureErr error
if task.ID > 0 {
if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
ID: task.ID,
TenantID: task.TenantID,
Status: "failed",
ErrorMessage: &errMsg,
CompletedAt: &now,
}); err != nil {
failureErr = errors.Join(failureErr, fmt.Errorf("update generation task failed status: %w", err))
}
_ = auditRepo.UpdateTaskRecordStatus(cleanupCtx, repository.TaskRecordInput{
TenantID: task.TenantID,
TaskType: kolGenerationTaskRecordType,
ResourceType: kolGenerationTaskRecordScope,
ResourceID: task.ID,
Status: "failed",
ErrorMessage: &errMsg,
})
_ = usageRepo.MarkFailedByTask(cleanupCtx, task.TenantID, task.ID, errMsg)
}
if task.ArticleID > 0 {
if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, task.ArticleID, task.TenantID, "failed"); err != nil {
failureErr = errors.Join(failureErr, fmt.Errorf("update article failed status: %w", err))
}
}
if task.QuotaReservationID > 0 {
var balance int
balanceErr := error(nil)
if task.OperatorID > 0 {
balance, balanceErr = quotaRepo.GetCurrentBalance(cleanupCtx, task.TenantID, "article_generation")
}
refunded, err := quotaRepo.RefundPendingReservation(cleanupCtx, task.QuotaReservationID, task.TenantID)
if err != nil {
failureErr = errors.Join(failureErr, fmt.Errorf("refund generation reservation: %w", err))
} else if refunded {
if balanceErr != nil {
failureErr = errors.Join(failureErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
} else if task.OperatorID > 0 {
reason := "generation_refund"
referenceType := "generation_task"
taskReferenceID := task.ID
if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
TenantID: task.TenantID,
OperatorID: task.OperatorID,
QuotaType: "article_generation",
Delta: 1,
BalanceAfter: balance + 1,
Reason: &reason,
ReferenceType: &referenceType,
ReferenceID: &taskReferenceID,
}); err != nil {
failureErr = errors.Join(failureErr, fmt.Errorf("insert generation refund ledger: %w", err))
}
}
}
}
InvalidateArticleCaches(cleanupCtx, cache, task.TenantID, &task.ArticleID)
if failureErr != nil {
RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
}
return failureErr
}