root
1eae6fb6d4
- add /questions/combination-fill endpoint with AI-driven, IP-region-aware matrix fill - extract ip2region resolver from ops/app into shared/ipregion for cross-service use - thread question_id filter through dashboard composite, citation summary, and collect-now - switch template wizard from keyword inputs to brand-question selection (primary + supplemental) - pass brand_question and supplemental_questions through assist/title/outline prompts - add AiWaitingModal + 45s client timeout and request_timeout error mapping for long AI flows Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1138 lines
36 KiB
Go
1138 lines
36 KiB
Go
package app
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha1"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"strings"
|
||
"time"
|
||
"unicode/utf8"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgconn"
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"github.com/geo-platform/tenant-api/internal/shared/auditlog"
|
||
"github.com/geo-platform/tenant-api/internal/shared/auth"
|
||
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
|
||
"github.com/geo-platform/tenant-api/internal/shared/ipregion"
|
||
"github.com/geo-platform/tenant-api/internal/shared/llm"
|
||
"github.com/geo-platform/tenant-api/internal/shared/middleware"
|
||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||
)
|
||
|
||
const (
|
||
questionCombinationMaxItems = 500
|
||
questionCombinationFillMax = 4
|
||
questionAIDistillMaxItems = 20
|
||
questionCombinationFillTTL = 10 * time.Minute
|
||
questionCombinationFillTimeout = 15 * time.Second
|
||
questionDistillCacheTTL = 5 * time.Minute
|
||
questionDistillTimeout = 30 * time.Second
|
||
)
|
||
|
||
type QuestionExpansionService struct {
|
||
pool *pgxpool.Pool
|
||
llm llm.Client
|
||
brand *BrandService
|
||
cache sharedcache.Cache
|
||
ipGeo *ipregion.Resolver
|
||
}
|
||
|
||
func NewQuestionExpansionService(pool *pgxpool.Pool, llmClient llm.Client, brand *BrandService) *QuestionExpansionService {
|
||
return &QuestionExpansionService{
|
||
pool: pool,
|
||
llm: llmClient,
|
||
brand: brand,
|
||
}
|
||
}
|
||
|
||
func (s *QuestionExpansionService) WithCache(c sharedcache.Cache) *QuestionExpansionService {
|
||
s.cache = c
|
||
return s
|
||
}
|
||
|
||
func (s *QuestionExpansionService) WithIPRegionResolver(resolver *ipregion.Resolver) *QuestionExpansionService {
|
||
s.ipGeo = resolver
|
||
return s
|
||
}
|
||
|
||
type QuestionCombinationRequest struct {
|
||
Region []string `json:"region"`
|
||
Prefix []string `json:"prefix"`
|
||
Core []string `json:"core" binding:"required"`
|
||
Industry []string `json:"industry" binding:"required"`
|
||
Suffix []string `json:"suffix"`
|
||
MaxItems int `json:"max_items"`
|
||
}
|
||
|
||
type QuestionDistillRequest struct {
|
||
SeedTopic string `json:"seed_topic" binding:"required,min=2"`
|
||
}
|
||
|
||
type QuestionCombinationFillRequest struct {
|
||
Region []string `json:"region"`
|
||
Prefix []string `json:"prefix"`
|
||
Core []string `json:"core"`
|
||
Industry []string `json:"industry"`
|
||
Suffix []string `json:"suffix"`
|
||
}
|
||
|
||
type QuestionCombinationFillResult struct {
|
||
Region []string `json:"region"`
|
||
Prefix []string `json:"prefix"`
|
||
Core []string `json:"core"`
|
||
Industry []string `json:"industry"`
|
||
Suffix []string `json:"suffix"`
|
||
AIPointsCharged int `json:"ai_points_charged,omitempty"`
|
||
CacheHit bool `json:"cache_hit,omitempty"`
|
||
}
|
||
|
||
type QuestionCandidate struct {
|
||
Text string `json:"text"`
|
||
Layer string `json:"layer"`
|
||
Intent string `json:"intent"`
|
||
Source string `json:"source"`
|
||
MissingSuffix bool `json:"missing_suffix"`
|
||
TooShort bool `json:"too_short"`
|
||
Duplicate bool `json:"duplicate"`
|
||
SuggestSkip bool `json:"suggest_skip"`
|
||
}
|
||
|
||
type QuestionCandidateResult struct {
|
||
Candidates []QuestionCandidate `json:"candidates"`
|
||
AIPointsCharged int `json:"ai_points_charged,omitempty"`
|
||
CacheHit bool `json:"cache_hit,omitempty"`
|
||
Truncated bool `json:"truncated,omitempty"`
|
||
}
|
||
|
||
type MaterializeQuestionsRequest struct {
|
||
Source string `json:"source"`
|
||
Questions []MaterializeQuestion `json:"questions" binding:"required,min=1"`
|
||
}
|
||
|
||
type MaterializeQuestion struct {
|
||
Text string `json:"text" binding:"required"`
|
||
}
|
||
|
||
type MaterializeQuestionsResult struct {
|
||
CreatedQuestions int `json:"created_questions"`
|
||
SkippedQuestions []SkipReason `json:"skipped_questions"`
|
||
}
|
||
|
||
type SkipReason struct {
|
||
Text string `json:"text"`
|
||
Reason string `json:"reason"`
|
||
}
|
||
|
||
type questionBrandContext struct {
|
||
TenantID int64
|
||
BrandID int64
|
||
BrandName string
|
||
Website *string
|
||
Description *string
|
||
PlanCode string
|
||
CompetitorNames []string
|
||
}
|
||
|
||
type aiDistillPayload struct {
|
||
Candidates []struct {
|
||
Text string `json:"text"`
|
||
Intent string `json:"intent"`
|
||
Layer string `json:"layer"`
|
||
} `json:"candidates"`
|
||
}
|
||
|
||
type aiCombinationFillPayload struct {
|
||
Region []string `json:"region"`
|
||
Prefix []string `json:"prefix"`
|
||
Core []string `json:"core"`
|
||
Industry []string `json:"industry"`
|
||
Suffix []string `json:"suffix"`
|
||
}
|
||
|
||
func (s *QuestionExpansionService) GenerateByCombination(ctx context.Context, brandID int64, req QuestionCombinationRequest) (*QuestionCandidateResult, error) {
|
||
actor := auth.MustActor(ctx)
|
||
brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
core := normalizeQuestionParts(req.Core)
|
||
industry := normalizeQuestionParts(req.Industry)
|
||
if len(core) == 0 {
|
||
return nil, response.ErrBadRequest(40091, "invalid_params", "core_required")
|
||
}
|
||
if len(industry) == 0 {
|
||
return nil, response.ErrBadRequest(40091, "invalid_params", "industry_required")
|
||
}
|
||
|
||
region := optionalQuestionParts(req.Region)
|
||
prefix := optionalQuestionParts(req.Prefix)
|
||
suffix := optionalQuestionParts(req.Suffix)
|
||
limit := req.MaxItems
|
||
if limit <= 0 || limit > questionCombinationMaxItems {
|
||
limit = questionCombinationMaxItems
|
||
}
|
||
|
||
existing, err := s.loadExistingQuestionKeys(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
candidates := make([]QuestionCandidate, 0, minInt(limit, 64))
|
||
seen := map[string]struct{}{}
|
||
truncated := false
|
||
|
||
for _, rg := range region {
|
||
for _, pf := range prefix {
|
||
for _, co := range core {
|
||
for _, in := range industry {
|
||
for _, sf := range suffix {
|
||
text := strings.TrimSpace(strings.Join(nonEmptyParts(rg, pf, co, in, sf), ""))
|
||
if text == "" {
|
||
continue
|
||
}
|
||
if sf != "" && !strings.HasSuffix(text, "?") && !strings.HasSuffix(text, "？") {
|
||
if !strings.Contains(sf, "?") && !strings.Contains(sf, "？") {
|
||
text += "？"
|
||
}
|
||
}
|
||
key := normalizeQuestionKey(text)
|
||
if _, ok := seen[key]; ok {
|
||
continue
|
||
}
|
||
seen[key] = struct{}{}
|
||
if len(candidates) >= limit {
|
||
truncated = true
|
||
continue
|
||
}
|
||
candidates = append(candidates, buildCandidate(text, QuestionSourceCombination, sf == "", existing, brandCtx))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return &QuestionCandidateResult{Candidates: candidates, Truncated: truncated}, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) GenerateByAIDistill(ctx context.Context, brandID int64, req QuestionDistillRequest) (*QuestionCandidateResult, error) {
|
||
actor := auth.MustActor(ctx)
|
||
seedTopic := normalizeQuestionText(req.SeedTopic)
|
||
if utf8.RuneCountInString(seedTopic) < 2 {
|
||
return nil, response.ErrBadRequest(40091, "invalid_params", "seed_topic is required")
|
||
}
|
||
|
||
brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
existing, err := s.loadExistingQuestionKeys(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
cacheKey := questionDistillCacheKey(actor.TenantID, brandID, questionDistillPromptVersion, brandCtx.BrandName, brandCtx.CompetitorNames, seedTopic)
|
||
if s.cache != nil {
|
||
if raw, cacheErr := s.cache.Get(ctx, cacheKey); cacheErr == nil && len(raw) > 0 {
|
||
var cached []QuestionCandidate
|
||
if json.Unmarshal(raw, &cached) == nil {
|
||
return &QuestionCandidateResult{Candidates: cached, CacheHit: true}, nil
|
||
}
|
||
}
|
||
}
|
||
|
||
if s.llm == nil {
|
||
return nil, response.ErrServiceUnavailable(50311, "llm_unavailable", "llm client is not configured")
|
||
}
|
||
if err := s.llm.Validate(); err != nil {
|
||
return nil, response.ErrServiceUnavailable(50311, "llm_unavailable", err.Error())
|
||
}
|
||
|
||
resourceType := "question_distill"
|
||
resourceUID := cacheKey
|
||
reservation, err := ReserveAIPoints(ctx, s.pool, s.cache, AIPointReserveInput{
|
||
TenantID: actor.TenantID,
|
||
OperatorID: actor.UserID,
|
||
UsageType: AIUsageTypeQuestionDistill,
|
||
ResourceType: &resourceType,
|
||
ResourceUID: &resourceUID,
|
||
MeteredText: seedTopic,
|
||
FixedPoints: 1,
|
||
Metadata: map[string]any{
|
||
"brand_id": brandID,
|
||
"seed_topic": seedTopic,
|
||
"prompt_version": questionDistillPromptVersion,
|
||
},
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
prompt := buildQuestionDistillPrompt(brandCtx.BrandName, brandCtx.CompetitorNames, seedTopic)
|
||
result, err := s.llm.Generate(ctx, llm.GenerateRequest{
|
||
Prompt: prompt,
|
||
Timeout: questionDistillTimeout,
|
||
MaxOutputTokens: 1600,
|
||
ResponseFormat: &llm.ResponseFormat{
|
||
Type: llm.ResponseFormatTypeJSONSchema,
|
||
Name: "question_distill_candidates",
|
||
Description: "Question expansion candidates for a brand.",
|
||
SchemaJSON: questionDistillSchema,
|
||
Strict: true,
|
||
},
|
||
}, nil)
|
||
if err != nil {
|
||
_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
|
||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||
return nil, response.ErrServiceUnavailable(50312, "llm_timeout", "AI generation timed out")
|
||
}
|
||
return nil, response.ErrServiceUnavailable(50311, "llm_unavailable", err.Error())
|
||
}
|
||
|
||
var payload aiDistillPayload
|
||
if err := json.Unmarshal([]byte(result.Content), &payload); err != nil {
|
||
_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
|
||
return nil, response.ErrBadRequest(50210, "llm_invalid_output", "AI output could not be parsed")
|
||
}
|
||
|
||
candidates := make([]QuestionCandidate, 0, minInt(len(payload.Candidates), questionAIDistillMaxItems))
|
||
seen := map[string]struct{}{}
|
||
for _, item := range payload.Candidates {
|
||
if len(candidates) >= questionAIDistillMaxItems {
|
||
break
|
||
}
|
||
text := normalizeQuestionText(item.Text)
|
||
key := normalizeQuestionKey(text)
|
||
if text == "" {
|
||
continue
|
||
}
|
||
if _, ok := seen[key]; ok {
|
||
continue
|
||
}
|
||
seen[key] = struct{}{}
|
||
candidates = append(candidates, buildCandidate(text, QuestionSourceAIDistill, false, existing, brandCtx))
|
||
}
|
||
|
||
if err := CompleteAIPoints(context.Background(), s.pool, actor.TenantID, *reservation, result.Model); err != nil {
|
||
return nil, response.ErrInternal(50127, "ai_points_commit_failed", "failed to confirm ai points")
|
||
}
|
||
|
||
if s.cache != nil {
|
||
if raw, err := json.Marshal(candidates); err == nil {
|
||
_ = s.cache.Set(context.Background(), cacheKey, raw, questionDistillCacheTTL)
|
||
}
|
||
}
|
||
|
||
return &QuestionCandidateResult{
|
||
Candidates: candidates,
|
||
AIPointsCharged: reservation.Points,
|
||
}, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) FillQuestionCombination(ctx context.Context, brandID int64, clientIP string, req QuestionCombinationFillRequest) (*QuestionCombinationFillResult, error) {
|
||
actor := auth.MustActor(ctx)
|
||
brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
regionTerms := questionCombinationRegionTermsFromIPRegion("")
|
||
if s.ipGeo != nil {
|
||
regionTerms = questionCombinationRegionTermsFromIPRegion(s.ipGeo.Lookup(clientIP))
|
||
}
|
||
fallback := buildQuestionCombinationFillFallback(brandCtx, regionTerms, req)
|
||
|
||
cacheKey := questionCombinationFillCacheKey(actor.TenantID, brandID, brandCtx, regionTerms, req)
|
||
if s.cache != nil {
|
||
if raw, cacheErr := s.cache.Get(ctx, cacheKey); cacheErr == nil && len(raw) > 0 {
|
||
var cached QuestionCombinationFillResult
|
||
if json.Unmarshal(raw, &cached) == nil {
|
||
cached.CacheHit = true
|
||
cached.AIPointsCharged = 0
|
||
return &cached, nil
|
||
}
|
||
}
|
||
}
|
||
|
||
if s.llm == nil || s.llm.Validate() != nil {
|
||
return &fallback, nil
|
||
}
|
||
|
||
resourceType := "question_combination_fill"
|
||
resourceUID := cacheKey
|
||
meteredText, _ := json.Marshal(map[string]any{
|
||
"brand": brandCtx.BrandName,
|
||
"description": nilToString(brandCtx.Description),
|
||
"region": fallback.Region,
|
||
"request": req,
|
||
})
|
||
reservation, err := ReserveAIPoints(ctx, s.pool, s.cache, AIPointReserveInput{
|
||
TenantID: actor.TenantID,
|
||
OperatorID: actor.UserID,
|
||
UsageType: AIUsageTypeQuestionCombinationFill,
|
||
ResourceType: &resourceType,
|
||
ResourceUID: &resourceUID,
|
||
MeteredText: string(meteredText),
|
||
FixedPoints: 1,
|
||
Metadata: map[string]any{
|
||
"brand_id": brandID,
|
||
"prompt_version": "question_combination_fill_v1",
|
||
},
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
result, err := s.llm.Generate(ctx, llm.GenerateRequest{
|
||
Prompt: buildQuestionCombinationFillPrompt(brandCtx, fallback.Region, req),
|
||
Timeout: questionCombinationFillTimeout,
|
||
MaxOutputTokens: 900,
|
||
ResponseFormat: &llm.ResponseFormat{
|
||
Type: llm.ResponseFormatTypeJSONSchema,
|
||
Name: "question_combination_fill",
|
||
Description: "Short word columns for question expansion combination tool.",
|
||
SchemaJSON: questionCombinationFillSchema,
|
||
Strict: true,
|
||
},
|
||
}, nil)
|
||
if err != nil {
|
||
_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
|
||
return &fallback, nil
|
||
}
|
||
|
||
var payload aiCombinationFillPayload
|
||
if err := json.Unmarshal([]byte(result.Content), &payload); err != nil {
|
||
_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
|
||
return &fallback, nil
|
||
}
|
||
|
||
filled := sanitizeQuestionCombinationFillResult(payload, fallback)
|
||
if err := CompleteAIPoints(context.Background(), s.pool, actor.TenantID, *reservation, result.Model); err != nil {
|
||
return nil, response.ErrInternal(50127, "ai_points_commit_failed", "failed to confirm ai points")
|
||
}
|
||
filled.AIPointsCharged = reservation.Points
|
||
if s.cache != nil {
|
||
cached := filled
|
||
cached.AIPointsCharged = 0
|
||
cached.CacheHit = false
|
||
if raw, err := json.Marshal(cached); err == nil {
|
||
_ = s.cache.Set(context.Background(), cacheKey, raw, questionCombinationFillTTL)
|
||
}
|
||
}
|
||
return &filled, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) ClassifyMetadata(ctx context.Context, brandID int64, texts []string) ([]ClassifiedQuestion, error) {
|
||
actor := auth.MustActor(ctx)
|
||
brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result := make([]ClassifiedQuestion, 0, len(texts))
|
||
for _, text := range texts {
|
||
trimmed := normalizeQuestionText(text)
|
||
if trimmed == "" {
|
||
continue
|
||
}
|
||
result = append(result, classifyQuestionText(trimmed, brandCtx.BrandName, brandCtx.CompetitorNames))
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) MaterializeQuestions(ctx context.Context, brandID int64, req MaterializeQuestionsRequest) (*MaterializeQuestionsResult, error) {
|
||
actor := auth.MustActor(ctx)
|
||
source := strings.TrimSpace(req.Source)
|
||
if source == "" {
|
||
source = QuestionSourceManual
|
||
}
|
||
if !isValidExternalQuestionSource(source) {
|
||
return nil, response.ErrBadRequest(40092, "invalid_enum", "source is invalid")
|
||
}
|
||
if len(req.Questions) == 0 {
|
||
return nil, response.ErrBadRequest(40093, "no_valid_questions", "questions is required")
|
||
}
|
||
|
||
brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
existing, err := s.loadExistingQuestionKeys(ctx, actor.TenantID, brandID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
currentCount, err := s.countTenantQuestions(ctx, actor.TenantID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
maxQuestions := s.brand.currentLimits().QuestionLimitForPlan(brandCtx.PlanCode)
|
||
remaining := maxQuestions - currentCount
|
||
if remaining < 0 {
|
||
remaining = 0
|
||
}
|
||
|
||
type acceptedQuestion struct {
|
||
Text string `json:"text"`
|
||
Layer string `json:"layer"`
|
||
Intent string `json:"intent"`
|
||
Source string `json:"source"`
|
||
}
|
||
|
||
accepted := make([]acceptedQuestion, 0, len(req.Questions))
|
||
skipped := make([]SkipReason, 0)
|
||
inBatch := map[string]struct{}{}
|
||
|
||
for _, item := range req.Questions {
|
||
text := normalizeQuestionText(item.Text)
|
||
key := normalizeQuestionKey(text)
|
||
if text == "" || utf8.RuneCountInString(text) < 4 {
|
||
skipped = append(skipped, SkipReason{Text: text, Reason: "too_short"})
|
||
continue
|
||
}
|
||
if !questionLooksValid(text) {
|
||
skipped = append(skipped, SkipReason{Text: text, Reason: "invalid_text"})
|
||
continue
|
||
}
|
||
if _, ok := existing[key]; ok {
|
||
skipped = append(skipped, SkipReason{Text: text, Reason: "duplicate"})
|
||
continue
|
||
}
|
||
if _, ok := inBatch[key]; ok {
|
||
skipped = append(skipped, SkipReason{Text: text, Reason: "duplicate_in_batch"})
|
||
continue
|
||
}
|
||
if remaining <= 0 {
|
||
skipped = append(skipped, SkipReason{Text: text, Reason: "quota_exceeded"})
|
||
continue
|
||
}
|
||
classified := classifyQuestionText(text, brandCtx.BrandName, brandCtx.CompetitorNames)
|
||
accepted = append(accepted, acceptedQuestion{
|
||
Text: text,
|
||
Layer: classified.Layer,
|
||
Intent: classified.Intent,
|
||
Source: source,
|
||
})
|
||
inBatch[key] = struct{}{}
|
||
remaining--
|
||
}
|
||
|
||
if len(accepted) == 0 {
|
||
return nil, response.ErrBadRequest(40093, "no_valid_questions", skippedReasonDetail(skipped))
|
||
}
|
||
|
||
tx, err := s.pool.Begin(ctx)
|
||
if err != nil {
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to begin transaction")
|
||
}
|
||
defer func() {
|
||
_ = tx.Rollback(ctx)
|
||
}()
|
||
|
||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))`, tenantQuestionQuotaLockKey(actor.TenantID), "questions"); err != nil {
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to lock question quota")
|
||
}
|
||
|
||
bucketID, err := s.ensureDefaultQuestionBucket(ctx, tx, actor.TenantID, brandID, brandCtx.BrandName)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
currentCount, err = countTenantQuestionsTx(ctx, tx, actor.TenantID)
|
||
if err != nil {
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to recount questions")
|
||
}
|
||
remaining = maxQuestions - currentCount
|
||
if remaining < 0 {
|
||
remaining = 0
|
||
}
|
||
if remaining < len(accepted) {
|
||
overflow := accepted[remaining:]
|
||
accepted = accepted[:remaining]
|
||
for _, q := range overflow {
|
||
skipped = append(skipped, SkipReason{Text: q.Text, Reason: "quota_exceeded_concurrent"})
|
||
}
|
||
}
|
||
if len(accepted) == 0 {
|
||
return nil, response.ErrBadRequest(40093, "no_valid_questions", skippedReasonDetail(skipped))
|
||
}
|
||
|
||
payload, err := json.Marshal(accepted)
|
||
if err != nil {
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to encode questions")
|
||
}
|
||
rows, err := tx.Query(ctx, `
|
||
INSERT INTO brand_questions (
|
||
tenant_id, brand_id, keyword_id, question_text,
|
||
layer, intent, source, status
|
||
)
|
||
SELECT $1, $2, $3, q.text, q.layer, q.intent, q.source, 'active'
|
||
FROM jsonb_to_recordset($4::jsonb)
|
||
AS q(text TEXT, layer TEXT, intent TEXT, source TEXT)
|
||
ON CONFLICT (tenant_id, brand_id, lower(btrim(question_text)))
|
||
WHERE deleted_at IS NULL
|
||
DO NOTHING
|
||
RETURNING question_text
|
||
`, actor.TenantID, brandID, bucketID, payload)
|
||
if err != nil {
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to insert questions")
|
||
}
|
||
inserted := map[string]struct{}{}
|
||
for rows.Next() {
|
||
var text string
|
||
if err := rows.Scan(&text); err != nil {
|
||
rows.Close()
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to scan inserted questions")
|
||
}
|
||
inserted[normalizeQuestionKey(text)] = struct{}{}
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
rows.Close()
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to iterate inserted questions")
|
||
}
|
||
rows.Close()
|
||
|
||
for _, q := range accepted {
|
||
if _, ok := inserted[normalizeQuestionKey(q.Text)]; !ok {
|
||
skipped = append(skipped, SkipReason{Text: q.Text, Reason: "duplicate_concurrent"})
|
||
}
|
||
}
|
||
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return nil, response.ErrInternal(50010, "materialize_failed", "failed to commit questions")
|
||
}
|
||
|
||
invalidateBrandCaches(ctx, s.cache, actor.TenantID, brandID)
|
||
s.logQuestionExpansionAudit(ctx, actor.UserID, actor.TenantID, brandID, source, len(inserted), skipped)
|
||
return &MaterializeQuestionsResult{
|
||
CreatedQuestions: len(inserted),
|
||
SkippedQuestions: skipped,
|
||
}, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) loadQuestionBrandContext(ctx context.Context, tenantID, brandID int64) (*questionBrandContext, error) {
|
||
var brandName string
|
||
var website *string
|
||
var description *string
|
||
err := s.pool.QueryRow(ctx, `
|
||
SELECT name, website, description
|
||
FROM brands
|
||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
|
||
`, brandID, tenantID).Scan(&brandName, &website, &description)
|
||
if err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return nil, response.ErrNotFound(40420, "brand_not_found", "brand not found")
|
||
}
|
||
return nil, response.ErrInternal(50010, "query_failed", "failed to load brand")
|
||
}
|
||
|
||
rows, err := s.pool.Query(ctx, `
|
||
SELECT name
|
||
FROM competitors
|
||
WHERE brand_id = $1 AND tenant_id = $2 AND deleted_at IS NULL
|
||
ORDER BY id ASC
|
||
`, brandID, tenantID)
|
||
if err != nil {
|
||
return nil, response.ErrInternal(50010, "query_failed", "failed to load competitors")
|
||
}
|
||
defer rows.Close()
|
||
competitors := make([]string, 0)
|
||
for rows.Next() {
|
||
var name string
|
||
if err := rows.Scan(&name); err != nil {
|
||
return nil, response.ErrInternal(50010, "scan_failed", err.Error())
|
||
}
|
||
competitors = append(competitors, name)
|
||
}
|
||
|
||
plan, err := s.brand.loadBrandLibraryPlan(ctx, tenantID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
return &questionBrandContext{
|
||
TenantID: tenantID,
|
||
BrandID: brandID,
|
||
BrandName: brandName,
|
||
Website: website,
|
||
Description: description,
|
||
PlanCode: plan.PlanCode,
|
||
CompetitorNames: competitors,
|
||
}, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) loadExistingQuestionKeys(ctx context.Context, tenantID, brandID int64) (map[string]struct{}, error) {
|
||
rows, err := s.pool.Query(ctx, `
|
||
SELECT lower(btrim(question_text))
|
||
FROM brand_questions
|
||
WHERE tenant_id = $1 AND brand_id = $2 AND deleted_at IS NULL
|
||
`, tenantID, brandID)
|
||
if err != nil {
|
||
return nil, response.ErrInternal(50010, "query_failed", "failed to load existing questions")
|
||
}
|
||
defer rows.Close()
|
||
|
||
result := map[string]struct{}{}
|
||
for rows.Next() {
|
||
var key string
|
||
if err := rows.Scan(&key); err != nil {
|
||
return nil, response.ErrInternal(50010, "scan_failed", err.Error())
|
||
}
|
||
result[key] = struct{}{}
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
func (s *QuestionExpansionService) countTenantQuestions(ctx context.Context, tenantID int64) (int, error) {
|
||
return countTenantQuestionsTx(ctx, s.pool, tenantID)
|
||
}
|
||
|
||
func countTenantQuestionsTx(ctx context.Context, db interface {
|
||
QueryRow(context.Context, string, ...any) pgx.Row
|
||
}, tenantID int64) (int, error) {
|
||
var count int
|
||
err := db.QueryRow(ctx, `
|
||
SELECT COUNT(*)::INT
|
||
FROM brand_questions
|
||
WHERE tenant_id = $1 AND deleted_at IS NULL
|
||
`, tenantID).Scan(&count)
|
||
return count, err
|
||
}
|
||
|
||
func (s *QuestionExpansionService) ensureDefaultQuestionBucket(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, brandName string) (int64, error) {
|
||
return ensureDefaultQuestionBucketTx(ctx, tx, tenantID, brandID, brandName)
|
||
}
|
||
|
||
func (s *QuestionExpansionService) logQuestionExpansionAudit(ctx context.Context, operatorID, tenantID, brandID int64, source string, created int, skipped []SkipReason) {
|
||
if s == nil || s.brand == nil || s.brand.auditLogs == nil {
|
||
return
|
||
}
|
||
afterJSON, _ := json.Marshal(map[string]interface{}{
|
||
"brand_id": brandID,
|
||
"source": source,
|
||
"created_questions": created,
|
||
"skipped_count": len(skipped),
|
||
})
|
||
result := "success"
|
||
resourceType := "brand"
|
||
requestID := middleware.RequestIDFromContext(ctx)
|
||
s.brand.auditLogs.Log(auditlog.Entry{
|
||
OperatorID: operatorID,
|
||
TenantID: &tenantID,
|
||
Module: "brand",
|
||
Action: "question_expansion.materialize",
|
||
ResourceType: &resourceType,
|
||
ResourceID: &brandID,
|
||
RequestID: nilIfEmptyString(requestID),
|
||
AfterJSON: afterJSON,
|
||
Result: &result,
|
||
})
|
||
}
|
||
|
||
func buildCandidate(text, source string, missingSuffix bool, existing map[string]struct{}, brandCtx *questionBrandContext) QuestionCandidate {
|
||
classified := classifyQuestionText(text, brandCtx.BrandName, brandCtx.CompetitorNames)
|
||
key := normalizeQuestionKey(text)
|
||
_, duplicate := existing[key]
|
||
tooShort := utf8.RuneCountInString(classified.Text) < 4
|
||
invalid := !questionLooksValid(classified.Text)
|
||
return QuestionCandidate{
|
||
Text: classified.Text,
|
||
Layer: classified.Layer,
|
||
Intent: classified.Intent,
|
||
Source: source,
|
||
MissingSuffix: missingSuffix,
|
||
TooShort: tooShort,
|
||
Duplicate: duplicate,
|
||
SuggestSkip: tooShort || invalid || duplicate,
|
||
}
|
||
}
|
||
|
||
func normalizeQuestionParts(values []string) []string {
|
||
result := make([]string, 0, len(values))
|
||
seen := map[string]struct{}{}
|
||
for _, value := range values {
|
||
trimmed := strings.TrimSpace(value)
|
||
if trimmed == "" {
|
||
continue
|
||
}
|
||
key := strings.ToLower(trimmed)
|
||
if _, ok := seen[key]; ok {
|
||
continue
|
||
}
|
||
seen[key] = struct{}{}
|
||
result = append(result, trimmed)
|
||
}
|
||
return result
|
||
}
|
||
|
||
func optionalQuestionParts(values []string) []string {
|
||
parts := normalizeQuestionParts(values)
|
||
if len(parts) == 0 {
|
||
return []string{""}
|
||
}
|
||
return parts
|
||
}
|
||
|
||
func buildQuestionCombinationFillFallback(brandCtx *questionBrandContext, regionTerms []string, req QuestionCombinationFillRequest) QuestionCombinationFillResult {
|
||
brandName := strings.TrimSpace(brandCtx.BrandName)
|
||
description := strings.TrimSpace(nilToString(brandCtx.Description))
|
||
category := inferQuestionCombinationCategory(brandName, description)
|
||
|
||
region := compactQuestionCombinationTerms(regionTerms, questionCombinationFillMax)
|
||
if len(region) == 0 {
|
||
region = compactQuestionCombinationTerms(req.Region, questionCombinationFillMax)
|
||
}
|
||
if len(region) == 0 {
|
||
region = []string{"全国", "本地", "华东"}
|
||
}
|
||
|
||
prefix := compactQuestionCombinationTerms(req.Prefix, questionCombinationFillMax)
|
||
if len(prefix) == 0 {
|
||
prefix = []string{"好用的", "专业的", "靠谱的", "热门的"}
|
||
}
|
||
|
||
core := compactQuestionCombinationTerms(req.Core, questionCombinationFillMax)
|
||
if len(core) == 0 {
|
||
core = questionCombinationCoreFallback(brandName, description)
|
||
}
|
||
|
||
industry := compactQuestionCombinationTerms(req.Industry, questionCombinationFillMax)
|
||
if len(industry) == 0 {
|
||
industry = []string{category, "平台", "工具", "服务商"}
|
||
}
|
||
|
||
suffix := compactQuestionCombinationTerms(req.Suffix, questionCombinationFillMax)
|
||
if len(suffix) == 0 {
|
||
suffix = []string{"哪家好", "怎么选", "推荐", "多少钱"}
|
||
}
|
||
|
||
return QuestionCombinationFillResult{
|
||
Region: compactQuestionCombinationTerms(region, questionCombinationFillMax),
|
||
Prefix: compactQuestionCombinationTerms(prefix, questionCombinationFillMax),
|
||
Core: compactQuestionCombinationTerms(core, questionCombinationFillMax),
|
||
Industry: compactQuestionCombinationTerms(industry, questionCombinationFillMax),
|
||
Suffix: compactQuestionCombinationTerms(suffix, questionCombinationFillMax),
|
||
}
|
||
}
|
||
|
||
func sanitizeQuestionCombinationFillResult(payload aiCombinationFillPayload, fallback QuestionCombinationFillResult) QuestionCombinationFillResult {
|
||
return QuestionCombinationFillResult{
|
||
Region: fillQuestionCombinationTerms(payload.Region, fallback.Region),
|
||
Prefix: fillQuestionCombinationTerms(payload.Prefix, fallback.Prefix),
|
||
Core: fillQuestionCombinationTerms(payload.Core, fallback.Core),
|
||
Industry: fillQuestionCombinationTerms(payload.Industry, fallback.Industry),
|
||
Suffix: fillQuestionCombinationTerms(payload.Suffix, fallback.Suffix),
|
||
}
|
||
}
|
||
|
||
func fillQuestionCombinationTerms(values, fallback []string) []string {
|
||
result := compactQuestionCombinationTerms(values, questionCombinationFillMax)
|
||
if len(result) >= 3 {
|
||
return result
|
||
}
|
||
for _, item := range fallback {
|
||
result = append(result, item)
|
||
result = compactQuestionCombinationTerms(result, questionCombinationFillMax)
|
||
if len(result) >= 3 {
|
||
break
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func compactQuestionCombinationTerms(values []string, limit int) []string {
|
||
if limit <= 0 {
|
||
limit = questionCombinationFillMax
|
||
}
|
||
result := make([]string, 0, minInt(len(values), limit))
|
||
seen := map[string]struct{}{}
|
||
for _, value := range values {
|
||
term := normalizeQuestionCombinationTerm(value)
|
||
if term == "" {
|
||
continue
|
||
}
|
||
key := strings.ToLower(term)
|
||
if _, ok := seen[key]; ok {
|
||
continue
|
||
}
|
||
seen[key] = struct{}{}
|
||
result = append(result, term)
|
||
if len(result) >= limit {
|
||
break
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func normalizeQuestionCombinationTerm(value string) string {
|
||
term := strings.TrimSpace(value)
|
||
term = strings.Trim(term, "，,。！？!?;；、 \t\r\n")
|
||
term = strings.ReplaceAll(term, " ", "")
|
||
term = strings.ReplaceAll(term, "\u3000", "")
|
||
if term == "" || term == "0" || strings.EqualFold(term, "null") {
|
||
return ""
|
||
}
|
||
if utf8.RuneCountInString(term) > 18 {
|
||
runes := []rune(term)
|
||
term = string(runes[:18])
|
||
}
|
||
return term
|
||
}
|
||
|
||
func questionCombinationRegionTermsFromIPRegion(region string) []string {
|
||
parts := strings.Split(strings.TrimSpace(region), "|")
|
||
if len(parts) < 4 {
|
||
return nil
|
||
}
|
||
country := normalizeChineseLocationTerm(parts[0])
|
||
province := normalizeChineseLocationTerm(parts[2])
|
||
city := normalizeChineseLocationTerm(parts[3])
|
||
if country != "" && country != "中国" {
|
||
return []string{country}
|
||
}
|
||
capital := provinceCapitalTerm(province)
|
||
area := chineseMacroRegion(province)
|
||
return compactQuestionCombinationTerms([]string{city, capital, area}, 3)
|
||
}
|
||
|
||
func normalizeChineseLocationTerm(value string) string {
|
||
term := normalizeQuestionCombinationTerm(value)
|
||
if term == "" || term == "本机" || term == "内网IP" {
|
||
return ""
|
||
}
|
||
replacer := strings.NewReplacer(
|
||
"特别行政区", "",
|
||
"维吾尔自治区", "",
|
||
"壮族自治区", "",
|
||
"回族自治区", "",
|
||
"自治区", "",
|
||
"省", "",
|
||
"市", "",
|
||
"地区", "",
|
||
"盟", "",
|
||
)
|
||
return replacer.Replace(term)
|
||
}
|
||
|
||
func provinceCapitalTerm(province string) string {
|
||
switch province {
|
||
case "北京":
|
||
return "北京"
|
||
case "上海":
|
||
return "上海"
|
||
case "天津":
|
||
return "天津"
|
||
case "重庆":
|
||
return "重庆"
|
||
case "河北":
|
||
return "石家庄"
|
||
case "山西":
|
||
return "太原"
|
||
case "内蒙古":
|
||
return "呼和浩特"
|
||
case "辽宁":
|
||
return "沈阳"
|
||
case "吉林":
|
||
return "长春"
|
||
case "黑龙江":
|
||
return "哈尔滨"
|
||
case "江苏":
|
||
return "南京"
|
||
case "浙江":
|
||
return "杭州"
|
||
case "安徽":
|
||
return "合肥"
|
||
case "福建":
|
||
return "福州"
|
||
case "江西":
|
||
return "南昌"
|
||
case "山东":
|
||
return "济南"
|
||
case "河南":
|
||
return "郑州"
|
||
case "湖北":
|
||
return "武汉"
|
||
case "湖南":
|
||
return "长沙"
|
||
case "广东":
|
||
return "广州"
|
||
case "广西":
|
||
return "南宁"
|
||
case "海南":
|
||
return "海口"
|
||
case "四川":
|
||
return "成都"
|
||
case "贵州":
|
||
return "贵阳"
|
||
case "云南":
|
||
return "昆明"
|
||
case "西藏":
|
||
return "拉萨"
|
||
case "陕西":
|
||
return "西安"
|
||
case "甘肃":
|
||
return "兰州"
|
||
case "青海":
|
||
return "西宁"
|
||
case "宁夏":
|
||
return "银川"
|
||
case "新疆":
|
||
return "乌鲁木齐"
|
||
case "香港":
|
||
return "香港"
|
||
case "澳门":
|
||
return "澳门"
|
||
case "台湾":
|
||
return "台北"
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
func chineseMacroRegion(province string) string {
|
||
switch province {
|
||
case "上海", "江苏", "浙江", "安徽", "福建", "江西", "山东", "台湾":
|
||
return "华东"
|
||
case "北京", "天津", "河北", "山西", "内蒙古":
|
||
return "华北"
|
||
case "河南", "湖北", "湖南":
|
||
return "华中"
|
||
case "广东", "广西", "海南", "香港", "澳门":
|
||
return "华南"
|
||
case "重庆", "四川", "贵州", "云南", "西藏":
|
||
return "西南"
|
||
case "陕西", "甘肃", "青海", "宁夏", "新疆":
|
||
return "西北"
|
||
case "辽宁", "吉林", "黑龙江":
|
||
return "东北"
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
func questionCombinationCoreFallback(brandName, description string) []string {
|
||
text := strings.ToLower(strings.TrimSpace(brandName + " " + description))
|
||
if strings.Contains(text, "geo") || strings.Contains(text, "ai") || strings.Contains(text, "搜索") || strings.Contains(text, "排名") {
|
||
return []string{"GEO", "AI搜索优化", "AI搜索排名", "品牌可见度"}
|
||
}
|
||
if strings.Contains(text, "教育") || strings.Contains(text, "培训") || strings.Contains(text, "课程") {
|
||
return []string{"课程", "培训", "学习", "机构"}
|
||
}
|
||
if strings.Contains(text, "医疗") || strings.Contains(text, "健康") || strings.Contains(text, "诊所") {
|
||
return []string{"医疗服务", "健康管理", "医生", "诊疗"}
|
||
}
|
||
if strings.Contains(text, "家居") || strings.Contains(text, "装修") || strings.Contains(text, "建材") {
|
||
return []string{"装修", "家居", "建材", "设计"}
|
||
}
|
||
return []string{strings.TrimSpace(brandName), "解决方案", "服务", "平台"}
|
||
}
|
||
|
||
func inferQuestionCombinationCategory(brandName, description string) string {
|
||
text := strings.ToLower(strings.TrimSpace(brandName + " " + description))
|
||
switch {
|
||
case strings.Contains(text, "geo") || strings.Contains(text, "seo") || strings.Contains(text, "搜索"):
|
||
return "营销"
|
||
case strings.Contains(text, "教育") || strings.Contains(text, "培训") || strings.Contains(text, "课程"):
|
||
return "教育"
|
||
case strings.Contains(text, "医疗") || strings.Contains(text, "健康"):
|
||
return "健康"
|
||
case strings.Contains(text, "家居") || strings.Contains(text, "装修") || strings.Contains(text, "建材"):
|
||
return "家居"
|
||
case strings.Contains(text, "餐饮") || strings.Contains(text, "美食"):
|
||
return "餐饮"
|
||
case strings.Contains(text, "旅游") || strings.Contains(text, "酒店"):
|
||
return "旅游"
|
||
default:
|
||
return "服务"
|
||
}
|
||
}
|
||
|
||
func questionCombinationFillCacheKey(tenantID, brandID int64, brandCtx *questionBrandContext, regionTerms []string, req QuestionCombinationFillRequest) string {
|
||
raw, _ := json.Marshal(struct {
|
||
TenantID int64 `json:"tenant_id"`
|
||
BrandID int64 `json:"brand_id"`
|
||
SchemaVersion string `json:"schema_version"`
|
||
BrandName string `json:"brand_name"`
|
||
Description string `json:"description"`
|
||
Competitors []string `json:"competitors"`
|
||
Region []string `json:"region"`
|
||
Request QuestionCombinationFillRequest `json:"request"`
|
||
}{
|
||
TenantID: tenantID,
|
||
BrandID: brandID,
|
||
SchemaVersion: "question_combination_fill_v1",
|
||
BrandName: strings.TrimSpace(brandCtx.BrandName),
|
||
Description: strings.TrimSpace(nilToString(brandCtx.Description)),
|
||
Competitors: brandCtx.CompetitorNames,
|
||
Region: regionTerms,
|
||
Request: QuestionCombinationFillRequest{
|
||
Region: compactQuestionCombinationTerms(req.Region, questionCombinationFillMax),
|
||
Prefix: compactQuestionCombinationTerms(req.Prefix, questionCombinationFillMax),
|
||
Core: compactQuestionCombinationTerms(req.Core, questionCombinationFillMax),
|
||
Industry: compactQuestionCombinationTerms(req.Industry, questionCombinationFillMax),
|
||
Suffix: compactQuestionCombinationTerms(req.Suffix, questionCombinationFillMax),
|
||
},
|
||
})
|
||
sum := sha1.Sum(raw)
|
||
return "question_combination_fill:" + hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
func nilToString(value *string) string {
|
||
if value == nil {
|
||
return ""
|
||
}
|
||
return *value
|
||
}
|
||
|
||
func nonEmptyParts(values ...string) []string {
|
||
result := make([]string, 0, len(values))
|
||
for _, value := range values {
|
||
if strings.TrimSpace(value) != "" {
|
||
result = append(result, strings.TrimSpace(value))
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func questionDistillCacheKey(tenantID, brandID int64, promptVersion, brandName string, competitors []string, seedTopic string) string {
|
||
raw, _ := json.Marshal(struct {
|
||
TenantID int64 `json:"tenant_id"`
|
||
BrandID int64 `json:"brand_id"`
|
||
PromptVersion string `json:"prompt_version"`
|
||
SchemaVersion string `json:"schema_version"`
|
||
BrandName string `json:"brand_name"`
|
||
Competitors []string `json:"competitors"`
|
||
SeedTopic string `json:"seed_topic"`
|
||
}{
|
||
TenantID: tenantID,
|
||
BrandID: brandID,
|
||
PromptVersion: promptVersion,
|
||
SchemaVersion: "question_distill_schema_v1",
|
||
BrandName: strings.TrimSpace(brandName),
|
||
Competitors: competitors,
|
||
SeedTopic: strings.TrimSpace(seedTopic),
|
||
})
|
||
sum := sha1.Sum(raw)
|
||
return "question_distill:" + hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
func skippedReasonDetail(skipped []SkipReason) string {
|
||
if len(skipped) == 0 {
|
||
return "no valid questions"
|
||
}
|
||
raw, err := json.Marshal(skipped)
|
||
if err != nil {
|
||
return "no valid questions"
|
||
}
|
||
return string(raw)
|
||
}
|
||
|
||
func isUniqueQuestionConstraintError(err error) bool {
|
||
var pgErr *pgconn.PgError
|
||
return errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "uk_brand_question_text_active"
|
||
}
|