geo/server/internal/tenant/app/question_expansion_service.go

package app

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgconn"
	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/geo-platform/tenant-api/internal/shared/auditlog"
	"github.com/geo-platform/tenant-api/internal/shared/auth"
	sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
	"github.com/geo-platform/tenant-api/internal/shared/ipregion"
	"github.com/geo-platform/tenant-api/internal/shared/llm"
	"github.com/geo-platform/tenant-api/internal/shared/middleware"
	"github.com/geo-platform/tenant-api/internal/shared/response"
)

const (
	questionCombinationMaxItems    = 500
	questionCombinationFillMax     = 4
	questionAIDistillMaxItems      = 20
	questionCombinationFillTTL     = 10 * time.Minute
	questionCombinationFillTimeout = 15 * time.Second
	questionDistillCacheTTL        = 5 * time.Minute
	questionDistillTimeout         = 30 * time.Second
)

type QuestionExpansionService struct {
	pool  *pgxpool.Pool
	llm   llm.Client
	brand *BrandService
	cache sharedcache.Cache
	ipGeo *ipregion.Resolver
}

func NewQuestionExpansionService(pool *pgxpool.Pool, llmClient llm.Client, brand *BrandService) *QuestionExpansionService {
	return &QuestionExpansionService{
		pool:  pool,
		llm:   llmClient,
		brand: brand,
	}
}

func (s *QuestionExpansionService) WithCache(c sharedcache.Cache) *QuestionExpansionService {
	s.cache = c
	return s
}

func (s *QuestionExpansionService) WithIPRegionResolver(resolver *ipregion.Resolver) *QuestionExpansionService {
	s.ipGeo = resolver
	return s
}

type QuestionCombinationRequest struct {
	Region   []string `json:"region"`
	Prefix   []string `json:"prefix"`
	Core     []string `json:"core" binding:"required"`
	Industry []string `json:"industry" binding:"required"`
	Suffix   []string `json:"suffix"`
	MaxItems int      `json:"max_items"`
}

type QuestionDistillRequest struct {
	SeedTopic string `json:"seed_topic" binding:"required,min=2"`
}

type QuestionCombinationFillRequest struct {
	Region   []string `json:"region"`
	Prefix   []string `json:"prefix"`
	Core     []string `json:"core"`
	Industry []string `json:"industry"`
	Suffix   []string `json:"suffix"`
}

type QuestionCombinationFillResult struct {
	Region          []string `json:"region"`
	Prefix          []string `json:"prefix"`
	Core            []string `json:"core"`
	Industry        []string `json:"industry"`
	Suffix          []string `json:"suffix"`
	AIPointsCharged int      `json:"ai_points_charged,omitempty"`
	CacheHit        bool     `json:"cache_hit,omitempty"`
}

type QuestionCandidate struct {
	Text          string `json:"text"`
	Layer         string `json:"layer"`
	Intent        string `json:"intent"`
	Source        string `json:"source"`
	MissingSuffix bool   `json:"missing_suffix"`
	TooShort      bool   `json:"too_short"`
	Duplicate     bool   `json:"duplicate"`
	SuggestSkip   bool   `json:"suggest_skip"`
}

type QuestionCandidateResult struct {
	Candidates      []QuestionCandidate `json:"candidates"`
	AIPointsCharged int                 `json:"ai_points_charged,omitempty"`
	CacheHit        bool                `json:"cache_hit,omitempty"`
	Truncated       bool                `json:"truncated,omitempty"`
}

type MaterializeQuestionsRequest struct {
	Source    string                `json:"source"`
	Questions []MaterializeQuestion `json:"questions" binding:"required,min=1"`
}

type MaterializeQuestion struct {
	Text string `json:"text" binding:"required"`
}

type MaterializeQuestionsResult struct {
	CreatedQuestions int          `json:"created_questions"`
	SkippedQuestions []SkipReason `json:"skipped_questions"`
}

type SkipReason struct {
	Text   string `json:"text"`
	Reason string `json:"reason"`
}

type questionBrandContext struct {
	TenantID        int64
	BrandID         int64
	BrandName       string
	Website         *string
	Description     *string
	PlanCode        string
	CompetitorNames []string
}

type aiDistillPayload struct {
	Candidates []struct {
		Text   string `json:"text"`
		Intent string `json:"intent"`
		Layer  string `json:"layer"`
	} `json:"candidates"`
}

type aiCombinationFillPayload struct {
	Region   []string `json:"region"`
	Prefix   []string `json:"prefix"`
	Core     []string `json:"core"`
	Industry []string `json:"industry"`
	Suffix   []string `json:"suffix"`
}

func (s *QuestionExpansionService) GenerateByCombination(ctx context.Context, brandID int64, req QuestionCombinationRequest) (*QuestionCandidateResult, error) {
	actor := auth.MustActor(ctx)
	brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}

	core := normalizeQuestionParts(req.Core)
	industry := normalizeQuestionParts(req.Industry)
	if len(core) == 0 {
		return nil, response.ErrBadRequest(40091, "invalid_params", "core_required")
	}
	if len(industry) == 0 {
		return nil, response.ErrBadRequest(40091, "invalid_params", "industry_required")
	}

	region := optionalQuestionParts(req.Region)
	prefix := optionalQuestionParts(req.Prefix)
	suffix := optionalQuestionParts(req.Suffix)
	limit := req.MaxItems
	if limit <= 0 || limit > questionCombinationMaxItems {
		limit = questionCombinationMaxItems
	}

	existing, err := s.loadExistingQuestionKeys(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}

	candidates := make([]QuestionCandidate, 0, minInt(limit, 64))
	seen := map[string]struct{}{}
	truncated := false

	for _, rg := range region {
		for _, pf := range prefix {
			for _, co := range core {
				for _, in := range industry {
					for _, sf := range suffix {
						text := normalizeQuestionText(strings.Join(nonEmptyParts(rg, pf, co, in, sf), ""))
						if text == "" {
							continue
						}
						key := normalizeQuestionKey(text)
						if _, ok := seen[key]; ok {
							continue
						}
						seen[key] = struct{}{}
						if len(candidates) >= limit {
							truncated = true
							continue
						}
						candidates = append(candidates, buildCandidate(text, QuestionSourceCombination, sf == "", existing, brandCtx))
					}
				}
			}
		}
	}

	return &QuestionCandidateResult{Candidates: candidates, Truncated: truncated}, nil
}

func (s *QuestionExpansionService) GenerateByAIDistill(ctx context.Context, brandID int64, req QuestionDistillRequest) (*QuestionCandidateResult, error) {
	actor := auth.MustActor(ctx)
	seedTopic := normalizeQuestionText(req.SeedTopic)
	if utf8.RuneCountInString(seedTopic) < 2 {
		return nil, response.ErrBadRequest(40091, "invalid_seed_topic", "seed_topic must contain at least 2 characters")
	}

	brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}

	existing, err := s.loadExistingQuestionKeys(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}

	prompt := buildQuestionDistillPrompt(brandCtx.BrandName, brandCtx.CompetitorNames, seedTopic)
	promptVersion := questionPromptHash(prompt)
	cacheKey := questionDistillCacheKey(actor.TenantID, brandID, promptVersion, brandCtx.BrandName, brandCtx.CompetitorNames, seedTopic)
	if s.cache != nil {
		if raw, cacheErr := s.cache.Get(ctx, cacheKey); cacheErr == nil && len(raw) > 0 {
			var cached []QuestionCandidate
			if json.Unmarshal(raw, &cached) == nil {
				return &QuestionCandidateResult{Candidates: cached, CacheHit: true}, nil
			}
		}
	}

	if s.llm == nil {
		return nil, response.ErrServiceUnavailable(50311, "llm_unavailable", "llm client is not configured")
	}
	if err := s.llm.Validate(); err != nil {
		return nil, response.ErrServiceUnavailable(50311, "llm_unavailable", err.Error())
	}

	resourceType := "question_distill"
	resourceUID := cacheKey
	reservation, err := ReserveAIPoints(ctx, s.pool, s.cache, AIPointReserveInput{
		TenantID:     actor.TenantID,
		OperatorID:   actor.UserID,
		UsageType:    AIUsageTypeQuestionDistill,
		ResourceType: &resourceType,
		ResourceUID:  &resourceUID,
		MeteredText:  seedTopic,
		FixedPoints:  1,
		Metadata: map[string]any{
			"brand_id":       brandID,
			"seed_topic":     seedTopic,
			"prompt_version": promptVersion,
		},
	})
	if err != nil {
		return nil, err
	}

	result, err := s.llm.Generate(ctx, llm.GenerateRequest{
		Prompt:          prompt,
		Timeout:         questionDistillTimeout,
		MaxOutputTokens: 1600,
		ResponseFormat: &llm.ResponseFormat{
			Type:        llm.ResponseFormatTypeJSONSchema,
			Name:        "question_distill_candidates",
			Description: "Question expansion candidates for a brand.",
			SchemaJSON:  questionDistillSchema,
			Strict:      true,
		},
	}, nil)
	if err != nil {
		_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
		if errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
			return nil, response.ErrServiceUnavailable(50312, "llm_timeout", "AI generation timed out")
		}
		return nil, response.ErrServiceUnavailable(50311, "llm_unavailable", err.Error())
	}

	var payload aiDistillPayload
	if err := json.Unmarshal([]byte(result.Content), &payload); err != nil {
		_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
		return nil, response.ErrBadRequest(50210, "llm_invalid_output", "AI output could not be parsed")
	}

	candidates := make([]QuestionCandidate, 0, minInt(len(payload.Candidates), questionAIDistillMaxItems))
	seen := map[string]struct{}{}
	for _, item := range payload.Candidates {
		if len(candidates) >= questionAIDistillMaxItems {
			break
		}
		text := normalizeQuestionText(item.Text)
		key := normalizeQuestionKey(text)
		if text == "" {
			continue
		}
		if questionContainsExcludedEntity(text, brandCtx.BrandName, brandCtx.CompetitorNames) {
			continue
		}
		if _, ok := seen[key]; ok {
			continue
		}
		seen[key] = struct{}{}
		candidates = append(candidates, buildCandidate(text, QuestionSourceAIDistill, false, existing, brandCtx))
	}

	if err := CompleteAIPoints(context.Background(), s.pool, actor.TenantID, *reservation, result.Model); err != nil {
		return nil, response.ErrInternal(50127, "ai_points_commit_failed", "failed to confirm ai points")
	}

	if s.cache != nil {
		if raw, err := json.Marshal(candidates); err == nil {
			_ = s.cache.Set(context.Background(), cacheKey, raw, questionDistillCacheTTL)
		}
	}

	return &QuestionCandidateResult{
		Candidates:      candidates,
		AIPointsCharged: reservation.Points,
	}, nil
}

func (s *QuestionExpansionService) FillQuestionCombination(ctx context.Context, brandID int64, clientIP string, req QuestionCombinationFillRequest) (*QuestionCombinationFillResult, error) {
	actor := auth.MustActor(ctx)
	brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}

	regionTerms := questionCombinationRegionTermsFromIPRegion("")
	if s.ipGeo != nil {
		regionTerms = questionCombinationRegionTermsFromIPRegion(s.ipGeo.Lookup(clientIP))
	}
	fallback := buildQuestionCombinationFillFallback(brandCtx, regionTerms, req)
	prompt := buildQuestionCombinationFillPrompt(brandCtx, fallback.Region, req)
	promptVersion := questionPromptHash(prompt)

	cacheKey := questionCombinationFillCacheKey(actor.TenantID, brandID, promptVersion, brandCtx, regionTerms, req)
	if s.cache != nil {
		if raw, cacheErr := s.cache.Get(ctx, cacheKey); cacheErr == nil && len(raw) > 0 {
			var cached QuestionCombinationFillResult
			if json.Unmarshal(raw, &cached) == nil {
				cached.CacheHit = true
				cached.AIPointsCharged = 0
				return &cached, nil
			}
		}
	}

	if s.llm == nil || s.llm.Validate() != nil {
		return &fallback, nil
	}

	resourceType := "question_combination_fill"
	resourceUID := cacheKey
	meteredText, _ := json.Marshal(map[string]any{
		"brand":       brandCtx.BrandName,
		"description": nilToString(brandCtx.Description),
		"region":      fallback.Region,
		"request":     req,
	})
	reservation, err := ReserveAIPoints(ctx, s.pool, s.cache, AIPointReserveInput{
		TenantID:     actor.TenantID,
		OperatorID:   actor.UserID,
		UsageType:    AIUsageTypeQuestionCombinationFill,
		ResourceType: &resourceType,
		ResourceUID:  &resourceUID,
		MeteredText:  string(meteredText),
		FixedPoints:  1,
		Metadata: map[string]any{
			"brand_id":       brandID,
			"prompt_version": promptVersion,
		},
	})
	if err != nil {
		return nil, err
	}

	result, err := s.llm.Generate(ctx, llm.GenerateRequest{
		Prompt:          prompt,
		Timeout:         questionCombinationFillTimeout,
		MaxOutputTokens: 900,
		ResponseFormat: &llm.ResponseFormat{
			Type:        llm.ResponseFormatTypeJSONSchema,
			Name:        "question_combination_fill",
			Description: "Short word columns for question expansion combination tool.",
			SchemaJSON:  questionCombinationFillSchema,
			Strict:      true,
		},
	}, nil)
	if err != nil {
		_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
		return &fallback, nil
	}

	var payload aiCombinationFillPayload
	if err := json.Unmarshal([]byte(result.Content), &payload); err != nil {
		_ = RefundAIPoints(context.Background(), s.pool, s.cache, actor.TenantID, actor.UserID, *reservation, err.Error())
		return &fallback, nil
	}

	filled := sanitizeQuestionCombinationFillResult(payload, fallback)
	if err := CompleteAIPoints(context.Background(), s.pool, actor.TenantID, *reservation, result.Model); err != nil {
		return nil, response.ErrInternal(50127, "ai_points_commit_failed", "failed to confirm ai points")
	}
	filled.AIPointsCharged = reservation.Points
	if s.cache != nil {
		cached := filled
		cached.AIPointsCharged = 0
		cached.CacheHit = false
		if raw, err := json.Marshal(cached); err == nil {
			_ = s.cache.Set(context.Background(), cacheKey, raw, questionCombinationFillTTL)
		}
	}
	return &filled, nil
}

func (s *QuestionExpansionService) ClassifyMetadata(ctx context.Context, brandID int64, texts []string) ([]ClassifiedQuestion, error) {
	actor := auth.MustActor(ctx)
	brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}
	result := make([]ClassifiedQuestion, 0, len(texts))
	for _, text := range texts {
		trimmed := normalizeQuestionText(text)
		if trimmed == "" {
			continue
		}
		result = append(result, classifyQuestionText(trimmed, brandCtx.BrandName, brandCtx.CompetitorNames))
	}
	return result, nil
}

func (s *QuestionExpansionService) MaterializeQuestions(ctx context.Context, brandID int64, req MaterializeQuestionsRequest) (*MaterializeQuestionsResult, error) {
	actor := auth.MustActor(ctx)
	source := strings.TrimSpace(req.Source)
	if source == "" {
		source = QuestionSourceManual
	}
	if !isValidExternalQuestionSource(source) {
		return nil, response.ErrBadRequest(40092, "invalid_enum", "source is invalid")
	}
	if len(req.Questions) == 0 {
		return nil, response.ErrBadRequest(40093, "no_valid_questions", "questions is required")
	}

	brandCtx, err := s.loadQuestionBrandContext(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}
	existing, err := s.loadExistingQuestionKeys(ctx, actor.TenantID, brandID)
	if err != nil {
		return nil, err
	}
	currentCount, err := s.countTenantQuestions(ctx, actor.TenantID)
	if err != nil {
		return nil, err
	}

	maxQuestions := s.brand.currentLimits().QuestionLimitForPlan(brandCtx.PlanCode)
	remaining := maxQuestions - currentCount
	if remaining < 0 {
		remaining = 0
	}

	type acceptedQuestion struct {
		Text   string `json:"text"`
		Layer  string `json:"layer"`
		Intent string `json:"intent"`
		Source string `json:"source"`
	}

	accepted := make([]acceptedQuestion, 0, len(req.Questions))
	skipped := make([]SkipReason, 0)
	inBatch := map[string]struct{}{}

	for _, item := range req.Questions {
		text := normalizeQuestionText(item.Text)
		key := normalizeQuestionKey(text)
		if text == "" || utf8.RuneCountInString(text) < 4 {
			skipped = append(skipped, SkipReason{Text: text, Reason: "too_short"})
			continue
		}
		if !questionLooksValid(text) {
			skipped = append(skipped, SkipReason{Text: text, Reason: "invalid_text"})
			continue
		}
		if _, ok := existing[key]; ok {
			skipped = append(skipped, SkipReason{Text: text, Reason: "duplicate"})
			continue
		}
		if _, ok := inBatch[key]; ok {
			skipped = append(skipped, SkipReason{Text: text, Reason: "duplicate_in_batch"})
			continue
		}
		if remaining <= 0 {
			skipped = append(skipped, SkipReason{Text: text, Reason: "quota_exceeded"})
			continue
		}
		classified := classifyQuestionText(text, brandCtx.BrandName, brandCtx.CompetitorNames)
		accepted = append(accepted, acceptedQuestion{
			Text:   text,
			Layer:  classified.Layer,
			Intent: classified.Intent,
			Source: source,
		})
		inBatch[key] = struct{}{}
		remaining--
	}

	if len(accepted) == 0 {
		return nil, response.ErrBadRequest(40093, "no_valid_questions", skippedReasonDetail(skipped))
	}

	tx, err := s.pool.Begin(ctx)
	if err != nil {
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to begin transaction")
	}
	defer func() {
		_ = tx.Rollback(ctx)
	}()

	if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))`, tenantQuestionQuotaLockKey(actor.TenantID), "questions"); err != nil {
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to lock question quota")
	}

	bucketID, err := s.ensureDefaultQuestionBucket(ctx, tx, actor.TenantID, brandID, brandCtx.BrandName)
	if err != nil {
		return nil, err
	}

	currentCount, err = countTenantQuestionsTx(ctx, tx, actor.TenantID)
	if err != nil {
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to recount questions")
	}
	remaining = maxQuestions - currentCount
	if remaining < 0 {
		remaining = 0
	}
	if remaining < len(accepted) {
		overflow := accepted[remaining:]
		accepted = accepted[:remaining]
		for _, q := range overflow {
			skipped = append(skipped, SkipReason{Text: q.Text, Reason: "quota_exceeded_concurrent"})
		}
	}
	if len(accepted) == 0 {
		return nil, response.ErrBadRequest(40093, "no_valid_questions", skippedReasonDetail(skipped))
	}

	payload, err := json.Marshal(accepted)
	if err != nil {
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to encode questions")
	}
	rows, err := tx.Query(ctx, `
		INSERT INTO brand_questions (
			tenant_id, brand_id, keyword_id, question_text,
			layer, intent, source, status
		)
		SELECT $1, $2, $3, q.text, q.layer, q.intent, q.source, 'active'
		FROM jsonb_to_recordset($4::jsonb)
			AS q(text TEXT, layer TEXT, intent TEXT, source TEXT)
		ON CONFLICT (tenant_id, brand_id, lower(btrim(question_text)))
			WHERE deleted_at IS NULL
		DO NOTHING
		RETURNING question_text
	`, actor.TenantID, brandID, bucketID, payload)
	if err != nil {
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to insert questions")
	}
	inserted := map[string]struct{}{}
	for rows.Next() {
		var text string
		if err := rows.Scan(&text); err != nil {
			rows.Close()
			return nil, response.ErrInternal(50010, "materialize_failed", "failed to scan inserted questions")
		}
		inserted[normalizeQuestionKey(text)] = struct{}{}
	}
	if err := rows.Err(); err != nil {
		rows.Close()
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to iterate inserted questions")
	}
	rows.Close()

	for _, q := range accepted {
		if _, ok := inserted[normalizeQuestionKey(q.Text)]; !ok {
			skipped = append(skipped, SkipReason{Text: q.Text, Reason: "duplicate_concurrent"})
		}
	}

	if err := tx.Commit(ctx); err != nil {
		return nil, response.ErrInternal(50010, "materialize_failed", "failed to commit questions")
	}

	invalidateBrandCaches(ctx, s.cache, actor.TenantID, brandID)
	s.logQuestionExpansionAudit(ctx, actor.UserID, actor.TenantID, brandID, source, len(inserted), skipped)
	return &MaterializeQuestionsResult{
		CreatedQuestions: len(inserted),
		SkippedQuestions: skipped,
	}, nil
}

func (s *QuestionExpansionService) loadQuestionBrandContext(ctx context.Context, tenantID, brandID int64) (*questionBrandContext, error) {
	var brandName string
	var website *string
	var description *string
	err := s.pool.QueryRow(ctx, `
		SELECT name, website, description
		FROM brands
		WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
	`, brandID, tenantID).Scan(&brandName, &website, &description)
	if err != nil {
		if errors.Is(err, pgx.ErrNoRows) {
			return nil, response.ErrNotFound(40420, "brand_not_found", "brand not found")
		}
		return nil, response.ErrInternal(50010, "query_failed", "failed to load brand")
	}

	rows, err := s.pool.Query(ctx, `
		SELECT name
		FROM competitors
		WHERE brand_id = $1 AND tenant_id = $2 AND deleted_at IS NULL
		ORDER BY id ASC
	`, brandID, tenantID)
	if err != nil {
		return nil, response.ErrInternal(50010, "query_failed", "failed to load competitors")
	}
	defer rows.Close()
	competitors := make([]string, 0)
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, response.ErrInternal(50010, "scan_failed", err.Error())
		}
		competitors = append(competitors, name)
	}

	plan, err := s.brand.loadBrandLibraryPlan(ctx, tenantID)
	if err != nil {
		return nil, err
	}

	return &questionBrandContext{
		TenantID:        tenantID,
		BrandID:         brandID,
		BrandName:       brandName,
		Website:         website,
		Description:     description,
		PlanCode:        plan.PlanCode,
		CompetitorNames: competitors,
	}, nil
}

func (s *QuestionExpansionService) loadExistingQuestionKeys(ctx context.Context, tenantID, brandID int64) (map[string]struct{}, error) {
	rows, err := s.pool.Query(ctx, `
		SELECT lower(btrim(question_text))
		FROM brand_questions
		WHERE tenant_id = $1 AND brand_id = $2 AND deleted_at IS NULL
	`, tenantID, brandID)
	if err != nil {
		return nil, response.ErrInternal(50010, "query_failed", "failed to load existing questions")
	}
	defer rows.Close()

	result := map[string]struct{}{}
	for rows.Next() {
		var key string
		if err := rows.Scan(&key); err != nil {
			return nil, response.ErrInternal(50010, "scan_failed", err.Error())
		}
		result[key] = struct{}{}
	}
	return result, nil
}

func (s *QuestionExpansionService) countTenantQuestions(ctx context.Context, tenantID int64) (int, error) {
	return countTenantQuestionsTx(ctx, s.pool, tenantID)
}

func countTenantQuestionsTx(ctx context.Context, db interface {
	QueryRow(context.Context, string, ...any) pgx.Row
}, tenantID int64) (int, error) {
	var count int
	err := db.QueryRow(ctx, `
		SELECT COUNT(*)::INT
		FROM brand_questions
		WHERE tenant_id = $1 AND deleted_at IS NULL
	`, tenantID).Scan(&count)
	return count, err
}

func (s *QuestionExpansionService) ensureDefaultQuestionBucket(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, brandName string) (int64, error) {
	return ensureDefaultQuestionBucketTx(ctx, tx, tenantID, brandID, brandName)
}

func (s *QuestionExpansionService) logQuestionExpansionAudit(ctx context.Context, operatorID, tenantID, brandID int64, source string, created int, skipped []SkipReason) {
	if s == nil || s.brand == nil || s.brand.auditLogs == nil {
		return
	}
	afterJSON, _ := json.Marshal(map[string]interface{}{
		"brand_id":          brandID,
		"source":            source,
		"created_questions": created,
		"skipped_count":     len(skipped),
	})
	result := "success"
	resourceType := "brand"
	requestID := middleware.RequestIDFromContext(ctx)
	s.brand.auditLogs.Log(auditlog.Entry{
		OperatorID:   operatorID,
		TenantID:     &tenantID,
		Module:       "brand",
		Action:       "question_expansion.materialize",
		ResourceType: &resourceType,
		ResourceID:   &brandID,
		RequestID:    nilIfEmptyString(requestID),
		AfterJSON:    afterJSON,
		Result:       &result,
	})
}

func buildCandidate(text, source string, missingSuffix bool, existing map[string]struct{}, brandCtx *questionBrandContext) QuestionCandidate {
	classified := classifyQuestionText(text, brandCtx.BrandName, brandCtx.CompetitorNames)
	key := normalizeQuestionKey(text)
	_, duplicate := existing[key]
	tooShort := utf8.RuneCountInString(classified.Text) < 4
	invalid := !questionLooksValid(classified.Text)
	return QuestionCandidate{
		Text:          classified.Text,
		Layer:         classified.Layer,
		Intent:        classified.Intent,
		Source:        source,
		MissingSuffix: missingSuffix,
		TooShort:      tooShort,
		Duplicate:     duplicate,
		SuggestSkip:   tooShort || invalid || duplicate,
	}
}

func questionContainsExcludedEntity(text, brandName string, competitorNames []string) bool {
	normalized := normalizeQuestionKey(text)
	for _, entity := range append([]string{brandName}, competitorNames...) {
		for _, entityKey := range questionEntityExclusionKeys(entity) {
			if strings.Contains(normalized, entityKey) {
				return true
			}
		}
	}
	return false
}

func questionEntityExclusionKeys(entity string) []string {
	base := normalizeQuestionKey(entity)
	if utf8.RuneCountInString(base) < 2 {
		return nil
	}
	keys := make([]string, 0, 4)
	addKey := func(value string) {
		value = strings.TrimSpace(value)
		if utf8.RuneCountInString(value) < 2 {
			return
		}
		for _, key := range keys {
			if key == value {
				return
			}
		}
		keys = append(keys, value)
	}
	addKey(base)

	companyName := base
	for _, suffix := range []string{
		"有限责任公司",
		"股份有限公司",
		"集团有限公司",
		"控股有限公司",
		"科技有限公司",
		"销售有限公司",
		"用品有限公司",
		"有限公司",
		"股份公司",
		"集团公司",
		"公司",
	} {
		if strings.HasSuffix(companyName, suffix) {
			companyName = strings.TrimSuffix(companyName, suffix)
			addKey(companyName)
			break
		}
	}

	descriptorName := companyName
	for _, suffix := range []string{
		"用品销售",
		"产品销售",
		"销售",
		"用品",
		"产品",
		"服务",
		"科技",
		"商贸",
		"贸易",
		"网络",
		"信息",
		"文化",
		"传媒",
	} {
		if strings.HasSuffix(descriptorName, suffix) {
			descriptorName = strings.TrimSuffix(descriptorName, suffix)
			addKey(descriptorName)
			break
		}
	}
	return keys
}

func normalizeQuestionParts(values []string) []string {
	result := make([]string, 0, len(values))
	seen := map[string]struct{}{}
	for _, value := range values {
		trimmed := normalizeQuestionText(value)
		if trimmed == "" {
			continue
		}
		key := strings.ToLower(trimmed)
		if _, ok := seen[key]; ok {
			continue
		}
		seen[key] = struct{}{}
		result = append(result, trimmed)
	}
	return result
}

func optionalQuestionParts(values []string) []string {
	parts := normalizeQuestionParts(values)
	if len(parts) == 0 {
		return []string{""}
	}
	return parts
}

func buildQuestionCombinationFillFallback(brandCtx *questionBrandContext, regionTerms []string, req QuestionCombinationFillRequest) QuestionCombinationFillResult {
	brandName := strings.TrimSpace(brandCtx.BrandName)
	description := strings.TrimSpace(nilToString(brandCtx.Description))
	category := inferQuestionCombinationCategory(brandName, description)

	region := compactQuestionCombinationTerms(regionTerms, questionCombinationFillMax)
	if len(region) == 0 {
		region = compactQuestionCombinationTerms(req.Region, questionCombinationFillMax)
	}
	if len(region) == 0 {
		region = []string{"全国", "合肥", "华东"}
	}

	prefix := compactQuestionCombinationTerms(req.Prefix, questionCombinationFillMax)
	if len(prefix) == 0 {
		prefix = []string{"性价比高的", "口碑好的", "专业的", "靠谱的"}
	}

	core := compactQuestionCombinationTerms(req.Core, questionCombinationFillMax)
	if len(core) == 0 {
		core = questionCombinationPrecisionFallback(brandName, description)
	}

	industry := compactQuestionCombinationTerms(req.Industry, questionCombinationFillMax)
	if len(industry) == 0 {
		industry = questionCombinationIndustryFallback(category)
	}

	suffix := compactQuestionCombinationTerms(req.Suffix, questionCombinationFillMax)
	if len(suffix) == 0 {
		suffix = []string{"哪家好", "怎么选", "推荐", "避坑"}
	}

	return QuestionCombinationFillResult{
		Region:   compactQuestionCombinationTerms(region, questionCombinationFillMax),
		Prefix:   compactQuestionCombinationTerms(prefix, questionCombinationFillMax),
		Core:     compactQuestionCombinationTerms(core, questionCombinationFillMax),
		Industry: compactQuestionCombinationTerms(industry, questionCombinationFillMax),
		Suffix:   compactQuestionCombinationTerms(suffix, questionCombinationFillMax),
	}
}

func sanitizeQuestionCombinationFillResult(payload aiCombinationFillPayload, fallback QuestionCombinationFillResult) QuestionCombinationFillResult {
	return QuestionCombinationFillResult{
		Region:   fillQuestionCombinationTerms(payload.Region, fallback.Region),
		Prefix:   fillQuestionCombinationTerms(payload.Prefix, fallback.Prefix),
		Core:     fillQuestionCombinationTerms(payload.Core, fallback.Core),
		Industry: fillQuestionCombinationTerms(payload.Industry, fallback.Industry),
		Suffix:   fillQuestionCombinationTerms(payload.Suffix, fallback.Suffix),
	}
}

func fillQuestionCombinationTerms(values, fallback []string) []string {
	result := compactQuestionCombinationTerms(values, questionCombinationFillMax)
	if len(result) >= 3 {
		return result
	}
	for _, item := range fallback {
		result = append(result, item)
		result = compactQuestionCombinationTerms(result, questionCombinationFillMax)
		if len(result) >= 3 {
			break
		}
	}
	return result
}

func compactQuestionCombinationTerms(values []string, limit int) []string {
	if limit <= 0 {
		limit = questionCombinationFillMax
	}
	result := make([]string, 0, minInt(len(values), limit))
	seen := map[string]struct{}{}
	for _, value := range values {
		term := normalizeQuestionCombinationTerm(value)
		if term == "" {
			continue
		}
		key := strings.ToLower(term)
		if _, ok := seen[key]; ok {
			continue
		}
		seen[key] = struct{}{}
		result = append(result, term)
		if len(result) >= limit {
			break
		}
	}
	return result
}

func normalizeQuestionCombinationTerm(value string) string {
	term := strings.TrimSpace(value)
	term = strings.Trim(term, "，,。！？!?;；、 \t\r\n")
	term = strings.ReplaceAll(term, " ", "")
	term = strings.ReplaceAll(term, "\u3000", "")
	if term == "" || term == "0" || strings.EqualFold(term, "null") {
		return ""
	}
	if utf8.RuneCountInString(term) > 18 {
		runes := []rune(term)
		term = string(runes[:18])
	}
	return term
}

func questionCombinationRegionTermsFromIPRegion(region string) []string {
	parts := strings.Split(strings.TrimSpace(region), "|")
	if len(parts) < 4 {
		return nil
	}
	country := normalizeChineseLocationTerm(parts[0])
	province := normalizeChineseLocationTerm(parts[2])
	city := normalizeChineseLocationTerm(parts[3])
	if country != "" && country != "中国" {
		return []string{country}
	}
	capital := provinceCapitalTerm(province)
	area := chineseMacroRegion(province)
	return compactQuestionCombinationTerms([]string{city, capital, area}, 3)
}

func normalizeChineseLocationTerm(value string) string {
	term := normalizeQuestionCombinationTerm(value)
	if term == "" || term == "本机" || term == "内网IP" {
		return ""
	}
	replacer := strings.NewReplacer(
		"特别行政区", "",
		"维吾尔自治区", "",
		"壮族自治区", "",
		"回族自治区", "",
		"自治区", "",
		"省", "",
		"市", "",
		"地区", "",
		"盟", "",
	)
	return replacer.Replace(term)
}

func provinceCapitalTerm(province string) string {
	switch province {
	case "北京":
		return "北京"
	case "上海":
		return "上海"
	case "天津":
		return "天津"
	case "重庆":
		return "重庆"
	case "河北":
		return "石家庄"
	case "山西":
		return "太原"
	case "内蒙古":
		return "呼和浩特"
	case "辽宁":
		return "沈阳"
	case "吉林":
		return "长春"
	case "黑龙江":
		return "哈尔滨"
	case "江苏":
		return "南京"
	case "浙江":
		return "杭州"
	case "安徽":
		return "合肥"
	case "福建":
		return "福州"
	case "江西":
		return "南昌"
	case "山东":
		return "济南"
	case "河南":
		return "郑州"
	case "湖北":
		return "武汉"
	case "湖南":
		return "长沙"
	case "广东":
		return "广州"
	case "广西":
		return "南宁"
	case "海南":
		return "海口"
	case "四川":
		return "成都"
	case "贵州":
		return "贵阳"
	case "云南":
		return "昆明"
	case "西藏":
		return "拉萨"
	case "陕西":
		return "西安"
	case "甘肃":
		return "兰州"
	case "青海":
		return "西宁"
	case "宁夏":
		return "银川"
	case "新疆":
		return "乌鲁木齐"
	case "香港":
		return "香港"
	case "澳门":
		return "澳门"
	case "台湾":
		return "台北"
	default:
		return ""
	}
}

func chineseMacroRegion(province string) string {
	switch province {
	case "上海", "江苏", "浙江", "安徽", "福建", "江西", "山东", "台湾":
		return "华东"
	case "北京", "天津", "河北", "山西", "内蒙古":
		return "华北"
	case "河南", "湖北", "湖南":
		return "华中"
	case "广东", "广西", "海南", "香港", "澳门":
		return "华南"
	case "重庆", "四川", "贵州", "云南", "西藏":
		return "西南"
	case "陕西", "甘肃", "青海", "宁夏", "新疆":
		return "西北"
	case "辽宁", "吉林", "黑龙江":
		return "东北"
	default:
		return ""
	}
}

func questionCombinationPrecisionFallback(brandName, description string) []string {
	text := strings.ToLower(strings.TrimSpace(brandName + " " + description))
	if strings.Contains(text, "geo") || strings.Contains(text, "ai") || strings.Contains(text, "搜索") || strings.Contains(text, "排名") {
		return []string{"品牌词", "AI搜索", "获客", "中小企业"}
	}
	if strings.Contains(text, "教育") || strings.Contains(text, "培训") || strings.Contains(text, "课程") {
		return []string{"成人", "少儿", "线上", "入门"}
	}
	if strings.Contains(text, "医疗") || strings.Contains(text, "健康") || strings.Contains(text, "诊所") {
		return []string{"本地", "复诊", "儿童", "上班族"}
	}
	if strings.Contains(text, "家居") || strings.Contains(text, "装修") || strings.Contains(text, "建材") {
		return []string{"精装", "100平", "小户型", "旧房改造"}
	}
	return []string{"企业", "本地", "新手", "中小企业"}
}

func questionCombinationIndustryFallback(category string) []string {
	switch category {
	case "营销":
		return []string{"GEO优化", "AI搜索优化", "品牌推广", "搜索营销"}
	case "教育":
		return []string{"培训机构", "在线课程", "教育机构", "学习平台"}
	case "健康":
		return []string{"健康管理", "医疗服务", "诊疗机构", "医生服务"}
	case "家居":
		return []string{"全屋定制", "家具定制", "家居定制", "定制家装"}
	case "餐饮":
		return []string{"餐饮店", "美食店", "连锁餐饮", "餐饮服务"}
	case "旅游":
		return []string{"旅游服务", "酒店预订", "旅行社", "旅游攻略"}
	default:
		return []string{"服务商", "解决方案", "平台", "公司"}
	}
}

func inferQuestionCombinationCategory(brandName, description string) string {
	text := strings.ToLower(strings.TrimSpace(brandName + " " + description))
	switch {
	case strings.Contains(text, "geo") || strings.Contains(text, "seo") || strings.Contains(text, "搜索"):
		return "营销"
	case strings.Contains(text, "教育") || strings.Contains(text, "培训") || strings.Contains(text, "课程"):
		return "教育"
	case strings.Contains(text, "医疗") || strings.Contains(text, "健康"):
		return "健康"
	case strings.Contains(text, "家居") || strings.Contains(text, "装修") || strings.Contains(text, "建材"):
		return "家居"
	case strings.Contains(text, "餐饮") || strings.Contains(text, "美食"):
		return "餐饮"
	case strings.Contains(text, "旅游") || strings.Contains(text, "酒店"):
		return "旅游"
	default:
		return "服务"
	}
}

func questionCombinationFillCacheKey(tenantID, brandID int64, promptVersion string, brandCtx *questionBrandContext, regionTerms []string, req QuestionCombinationFillRequest) string {
	raw, _ := json.Marshal(struct {
		TenantID      int64                          `json:"tenant_id"`
		BrandID       int64                          `json:"brand_id"`
		PromptVersion string                         `json:"prompt_version"`
		SchemaVersion string                         `json:"schema_version"`
		BrandName     string                         `json:"brand_name"`
		Description   string                         `json:"description"`
		Competitors   []string                       `json:"competitors"`
		Region        []string                       `json:"region"`
		Request       QuestionCombinationFillRequest `json:"request"`
	}{
		TenantID:      tenantID,
		BrandID:       brandID,
		PromptVersion: promptVersion,
		SchemaVersion: "question_combination_fill_v1",
		BrandName:     strings.TrimSpace(brandCtx.BrandName),
		Description:   strings.TrimSpace(nilToString(brandCtx.Description)),
		Competitors:   brandCtx.CompetitorNames,
		Region:        regionTerms,
		Request: QuestionCombinationFillRequest{
			Region:   compactQuestionCombinationTerms(req.Region, questionCombinationFillMax),
			Prefix:   compactQuestionCombinationTerms(req.Prefix, questionCombinationFillMax),
			Core:     compactQuestionCombinationTerms(req.Core, questionCombinationFillMax),
			Industry: compactQuestionCombinationTerms(req.Industry, questionCombinationFillMax),
			Suffix:   compactQuestionCombinationTerms(req.Suffix, questionCombinationFillMax),
		},
	})
	sum := sha1.Sum(raw)
	return "question_combination_fill:" + hex.EncodeToString(sum[:])
}

func questionPromptHash(prompt string) string {
	sum := sha1.Sum([]byte(strings.TrimSpace(prompt)))
	return hex.EncodeToString(sum[:])
}

func nilToString(value *string) string {
	if value == nil {
		return ""
	}
	return *value
}

func nonEmptyParts(values ...string) []string {
	result := make([]string, 0, len(values))
	for _, value := range values {
		if strings.TrimSpace(value) != "" {
			result = append(result, strings.TrimSpace(value))
		}
	}
	return result
}

func questionDistillCacheKey(tenantID, brandID int64, promptVersion, brandName string, competitors []string, seedTopic string) string {
	raw, _ := json.Marshal(struct {
		TenantID      int64    `json:"tenant_id"`
		BrandID       int64    `json:"brand_id"`
		PromptVersion string   `json:"prompt_version"`
		SchemaVersion string   `json:"schema_version"`
		BrandName     string   `json:"brand_name"`
		Competitors   []string `json:"competitors"`
		SeedTopic     string   `json:"seed_topic"`
	}{
		TenantID:      tenantID,
		BrandID:       brandID,
		PromptVersion: promptVersion,
		SchemaVersion: "question_distill_schema_v1",
		BrandName:     strings.TrimSpace(brandName),
		Competitors:   competitors,
		SeedTopic:     strings.TrimSpace(seedTopic),
	})
	sum := sha1.Sum(raw)
	return "question_distill:" + hex.EncodeToString(sum[:])
}

func skippedReasonDetail(skipped []SkipReason) string {
	if len(skipped) == 0 {
		return "no valid questions"
	}
	raw, err := json.Marshal(skipped)
	if err != nil {
		return "no valid questions"
	}
	return string(raw)
}

func isUniqueQuestionConstraintError(err error) bool {
	var pgErr *pgconn.PgError
	return errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "uk_brand_question_text_active"
}