geo/server/internal/tenant/app/knowledge_service.go

package app

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime/debug"
	"slices"
	"strings"
	"sync"
	"time"

	"github.com/google/uuid"
	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"
	"go.uber.org/zap"

	"github.com/geo-platform/tenant-api/internal/shared/auth"
	"github.com/geo-platform/tenant-api/internal/shared/config"
	"github.com/geo-platform/tenant-api/internal/shared/llm"
	"github.com/geo-platform/tenant-api/internal/shared/objectstorage"
	"github.com/geo-platform/tenant-api/internal/shared/response"
	"github.com/geo-platform/tenant-api/internal/shared/retrieval"
	"github.com/geo-platform/tenant-api/internal/tenant/prompts"
)

const (
	defaultKnowledgeQueueSize      = 128
	defaultKnowledgeCleanupPoll    = 15 * time.Second
	defaultKnowledgeChildGroupName = "默认目录"
)

type KnowledgeService struct {
	pool               *pgxpool.Pool
	provider           retrieval.Provider
	store              retrieval.VectorStore
	objectStorage      objectstorage.Client
	llmClient          llm.Client
	logger             *zap.Logger
	chunkSize          int
	chunkOverlap       int
	embeddingBatchSize int
	recallLimit        int
	rerankTopN         int
	arkBaseURL         string
	arkAPIKey          string
	urlMarkdownModel   string
	httpClient         *http.Client
	jobs               chan knowledgeParseJob
	cleanupMu          sync.Mutex
	cleanupInFlight    map[string]struct{}
}

type knowledgeParseJob struct {
	TenantID    int64
	ItemID      int64
	ParseTaskID int64
}

type knowledgeCleanupJob struct {
	TenantID int64
	ItemID   int64
}

type knowledgeItemRecord struct {
	ID          int64
	TenantID    int64
	GroupID     int64
	GroupName   string
	Name        string
	SourceType  string
	SourceURI   *string
	StorageKey  string
	ContentText *string
	Markdown    *string
	ItemVersion int
}

type KnowledgeGroupRequest struct {
	Name      string `json:"name" binding:"required"`
	ParentID  *int64 `json:"parent_id"`
	SortOrder *int   `json:"sort_order"`
}

type KnowledgeGroupResponse struct {
	ID        int64                    `json:"id"`
	Name      string                   `json:"name"`
	ParentID  *int64                   `json:"parent_id"`
	SortOrder int                      `json:"sort_order"`
	ItemCount int64                    `json:"item_count"`
	CreatedAt string                   `json:"created_at"`
	Children  []KnowledgeGroupResponse `json:"children"`
}

type KnowledgeItemResponse struct {
	ID           int64   `json:"id"`
	GroupID      int64   `json:"group_id"`
	GroupName    string  `json:"group_name"`
	Name         string  `json:"name"`
	SourceType   string  `json:"source_type"`
	SourceURI    *string `json:"source_uri"`
	Status       string  `json:"status"`
	SizeBytes    int64   `json:"size_bytes"`
	ErrorMessage *string `json:"error_message"`
	CreatedAt    string  `json:"created_at"`
	UpdatedAt    string  `json:"updated_at"`
}

type KnowledgeItemDetailResponse struct {
	ID              int64   `json:"id"`
	GroupID         int64   `json:"group_id"`
	GroupName       string  `json:"group_name"`
	Name            string  `json:"name"`
	SourceType      string  `json:"source_type"`
	SourceURI       *string `json:"source_uri"`
	Status          string  `json:"status"`
	SizeBytes       int64   `json:"size_bytes"`
	ErrorMessage    *string `json:"error_message"`
	MarkdownContent *string `json:"markdown_content"`
	CreatedAt       string  `json:"created_at"`
	UpdatedAt       string  `json:"updated_at"`
}

type KnowledgeTextItemRequest struct {
	GroupID int64  `json:"group_id" binding:"required"`
	Name    string `json:"name" binding:"required"`
	Content string `json:"content" binding:"required"`
}

type KnowledgeURLItemRequest struct {
	GroupID int64  `json:"group_id" binding:"required"`
	Name    string `json:"name" binding:"required"`
	URL     string `json:"url" binding:"required"`
}

type KnowledgeContext struct {
	GroupIDs   []int64
	GroupNames []string
	Snippets   []KnowledgeSnippet
}

type KnowledgeSnippet struct {
	PointID         string
	GroupID         int64
	GroupName       string
	KnowledgeItemID int64
	ItemName        string
	Text            string
	Score           float64
}

func NewKnowledgeService(
	pool *pgxpool.Pool,
	provider retrieval.Provider,
	store retrieval.VectorStore,
	objectStorage objectstorage.Client,
	llmClient llm.Client,
	logger *zap.Logger,
	cfg config.RetrievalConfig,
	llmCfg config.LLMConfig,
	queueSize int,
	workerCount int,
) *KnowledgeService {
	chunkSize := cfg.ChunkSize
	if chunkSize <= 0 {
		chunkSize = 900
	}
	chunkOverlap := cfg.ChunkOverlap
	if chunkOverlap < 0 {
		chunkOverlap = 120
	}
	batchSize := cfg.EmbeddingBatchSize
	if batchSize <= 0 {
		batchSize = 16
	}
	recallLimit := cfg.RecallLimit
	if recallLimit <= 0 {
		recallLimit = 12
	}
	rerankTopN := cfg.RerankTopN
	if rerankTopN <= 0 {
		rerankTopN = 6
	}

	svc := &KnowledgeService{
		pool:               pool,
		provider:           provider,
		store:              store,
		objectStorage:      objectStorage,
		llmClient:          llmClient,
		logger:             logger,
		chunkSize:          chunkSize,
		chunkOverlap:       chunkOverlap,
		embeddingBatchSize: batchSize,
		recallLimit:        recallLimit,
		rerankTopN:         rerankTopN,
		arkBaseURL:         strings.TrimSpace(llmCfg.BaseURL),
		arkAPIKey:          strings.TrimSpace(llmCfg.APIKey),
		urlMarkdownModel:   strings.TrimSpace(llmCfg.KnowledgeURLModel),
		httpClient:         &http.Client{Timeout: defaultKnowledgeURLParseTimeout},
		cleanupInFlight:    make(map[string]struct{}),
	}
	if svc.urlMarkdownModel == "" {
		svc.urlMarkdownModel = strings.TrimSpace(llmCfg.Model)
	}
	if svc.arkBaseURL == "" {
		svc.arkBaseURL = defaultKnowledgeArkBaseURL
	}

	if workerCount > 0 {
		if queueSize <= 0 {
			queueSize = defaultKnowledgeQueueSize
		}
		svc.jobs = make(chan knowledgeParseJob, queueSize)
		for i := 0; i < workerCount; i++ {
			go svc.runParseWorker()
		}
		go svc.recoverPendingParseTasks()
		go svc.runDeletedCleanupWorker()
	}

	return svc
}

func (s *KnowledgeService) ListGroups(ctx context.Context) ([]KnowledgeGroupResponse, error) {
	actor := auth.MustActor(ctx)
	rows, err := s.pool.Query(ctx, `
		SELECT id, name, parent_id, sort_order, created_at
		FROM knowledge_groups
		WHERE tenant_id = $1 AND deleted_at IS NULL
		ORDER BY sort_order, created_at, id
	`, actor.TenantID)
	if err != nil {
		return nil, response.ErrInternal(50050, "knowledge_groups_query_failed", "failed to list knowledge groups")
	}
	defer rows.Close()

	nodeMap := make(map[int64]*KnowledgeGroupResponse)
	parentMap := make(map[int64][]int64)
	orderedIDs := make([]int64, 0)
	for rows.Next() {
		node := &KnowledgeGroupResponse{}
		var createdAt time.Time
		if err := rows.Scan(&node.ID, &node.Name, &node.ParentID, &node.SortOrder, &createdAt); err != nil {
			return nil, response.ErrInternal(50050, "knowledge_groups_scan_failed", err.Error())
		}
		node.CreatedAt = createdAt.Format(time.RFC3339)
		node.Children = []KnowledgeGroupResponse{}
		nodeMap[node.ID] = node
		orderedIDs = append(orderedIDs, node.ID)
		if node.ParentID != nil {
			parentMap[*node.ParentID] = append(parentMap[*node.ParentID], node.ID)
		}
	}

	countRows, err := s.pool.Query(ctx, `
		SELECT group_id, COUNT(*)::bigint
		FROM knowledge_items
		WHERE tenant_id = $1 AND deleted_at IS NULL
		GROUP BY group_id
	`, actor.TenantID)
	if err != nil {
		return nil, response.ErrInternal(50050, "knowledge_group_counts_query_failed", "failed to list knowledge group counts")
	}
	defer countRows.Close()

	directCounts := make(map[int64]int64)
	for countRows.Next() {
		var groupID, itemCount int64
		if err := countRows.Scan(&groupID, &itemCount); err != nil {
			return nil, response.ErrInternal(50050, "knowledge_group_counts_scan_failed", err.Error())
		}
		directCounts[groupID] = itemCount
	}

	var sumCounts func(int64) int64
	sumCounts = func(groupID int64) int64 {
		total := directCounts[groupID]
		for _, childID := range parentMap[groupID] {
			total += sumCounts(childID)
		}
		if node := nodeMap[groupID]; node != nil {
			node.ItemCount = total
		}
		return total
	}

	var buildTree func(int64) KnowledgeGroupResponse
	buildTree = func(groupID int64) KnowledgeGroupResponse {
		node := nodeMap[groupID]
		if node == nil {
			return KnowledgeGroupResponse{}
		}
		result := *node
		childIDs := parentMap[groupID]
		result.Children = make([]KnowledgeGroupResponse, 0, len(childIDs))
		for _, childID := range childIDs {
			result.Children = append(result.Children, buildTree(childID))
		}
		return result
	}

	roots := make([]KnowledgeGroupResponse, 0)
	for _, groupID := range orderedIDs {
		node := nodeMap[groupID]
		if node == nil {
			continue
		}
		if node.ParentID == nil {
			sumCounts(groupID)
			roots = append(roots, buildTree(groupID))
		}
	}

	return roots, nil
}

func (s *KnowledgeService) CreateGroup(ctx context.Context, req KnowledgeGroupRequest) (*KnowledgeGroupResponse, error) {
	actor := auth.MustActor(ctx)

	name := strings.TrimSpace(req.Name)
	if name == "" {
		return nil, response.ErrBadRequest(40050, "knowledge_group_name_required", "knowledge group name is required")
	}

	var parentID *int64
	if req.ParentID != nil && *req.ParentID > 0 {
		if err := s.ensureRootGroup(ctx, actor.TenantID, *req.ParentID); err != nil {
			return nil, err
		}
		parentID = req.ParentID
	}

	sortOrder := 0
	if req.SortOrder != nil {
		sortOrder = *req.SortOrder
	}

	var (
		groupID   int64
		createdAt time.Time
	)
	err := s.pool.QueryRow(ctx, `
		INSERT INTO knowledge_groups (tenant_id, name, parent_id, sort_order)
		VALUES ($1, $2, $3, $4)
		RETURNING id, created_at
	`, actor.TenantID, name, parentID, sortOrder).Scan(&groupID, &createdAt)
	if err != nil {
		return nil, response.ErrConflict(40950, "knowledge_group_create_failed", "knowledge group already exists or cannot be created")
	}

	return &KnowledgeGroupResponse{
		ID:        groupID,
		Name:      name,
		ParentID:  parentID,
		SortOrder: sortOrder,
		ItemCount: 0,
		CreatedAt: createdAt.Format(time.RFC3339),
		Children:  []KnowledgeGroupResponse{},
	}, nil
}

func (s *KnowledgeService) UpdateGroup(ctx context.Context, groupID int64, req KnowledgeGroupRequest) error {
	actor := auth.MustActor(ctx)

	name := strings.TrimSpace(req.Name)
	if name == "" {
		return response.ErrBadRequest(40050, "knowledge_group_name_required", "knowledge group name is required")
	}

	sortOrder := 0
	if req.SortOrder != nil {
		sortOrder = *req.SortOrder
	}

	tag, err := s.pool.Exec(ctx, `
		UPDATE knowledge_groups
		SET name = $1, sort_order = $2, updated_at = NOW()
		WHERE id = $3 AND tenant_id = $4 AND deleted_at IS NULL
	`, name, sortOrder, groupID, actor.TenantID)
	if err != nil {
		return response.ErrConflict(40950, "knowledge_group_update_failed", "knowledge group already exists or cannot be updated")
	}
	if tag.RowsAffected() == 0 {
		return response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
	}
	return nil
}

func (s *KnowledgeService) DeleteGroup(ctx context.Context, groupID int64) error {
	actor := auth.MustActor(ctx)

	tx, err := s.pool.Begin(ctx)
	if err != nil {
		return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to begin delete transaction")
	}
	defer func() { _ = tx.Rollback(ctx) }()

	groupIDs, err := s.listGroupSubtreeIDs(ctx, tx, actor.TenantID, groupID)
	if err != nil {
		return err
	}
	if len(groupIDs) == 0 {
		return response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
	}

	itemIDs, err := s.listActiveKnowledgeItemIDs(ctx, tx, actor.TenantID, groupIDs)
	if err != nil {
		return err
	}

	if len(itemIDs) > 0 {
		if _, err := tx.Exec(ctx, `
			UPDATE knowledge_items
			SET deleted_at = NOW(), updated_at = NOW(), status = 'deleted'
			WHERE tenant_id = $1
			  AND deleted_at IS NULL
			  AND id = ANY($2::bigint[])
		`, actor.TenantID, itemIDs); err != nil {
			return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to delete knowledge items")
		}

		if _, err := tx.Exec(ctx, `
			UPDATE knowledge_parse_tasks
			SET status = 'deleted',
			    error_message = 'knowledge group deleted',
			    completed_at = COALESCE(completed_at, NOW()),
			    updated_at = NOW()
			WHERE tenant_id = $1
			  AND knowledge_item_id = ANY($2::bigint[])
			  AND status <> 'deleted'
		`, actor.TenantID, itemIDs); err != nil {
			return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to cancel knowledge parse tasks")
		}

		if _, err := tx.Exec(ctx, `
			UPDATE knowledge_chunks_meta
			SET status = 'deleting', updated_at = NOW()
			WHERE tenant_id = $1
			  AND knowledge_item_id = ANY($2::bigint[])
			  AND status <> 'deleted'
		`, actor.TenantID, itemIDs); err != nil {
			return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to mark knowledge chunks for deletion")
		}
	}

	if _, err := tx.Exec(ctx, `
		DELETE FROM prompt_rule_knowledge_groups
		WHERE knowledge_group_id = ANY($1::bigint[])
	`, groupIDs); err != nil {
		return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to delete knowledge group references")
	}

	if _, err := tx.Exec(ctx, `
		UPDATE knowledge_groups
		SET deleted_at = NOW(), updated_at = NOW()
		WHERE tenant_id = $1
		  AND deleted_at IS NULL
		  AND id = ANY($2::bigint[])
	`, actor.TenantID, groupIDs); err != nil {
		return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to delete knowledge groups")
	}

	if err := tx.Commit(ctx); err != nil {
		return response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to commit delete transaction")
	}

	for _, itemID := range itemIDs {
		go s.cleanupDeletedKnowledgeItemAsync(actor.TenantID, itemID)
	}

	if s.logger != nil {
		s.logger.Info("knowledge group soft deleted",
			zap.Int64("tenant_id", actor.TenantID),
			zap.Int64("knowledge_group_id", groupID),
			zap.Int("group_count", len(groupIDs)),
			zap.Int("item_count", len(itemIDs)),
		)
	}
	return nil
}

func (s *KnowledgeService) ListItems(ctx context.Context, groupID *int64) ([]KnowledgeItemResponse, error) {
	actor := auth.MustActor(ctx)
	if groupID != nil && *groupID > 0 {
		if err := s.ensureGroupAccessible(ctx, actor.TenantID, *groupID); err != nil {
			return nil, err
		}
	}

	query := `
		SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status,
		       ki.size_bytes, ki.error_message, ki.created_at, ki.updated_at
		FROM knowledge_items ki
		JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
		WHERE ki.tenant_id = $1
		  AND ki.deleted_at IS NULL
	`
	args := []any{actor.TenantID}
	if groupID != nil && *groupID > 0 {
		query += `
		  AND ki.group_id IN (
			  WITH RECURSIVE selected_groups AS (
				  SELECT id
				  FROM knowledge_groups
				  WHERE id = $2 AND tenant_id = $1 AND deleted_at IS NULL
				  UNION ALL
				  SELECT child.id
				  FROM knowledge_groups child
				  JOIN selected_groups parent ON child.parent_id = parent.id
				  WHERE child.tenant_id = $1 AND child.deleted_at IS NULL
			  )
			  SELECT id FROM selected_groups
		  )
		`
		args = append(args, *groupID)
	}
	query += ` ORDER BY ki.created_at DESC, ki.id DESC`

	rows, err := s.pool.Query(ctx, query, args...)
	if err != nil {
		return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to list knowledge items")
	}
	defer rows.Close()

	items := make([]KnowledgeItemResponse, 0)
	for rows.Next() {
		var (
			item                 KnowledgeItemResponse
			sourceURI            *string
			errorMessage         *string
			createdAt, updatedAt time.Time
		)
		if err := rows.Scan(
			&item.ID,
			&item.GroupID,
			&item.GroupName,
			&item.Name,
			&item.SourceType,
			&sourceURI,
			&item.Status,
			&item.SizeBytes,
			&errorMessage,
			&createdAt,
			&updatedAt,
		); err != nil {
			return nil, response.ErrInternal(50051, "knowledge_items_scan_failed", err.Error())
		}
		item.SourceURI = sourceURI
		item.ErrorMessage = errorMessage
		item.CreatedAt = createdAt.Format(time.RFC3339)
		item.UpdatedAt = updatedAt.Format(time.RFC3339)
		items = append(items, item)
	}

	return items, nil
}

func (s *KnowledgeService) GetItemDetail(ctx context.Context, itemID int64) (*KnowledgeItemDetailResponse, error) {
	actor := auth.MustActor(ctx)

	var (
		item                 KnowledgeItemDetailResponse
		sourceURI            *string
		errorMessage         *string
		markdownContent      *string
		createdAt, updatedAt time.Time
	)

	err := s.pool.QueryRow(ctx, `
		SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status,
		       ki.size_bytes, ki.error_message, ki.markdown_content, ki.created_at, ki.updated_at
		FROM knowledge_items ki
		JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
		WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
	`, itemID, actor.TenantID).Scan(
		&item.ID,
		&item.GroupID,
		&item.GroupName,
		&item.Name,
		&item.SourceType,
		&sourceURI,
		&item.Status,
		&item.SizeBytes,
		&errorMessage,
		&markdownContent,
		&createdAt,
		&updatedAt,
	)
	if err != nil {
		if errors.Is(err, pgx.ErrNoRows) {
			return nil, response.ErrNotFound(40450, "knowledge_item_not_found", "knowledge item not found")
		}
		return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge item")
	}

	item.SourceURI = sourceURI
	item.ErrorMessage = errorMessage
	item.MarkdownContent = markdownContent
	item.CreatedAt = createdAt.Format(time.RFC3339)
	item.UpdatedAt = updatedAt.Format(time.RFC3339)
	return &item, nil
}

func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) {
	actor := auth.MustActor(ctx)

	if err := s.validateIngestionDependencies("text"); err != nil {
		return nil, err
	}

	content := normalizeKnowledgeText(req.Content)
	if content == "" {
		return nil, response.ErrBadRequest(40051, "knowledge_content_required", "knowledge content is required")
	}

	return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
		GroupID:     req.GroupID,
		Name:        strings.TrimSpace(req.Name),
		SourceType:  "text",
		ContentText: &content,
		SizeBytes:   int64(len([]byte(content))),
	})
}

func (s *KnowledgeService) CreateURLItem(ctx context.Context, req KnowledgeURLItemRequest) (*KnowledgeItemResponse, error) {
	actor := auth.MustActor(ctx)

	if err := s.validateIngestionDependencies("website"); err != nil {
		return nil, err
	}

	rawURL := strings.TrimSpace(req.URL)
	if _, err := url.ParseRequestURI(rawURL); err != nil {
		return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid")
	}

	return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
		GroupID:    req.GroupID,
		Name:       strings.TrimSpace(req.Name),
		SourceType: "website",
		SourceURI:  &rawURL,
	})
}

func (s *KnowledgeService) CreateFileItem(
	ctx context.Context,
	groupID int64,
	name string,
	fileName string,
	content []byte,
) (*KnowledgeItemResponse, error) {
	actor := auth.MustActor(ctx)

	if err := s.validateIngestionDependencies("document"); err != nil {
		return nil, err
	}

	displayName := strings.TrimSpace(name)
	if displayName == "" {
		displayName = strings.TrimSuffix(filepath.Base(fileName), filepath.Ext(fileName))
	}

	objectKey := buildKnowledgeObjectKey(actor.TenantID, "document", fileName)
	if err := s.objectStorage.PutBytes(ctx, objectKey, content, detectKnowledgeContentType(fileName)); err != nil {
		return nil, response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
	}

	return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
		GroupID:    groupID,
		Name:       displayName,
		SourceType: "document",
		StorageKey: objectKey,
		SizeBytes:  int64(len(content)),
	})
}

func (s *KnowledgeService) DeleteItem(ctx context.Context, itemID int64) error {
	actor := auth.MustActor(ctx)

	tx, err := s.pool.Begin(ctx)
	if err != nil {
		return response.ErrInternal(50054, "knowledge_item_delete_failed", "failed to begin delete transaction")
	}
	defer func() { _ = tx.Rollback(ctx) }()

	tag, err := tx.Exec(ctx, `
		UPDATE knowledge_items
		SET deleted_at = NOW(), updated_at = NOW(), status = 'deleted'
		WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
	`, itemID, actor.TenantID)
	if err != nil {
		return response.ErrInternal(50054, "knowledge_item_delete_failed", "failed to delete knowledge item")
	}
	if tag.RowsAffected() == 0 {
		return response.ErrNotFound(40450, "knowledge_item_not_found", "knowledge item not found")
	}

	if _, err := tx.Exec(ctx, `
		UPDATE knowledge_parse_tasks
		SET status = 'deleted',
		    error_message = 'knowledge item deleted',
		    completed_at = COALESCE(completed_at, NOW()),
		    updated_at = NOW()
		WHERE knowledge_item_id = $1
		  AND tenant_id = $2
		  AND status IN ('pending', 'running')
	`, itemID, actor.TenantID); err != nil {
		return response.ErrInternal(50054, "knowledge_item_delete_failed", "failed to cancel knowledge parse task")
	}

	if _, err := tx.Exec(ctx, `
		UPDATE knowledge_chunks_meta
		SET status = 'deleting', updated_at = NOW()
		WHERE knowledge_item_id = $1 AND tenant_id = $2 AND status <> 'deleted'
	`, itemID, actor.TenantID); err != nil {
		return response.ErrInternal(50054, "knowledge_item_delete_failed", "failed to mark knowledge chunks for deletion")
	}

	if err := tx.Commit(ctx); err != nil {
		return response.ErrInternal(50054, "knowledge_item_delete_failed", "failed to commit delete transaction")
	}

	if s.logger != nil {
		s.logger.Info("knowledge item soft deleted",
			zap.Int64("tenant_id", actor.TenantID),
			zap.Int64("knowledge_item_id", itemID),
		)
	}
	go s.cleanupDeletedKnowledgeItemAsync(actor.TenantID, itemID)
	return nil
}

func (s *KnowledgeService) ResolveContext(
	ctx context.Context,
	tenantID int64,
	groupIDs []int64,
	query string,
) (*KnowledgeContext, error) {
	groupIDs = normalizeInt64IDs(groupIDs)
	if len(groupIDs) == 0 {
		return nil, nil
	}
	if strings.TrimSpace(query) == "" {
		return nil, nil
	}
	if err := s.provider.Validate(); err != nil {
		return nil, response.ErrServiceUnavailable(50351, "retrieval_unavailable", err.Error())
	}
	if err := s.store.Validate(); err != nil {
		return nil, response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
	}

	groupNames, err := s.listGroupNames(ctx, tenantID, groupIDs)
	if err != nil {
		return nil, err
	}
	if len(groupNames) == 0 {
		return nil, nil
	}

	vectors, err := s.provider.Embed(ctx, []string{query})
	if err != nil {
		return nil, response.ErrServiceUnavailable(50353, "embedding_failed", err.Error())
	}
	if len(vectors) == 0 || len(vectors[0]) == 0 {
		return nil, response.ErrServiceUnavailable(50353, "embedding_failed", "embedding vector is empty")
	}

	searchPoints, err := s.store.Search(ctx, retrieval.SearchRequest{
		Vector:   vectors[0],
		TenantID: tenantID,
		GroupIDs: groupIDs,
		Limit:    s.recallLimit,
	})
	if err != nil {
		return nil, response.ErrServiceUnavailable(50354, "qdrant_search_failed", err.Error())
	}
	if len(searchPoints) == 0 {
		return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}}, nil
	}

	candidates := make([]KnowledgeSnippet, 0, len(searchPoints))
	documents := make([]string, 0, len(searchPoints))
	for _, point := range searchPoints {
		text := strings.TrimSpace(anyString(point.Payload["text"]))
		if text == "" {
			continue
		}
		candidates = append(candidates, KnowledgeSnippet{
			PointID:         point.ID,
			GroupID:         anyInt64(point.Payload["group_id"]),
			GroupName:       strings.TrimSpace(anyString(point.Payload["group_name"])),
			KnowledgeItemID: anyInt64(point.Payload["item_id"]),
			ItemName:        strings.TrimSpace(anyString(point.Payload["item_name"])),
			Text:            text,
			Score:           point.Score,
		})
		documents = append(documents, text)
	}
	if len(candidates) == 0 {
		return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}}, nil
	}

	candidates, err = s.filterActiveKnowledgeSnippets(ctx, tenantID, candidates)
	if err != nil {
		return nil, err
	}
	if len(candidates) == 0 {
		return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}}, nil
	}

	documents = documents[:0]
	for _, candidate := range candidates {
		documents = append(documents, candidate.Text)
	}

	reranked, err := s.provider.Rerank(ctx, query, documents, minInt(s.rerankTopN, len(documents)))
	if err != nil {
		selected := candidates[:minInt(len(candidates), s.rerankTopN)]
		return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: selected}, nil
	}

	selected := make([]KnowledgeSnippet, 0, len(reranked))
	for _, item := range reranked {
		if item.Index < 0 || item.Index >= len(candidates) {
			continue
		}
		snippet := candidates[item.Index]
		snippet.Score = item.RelevanceScore
		if strings.TrimSpace(item.Text) != "" {
			snippet.Text = strings.TrimSpace(item.Text)
		}
		selected = append(selected, snippet)
	}

	return &KnowledgeContext{
		GroupIDs:   groupIDs,
		GroupNames: groupNames,
		Snippets:   selected,
	}, nil
}

func (s *KnowledgeService) RenderPromptSection(ctx *KnowledgeContext) string {
	if ctx == nil || len(ctx.Snippets) == 0 {
		return ""
	}

	lines := prompts.KnowledgePromptIntroLines()
	for index, snippet := range ctx.Snippets {
		label := strings.TrimSpace(snippet.ItemName)
		if label == "" {
			label = prompts.KnowledgeSnippetLabel(index)
		}
		groupName := strings.TrimSpace(snippet.GroupName)
		if groupName != "" {
			label = fmt.Sprintf("%s / %s", groupName, label)
		}
		lines = append(lines, fmt.Sprintf("%d. %s\n%s", index+1, label, snippet.Text))
	}
	return strings.Join(lines, "\n")
}

type knowledgeIngestInput struct {
	GroupID     int64
	Name        string
	SourceType  string
	SourceURI   *string
	StorageKey  string
	ContentText *string
	SizeBytes   int64
}

func (s *KnowledgeService) validateIngestionDependencies(sourceType string) error {
	if err := s.provider.Validate(); err != nil {
		return response.ErrServiceUnavailable(50351, "retrieval_unavailable", err.Error())
	}
	if err := s.store.Validate(); err != nil {
		return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
	}
	if sourceType == "document" {
		if err := s.objectStorage.Validate(); err != nil {
			return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
		}
	}
	if sourceType == "website" {
		if strings.TrimSpace(s.arkAPIKey) == "" {
			return response.ErrServiceUnavailable(50358, "knowledge_webpage_parser_unavailable", "ark api key is not configured")
		}
		if err := s.llmClient.Validate(); err != nil {
			return response.ErrServiceUnavailable(50359, "knowledge_markdown_formatter_unavailable", err.Error())
		}
	}
	return nil
}

func (s *KnowledgeService) queueKnowledgeItem(
	ctx context.Context,
	tenantID int64,
	input knowledgeIngestInput,
) (*KnowledgeItemResponse, error) {
	resolvedGroupID, err := s.resolveKnowledgeStorageGroupID(ctx, tenantID, input.GroupID)
	if err != nil {
		return nil, err
	}
	input.GroupID = resolvedGroupID

	input.Name = strings.TrimSpace(input.Name)
	if input.Name == "" {
		return nil, response.ErrBadRequest(40054, "knowledge_name_required", "knowledge item name is required")
	}

	if input.ContentText != nil {
		contentText := normalizeKnowledgeText(*input.ContentText)
		if contentText == "" {
			return nil, response.ErrBadRequest(40051, "knowledge_content_required", "knowledge content is required")
		}
		input.ContentText = &contentText
	}

	itemID, parseTaskID, err := s.createPendingKnowledgeItem(ctx, tenantID, input)
	if err != nil {
		return nil, err
	}

	if err := s.enqueueParseJob(knowledgeParseJob{
		TenantID:    tenantID,
		ItemID:      itemID,
		ParseTaskID: parseTaskID,
	}); err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), tenantID, itemID, parseTaskID, err)
		return nil, response.ErrServiceUnavailable(50356, "knowledge_queue_unavailable", "knowledge parse queue is busy, please retry")
	}

	items, err := s.listKnowledgeItemsByIDs(ctx, tenantID, []int64{itemID})
	if err != nil {
		return nil, err
	}
	if len(items) == 0 {
		return nil, response.ErrInternal(50055, "knowledge_item_create_failed", "knowledge item was not found after creation")
	}
	return &items[0], nil
}

func (s *KnowledgeService) enqueueParseJob(job knowledgeParseJob) error {
	if s.jobs == nil {
		return fmt.Errorf("knowledge parse workers are not enabled")
	}

	select {
	case s.jobs <- job:
		return nil
	default:
		return fmt.Errorf("knowledge parse queue is full")
	}
}

func (s *KnowledgeService) runParseWorker() {
	for job := range s.jobs {
		func(job knowledgeParseJob) {
			defer func() {
				if recovered := recover(); recovered != nil {
					err := fmt.Errorf("knowledge parse panic: %v", recovered)
					if s.logger != nil {
						s.logger.Error("knowledge parse worker panicked",
							zap.Int64("tenant_id", job.TenantID),
							zap.Int64("knowledge_item_id", job.ItemID),
							zap.Int64("parse_task_id", job.ParseTaskID),
							zap.Any("panic", recovered),
							zap.ByteString("stack", debug.Stack()),
						)
					}
					_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
				}
			}()

			s.executeParseJob(context.Background(), job)
		}(job)
	}
}

func (s *KnowledgeService) recoverPendingParseTasks() {
	if s.jobs == nil {
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	rows, err := s.pool.Query(ctx, `
		SELECT kpt.tenant_id, kpt.knowledge_item_id, kpt.id
		FROM knowledge_parse_tasks kpt
		JOIN knowledge_items ki ON ki.id = kpt.knowledge_item_id
		WHERE kpt.status IN ('pending', 'running')
		  AND ki.deleted_at IS NULL
		  AND ki.status IN ('pending', 'processing')
		ORDER BY kpt.created_at ASC, kpt.id ASC
	`)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		var job knowledgeParseJob
		if scanErr := rows.Scan(&job.TenantID, &job.ItemID, &job.ParseTaskID); scanErr != nil {
			continue
		}
		if err := s.enqueueParseJob(job); err != nil {
			return
		}
	}
}

func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgeParseJob) {
	item, err := s.loadKnowledgeItemForParse(ctx, job.TenantID, job.ItemID)
	if err != nil || item == nil {
		return
	}

	if err := s.markKnowledgeIngestionRunning(ctx, job.TenantID, job.ItemID, job.ParseTaskID); err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}

	parsedContent, err := s.readKnowledgeItemParsedContent(ctx, item)
	if err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}
	if item.SourceType == "website" {
		if resolvedName := extractKnowledgeMarkdownTitle(parsedContent.Markdown); resolvedName != "" {
			item.Name = resolvedName
		}
	}

	chunks := splitKnowledgeTextIntoChunks(parsedContent.Text, s.chunkSize, s.chunkOverlap)
	if len(chunks) == 0 {
		err = errors.New("knowledge content is empty after parsing")
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}

	pointIDs, err := s.listKnowledgePointIDs(ctx, job.TenantID, job.ItemID)
	if err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}
	if len(pointIDs) > 0 {
		if err := s.store.DeletePoints(ctx, pointIDs); err != nil {
			_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
			return
		}
	}

	embeddings, err := s.embedChunks(ctx, chunks)
	if err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}
	if len(embeddings) == 0 || len(embeddings[0]) == 0 {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, errors.New("embedding vector is empty"))
		return
	}

	if err := s.store.EnsureCollection(ctx, len(embeddings[0])); err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}

	insertedPointIDs := make([]string, 0, len(chunks))
	defer func() {
		if err != nil && len(insertedPointIDs) > 0 {
			_ = s.store.DeletePoints(context.Background(), insertedPointIDs)
		}
	}()

	points := make([]retrieval.VectorPoint, 0, len(chunks))
	metas := make([]knowledgeChunkMetaInput, 0, len(chunks))
	for index, chunk := range chunks {
		pointID := uuid.NewString()
		insertedPointIDs = append(insertedPointIDs, pointID)
		points = append(points, retrieval.VectorPoint{
			ID:     pointID,
			Vector: embeddings[index],
			Payload: map[string]any{
				"tenant_id":    job.TenantID,
				"group_id":     item.GroupID,
				"group_name":   item.GroupName,
				"item_id":      item.ID,
				"item_name":    item.Name,
				"chunk_index":  index,
				"item_version": item.ItemVersion,
				"status":       "active",
				"source_type":  item.SourceType,
				"text":         chunk,
			},
		})
		metas = append(metas, knowledgeChunkMetaInput{
			ChunkIndex:    index,
			TokenCount:    estimateKnowledgeTokenCount(chunk),
			QdrantPointID: pointID,
			ItemVersion:   item.ItemVersion,
			Status:        "active",
		})
	}

	if err = s.store.Upsert(ctx, points); err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}

	if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, item.Name, parsedContent, metas); err != nil {
		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
		return
	}
}

func (s *KnowledgeService) loadKnowledgeItemForParse(
	ctx context.Context,
	tenantID, itemID int64,
) (*knowledgeItemRecord, error) {
	record := &knowledgeItemRecord{}
	var contentText *string
	var markdownContent *string
	err := s.pool.QueryRow(ctx, `
		SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
		       ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
		FROM knowledge_items ki
		JOIN knowledge_groups kg ON kg.id = ki.group_id
		WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
	`, itemID, tenantID).Scan(
		&record.ID,
		&record.TenantID,
		&record.GroupID,
		&record.GroupName,
		&record.Name,
		&record.SourceType,
		&record.SourceURI,
		&record.StorageKey,
		&contentText,
		&markdownContent,
		&record.ItemVersion,
	)
	if err != nil {
		return nil, nil
	}
	record.ContentText = contentText
	record.Markdown = markdownContent
	return record, nil
}

func (s *KnowledgeService) readKnowledgeItemParsedContent(ctx context.Context, item *knowledgeItemRecord) (*knowledgeParsedContent, error) {
	if item == nil {
		return nil, fmt.Errorf("knowledge item is empty")
	}

	switch item.SourceType {
	case "text":
		return parseKnowledgeTextInput(item.Name, derefKnowledgeString(item.ContentText)), nil
	case "document":
		data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
		if err != nil {
			return nil, err
		}
		return extractKnowledgeContentFromFile(item.StorageKey, data)
	case "website":
		if item.SourceURI == nil || strings.TrimSpace(*item.SourceURI) == "" {
			return nil, fmt.Errorf("knowledge website url is empty")
		}
		return s.parseWebsiteKnowledge(ctx, strings.TrimSpace(*item.SourceURI))
	default:
		return nil, fmt.Errorf("unsupported knowledge source type %q", item.SourceType)
	}
}

type knowledgeChunkMetaInput struct {
	ChunkIndex    int
	TokenCount    int
	QdrantPointID string
	ItemVersion   int
	Status        string
}

func (s *KnowledgeService) createPendingKnowledgeItem(
	ctx context.Context,
	tenantID int64,
	input knowledgeIngestInput,
) (int64, int64, error) {
	tx, err := s.pool.Begin(ctx)
	if err != nil {
		return 0, 0, response.ErrInternal(50055, "knowledge_item_create_failed", "failed to begin knowledge transaction")
	}
	defer func() { _ = tx.Rollback(ctx) }()

	var itemID int64
	err = tx.QueryRow(ctx, `
		INSERT INTO knowledge_items (
			tenant_id, group_id, source_type, name, source_uri, storage_key,
			content_text, markdown_content, status, size_bytes, item_version
		)
		VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, 'pending', $8, 1)
		RETURNING id
	`, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID)
	if err != nil {
		return 0, 0, response.ErrConflict(40951, "knowledge_item_create_failed", "failed to create knowledge item")
	}

	var taskID int64
	err = tx.QueryRow(ctx, `
		INSERT INTO knowledge_parse_tasks (
			tenant_id, knowledge_item_id, source_type, status
		)
		VALUES ($1, $2, $3, 'pending')
		RETURNING id
	`, tenantID, itemID, input.SourceType).Scan(&taskID)
	if err != nil {
		return 0, 0, response.ErrInternal(50055, "knowledge_item_create_failed", "failed to create knowledge parse task")
	}

	if err := tx.Commit(ctx); err != nil {
		return 0, 0, response.ErrInternal(50055, "knowledge_item_create_failed", "failed to commit knowledge transaction")
	}
	return itemID, taskID, nil
}

func (s *KnowledgeService) markKnowledgeIngestionRunning(
	ctx context.Context,
	tenantID, itemID, parseTaskID int64,
) error {
	if _, err := s.pool.Exec(ctx, `
		UPDATE knowledge_items
		SET status = 'processing', error_message = NULL, updated_at = NOW()
		WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
	`, itemID, tenantID); err != nil {
		return err
	}

	if _, err := s.pool.Exec(ctx, `
		UPDATE knowledge_parse_tasks
		SET status = 'running',
		    error_message = NULL,
		    started_at = COALESCE(started_at, NOW()),
		    updated_at = NOW()
		WHERE id = $1 AND tenant_id = $2
	`, parseTaskID, tenantID); err != nil {
		return err
	}

	return nil
}

func (s *KnowledgeService) markKnowledgeIngestionCompleted(
	ctx context.Context,
	tenantID, itemID, parseTaskID int64,
	itemName string,
	content *knowledgeParsedContent,
	metas []knowledgeChunkMetaInput,
) error {
	if content == nil {
		return errors.New("knowledge content is empty")
	}
	if strings.TrimSpace(content.Text) == "" {
		return errors.New("knowledge text content is empty")
	}
	if strings.TrimSpace(content.Markdown) == "" {
		content.Markdown = buildKnowledgeMarkdownFromText("", content.Text)
	}
	itemName = strings.TrimSpace(itemName)

	tx, err := s.pool.Begin(ctx)
	if err != nil {
		return err
	}
	defer func() { _ = tx.Rollback(ctx) }()

	if _, err := tx.Exec(ctx, `
		DELETE FROM knowledge_chunks_meta
		WHERE tenant_id = $1 AND knowledge_item_id = $2
	`, tenantID, itemID); err != nil {
		return err
	}

	for _, meta := range metas {
		if _, err := tx.Exec(ctx, `
			INSERT INTO knowledge_chunks_meta (
				tenant_id, knowledge_item_id, chunk_index, token_count,
				qdrant_point_id, item_version, status
			)
			VALUES ($1, $2, $3, $4, $5, $6, $7)
		`, tenantID, itemID, meta.ChunkIndex, meta.TokenCount, meta.QdrantPointID, meta.ItemVersion, meta.Status); err != nil {
			return err
		}
	}

	if _, err := tx.Exec(ctx, `
		UPDATE knowledge_items
		SET status = 'completed',
		    error_message = NULL,
		    name = CASE WHEN $3 <> '' THEN $3 ELSE name END,
		    content_text = $4,
		    markdown_content = $5,
		    updated_at = NOW()
		WHERE id = $1 AND tenant_id = $2
	`, itemID, tenantID, itemName, content.Text, content.Markdown); err != nil {
		return err
	}

	if _, err := tx.Exec(ctx, `
		UPDATE knowledge_parse_tasks
		SET status = 'completed', error_message = NULL, completed_at = NOW(), updated_at = NOW()
		WHERE id = $1 AND tenant_id = $2
	`, parseTaskID, tenantID); err != nil {
		return err
	}

	return tx.Commit(ctx)
}

func (s *KnowledgeService) markKnowledgeIngestionFailed(
	ctx context.Context,
	tenantID, itemID, parseTaskID int64,
	cause error,
) error {
	errMessage := "knowledge ingestion failed"
	if cause != nil {
		errMessage = strings.TrimSpace(cause.Error())
	}

	if s.logger != nil {
		s.logger.Error("knowledge parse failed",
			zap.Int64("tenant_id", tenantID),
			zap.Int64("knowledge_item_id", itemID),
			zap.Int64("parse_task_id", parseTaskID),
			zap.String("error_message", errMessage),
		)
	}

	_, err := s.pool.Exec(ctx, `
		UPDATE knowledge_items
		SET status = 'failed', error_message = $1, updated_at = NOW()
		WHERE id = $2 AND tenant_id = $3
	`, errMessage, itemID, tenantID)
	if err != nil {
		return err
	}

	_, err = s.pool.Exec(ctx, `
		UPDATE knowledge_parse_tasks
		SET status = 'failed', error_message = $1, completed_at = NOW(), updated_at = NOW()
		WHERE id = $2 AND tenant_id = $3
	`, errMessage, parseTaskID, tenantID)
	return err
}

func buildKnowledgeObjectKey(tenantID int64, sourceType, fileName string) string {
	fileName = sanitizeKnowledgeFileName(fileName)
	if fileName == "" {
		fileName = sourceType + ".bin"
	}
	return fmt.Sprintf(
		"knowledge/%d/%s/%s/%s-%s",
		tenantID,
		sourceType,
		time.Now().UTC().Format("2006/01/02"),
		uuid.NewString(),
		fileName,
	)
}

func sanitizeKnowledgeFileName(fileName string) string {
	fileName = filepath.Base(strings.TrimSpace(fileName))
	replacer := strings.NewReplacer(
		"/", "-",
		"\\", "-",
		":", "-",
		"?", "-",
		"&", "-",
		"=", "-",
		"#", "-",
	)
	return replacer.Replace(fileName)
}

func detectKnowledgeContentType(fileName string) string {
	switch strings.ToLower(filepath.Ext(fileName)) {
	case ".txt", ".md", ".markdown":
		return "text/plain; charset=utf-8"
	case ".html", ".htm":
		return "text/html; charset=utf-8"
	case ".pdf":
		return "application/pdf"
	case ".docx":
		return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
	case ".xlsx":
		return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
	case ".json":
		return "application/json"
	case ".csv":
		return "text/csv; charset=utf-8"
	default:
		return "application/octet-stream"
	}
}

func derefKnowledgeString(value *string) string {
	if value == nil {
		return ""
	}
	return strings.TrimSpace(*value)
}

func (s *KnowledgeService) ensureGroupAccessible(ctx context.Context, tenantID, groupID int64) error {
	var exists bool
	err := s.pool.QueryRow(ctx, `
		SELECT EXISTS(
			SELECT 1
			FROM knowledge_groups
			WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
		)
	`, groupID, tenantID).Scan(&exists)
	if err != nil {
		return response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
	}
	if !exists {
		return response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
	}
	return nil
}

func (s *KnowledgeService) ensureRootGroup(ctx context.Context, tenantID, groupID int64) error {
	if err := s.ensureGroupAccessible(ctx, tenantID, groupID); err != nil {
		return err
	}

	var isRoot bool
	err := s.pool.QueryRow(ctx, `
		SELECT EXISTS(
			SELECT 1
			FROM knowledge_groups
			WHERE id = $1
			  AND tenant_id = $2
			  AND deleted_at IS NULL
			  AND parent_id IS NULL
		)
	`, groupID, tenantID).Scan(&isRoot)
	if err != nil {
		return response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
	}
	if !isRoot {
		return response.ErrBadRequest(40050, "knowledge_group_depth_exceeded", "third-level knowledge groups are not supported")
	}
	return nil
}

func (s *KnowledgeService) ensureStorableGroup(ctx context.Context, tenantID, groupID int64) error {
	if err := s.ensureGroupAccessible(ctx, tenantID, groupID); err != nil {
		return err
	}

	var storable bool
	err := s.pool.QueryRow(ctx, `
		SELECT EXISTS(
			SELECT 1
			FROM knowledge_groups child
			JOIN knowledge_groups parent
			  ON parent.id = child.parent_id
			 AND parent.tenant_id = child.tenant_id
			 AND parent.deleted_at IS NULL
			WHERE child.id = $1
			  AND child.tenant_id = $2
			  AND child.deleted_at IS NULL
			  AND parent.parent_id IS NULL
		)
	`, groupID, tenantID).Scan(&storable)
	if err != nil {
		return response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
	}
	if !storable {
		return response.ErrBadRequest(40053, "knowledge_group_storage_invalid", "knowledge items can only be stored in second-level groups")
	}
	return nil
}

func (s *KnowledgeService) resolveKnowledgeStorageGroupID(ctx context.Context, tenantID, groupID int64) (int64, error) {
	tx, err := s.pool.Begin(ctx)
	if err != nil {
		return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to begin knowledge group transaction")
	}
	defer func() { _ = tx.Rollback(ctx) }()

	resolvedGroupID, err := s.resolveKnowledgeStorageGroupIDTx(ctx, tx, tenantID, groupID)
	if err != nil {
		return 0, err
	}

	if err := tx.Commit(ctx); err != nil {
		return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to commit knowledge group transaction")
	}
	return resolvedGroupID, nil
}

func (s *KnowledgeService) resolveKnowledgeStorageGroupIDTx(
	ctx context.Context,
	tx pgx.Tx,
	tenantID, groupID int64,
) (int64, error) {
	var parentID *int64
	if err := tx.QueryRow(ctx, `
		SELECT parent_id
		FROM knowledge_groups
		WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
		FOR UPDATE
	`, groupID, tenantID).Scan(&parentID); err != nil {
		if errors.Is(err, pgx.ErrNoRows) {
			return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
		}
		return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
	}

	if parentID != nil {
		var grandParentID *int64
		if err := tx.QueryRow(ctx, `
			SELECT parent_id
			FROM knowledge_groups
			WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
		`, *parentID, tenantID).Scan(&grandParentID); err != nil {
			if errors.Is(err, pgx.ErrNoRows) {
				return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
			}
			return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
		}
		if grandParentID != nil {
			return 0, response.ErrBadRequest(40053, "knowledge_group_storage_invalid", "knowledge items can only be stored in second-level groups")
		}
		return groupID, nil
	}

	var childGroupID int64
	if err := tx.QueryRow(ctx, `
		SELECT id
		FROM knowledge_groups
		WHERE tenant_id = $1
		  AND parent_id = $2
		  AND name = $3
		  AND deleted_at IS NULL
		ORDER BY id
		LIMIT 1
	`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err == nil {
		return childGroupID, nil
	} else if !errors.Is(err, pgx.ErrNoRows) {
		return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
	}

	if err := tx.QueryRow(ctx, `
		INSERT INTO knowledge_groups (tenant_id, name, parent_id, sort_order)
		VALUES ($1, $2, $3, 0)
		RETURNING id
	`, tenantID, defaultKnowledgeChildGroupName, groupID).Scan(&childGroupID); err == nil {
		return childGroupID, nil
	}

	if err := tx.QueryRow(ctx, `
		SELECT id
		FROM knowledge_groups
		WHERE tenant_id = $1
		  AND parent_id = $2
		  AND name = $3
		  AND deleted_at IS NULL
		ORDER BY id
		LIMIT 1
	`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err != nil {
		return 0, response.ErrConflict(40950, "knowledge_group_create_failed", "failed to create default knowledge group")
	}
	return childGroupID, nil
}

func (s *KnowledgeService) listGroupSubtreeIDs(
	ctx context.Context,
	tx pgx.Tx,
	tenantID, groupID int64,
) ([]int64, error) {
	rows, err := tx.Query(ctx, `
		WITH RECURSIVE selected_groups AS (
			SELECT id
			FROM knowledge_groups
			WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
			UNION ALL
			SELECT child.id
			FROM knowledge_groups child
			JOIN selected_groups parent ON child.parent_id = parent.id
			WHERE child.tenant_id = $2 AND child.deleted_at IS NULL
		)
		SELECT id
		FROM selected_groups
	`, groupID, tenantID)
	if err != nil {
		return nil, response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to query knowledge groups")
	}
	defer rows.Close()

	groupIDs := make([]int64, 0)
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return nil, response.ErrInternal(50054, "knowledge_group_delete_failed", err.Error())
		}
		groupIDs = append(groupIDs, id)
	}
	return groupIDs, nil
}

func (s *KnowledgeService) listActiveKnowledgeItemIDs(
	ctx context.Context,
	tx pgx.Tx,
	tenantID int64,
	groupIDs []int64,
) ([]int64, error) {
	if len(groupIDs) == 0 {
		return []int64{}, nil
	}

	rows, err := tx.Query(ctx, `
		SELECT id
		FROM knowledge_items
		WHERE tenant_id = $1
		  AND deleted_at IS NULL
		  AND group_id = ANY($2::bigint[])
	`, tenantID, groupIDs)
	if err != nil {
		return nil, response.ErrInternal(50054, "knowledge_group_delete_failed", "failed to query knowledge items")
	}
	defer rows.Close()

	itemIDs := make([]int64, 0)
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return nil, response.ErrInternal(50054, "knowledge_group_delete_failed", err.Error())
		}
		itemIDs = append(itemIDs, id)
	}
	return itemIDs, nil
}

func (s *KnowledgeService) embedChunks(ctx context.Context, chunks []string) ([][]float64, error) {
	results := make([][]float64, 0, len(chunks))
	for start := 0; start < len(chunks); start += s.embeddingBatchSize {
		end := start + s.embeddingBatchSize
		if end > len(chunks) {
			end = len(chunks)
		}
		vectors, err := s.provider.Embed(ctx, chunks[start:end])
		if err != nil {
			return nil, err
		}
		results = append(results, vectors...)
	}
	if len(results) != len(chunks) {
		return nil, errors.New("embedding result count mismatch")
	}
	return results, nil
}

func (s *KnowledgeService) listKnowledgePointIDs(ctx context.Context, tenantID, itemID int64) ([]string, error) {
	rows, err := s.pool.Query(ctx, `
		SELECT qdrant_point_id
		FROM knowledge_chunks_meta
		WHERE tenant_id = $1 AND knowledge_item_id = $2 AND status <> 'deleted'
	`, tenantID, itemID)
	if err != nil {
		return nil, response.ErrInternal(50054, "knowledge_item_delete_failed", "failed to query knowledge item points")
	}
	defer rows.Close()

	pointIDs := make([]string, 0)
	for rows.Next() {
		var pointID string
		if err := rows.Scan(&pointID); err != nil {
			return nil, response.ErrInternal(50054, "knowledge_item_delete_failed", err.Error())
		}
		if strings.TrimSpace(pointID) != "" {
			pointIDs = append(pointIDs, pointID)
		}
	}
	return pointIDs, nil
}

func (s *KnowledgeService) filterActiveKnowledgeSnippets(
	ctx context.Context,
	tenantID int64,
	candidates []KnowledgeSnippet,
) ([]KnowledgeSnippet, error) {
	if len(candidates) == 0 {
		return []KnowledgeSnippet{}, nil
	}

	itemIDs := make([]int64, 0, len(candidates))
	for _, candidate := range candidates {
		if candidate.KnowledgeItemID > 0 {
			itemIDs = append(itemIDs, candidate.KnowledgeItemID)
		}
	}
	itemIDs = normalizeInt64IDs(itemIDs)
	if len(itemIDs) == 0 {
		return []KnowledgeSnippet{}, nil
	}

	rows, err := s.pool.Query(ctx, `
		SELECT id
		FROM knowledge_items
		WHERE tenant_id = $1
		  AND id = ANY($2::bigint[])
		  AND deleted_at IS NULL
	`, tenantID, itemIDs)
	if err != nil {
		return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to filter deleted knowledge items")
	}
	defer rows.Close()

	active := make(map[int64]struct{}, len(itemIDs))
	for rows.Next() {
		var itemID int64
		if err := rows.Scan(&itemID); err != nil {
			return nil, response.ErrInternal(50051, "knowledge_items_scan_failed", err.Error())
		}
		active[itemID] = struct{}{}
	}

	filtered := make([]KnowledgeSnippet, 0, len(candidates))
	for _, candidate := range candidates {
		if _, ok := active[candidate.KnowledgeItemID]; ok {
			filtered = append(filtered, candidate)
		}
	}
	return filtered, nil
}

func (s *KnowledgeService) cleanupDeletedKnowledgeItemAsync(tenantID, itemID int64) {
	if !s.startCleanupJob(tenantID, itemID) {
		return
	}

	go func() {
		defer s.finishCleanupJob(tenantID, itemID)

		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
		defer cancel()
		s.cleanupDeletedKnowledgeItem(ctx, tenantID, itemID)
	}()
}

func (s *KnowledgeService) runDeletedCleanupWorker() {
	s.scheduleDeletedKnowledgeCleanup()

	ticker := time.NewTicker(defaultKnowledgeCleanupPoll)
	defer ticker.Stop()

	for range ticker.C {
		s.scheduleDeletedKnowledgeCleanup()
	}
}

func (s *KnowledgeService) scheduleDeletedKnowledgeCleanup() {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	jobs, err := s.listKnowledgeCleanupCandidates(ctx, 20)
	if err != nil {
		return
	}

	for _, job := range jobs {
		s.cleanupDeletedKnowledgeItemAsync(job.TenantID, job.ItemID)
	}
}

func (s *KnowledgeService) listKnowledgeCleanupCandidates(
	ctx context.Context,
	limit int,
) ([]knowledgeCleanupJob, error) {
	if limit <= 0 {
		limit = 20
	}

	rows, err := s.pool.Query(ctx, `
		SELECT ki.tenant_id, ki.id
		FROM knowledge_items ki
		WHERE ki.deleted_at IS NOT NULL
		ORDER BY ki.updated_at ASC, ki.id ASC
		LIMIT $1
	`, limit)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	jobs := make([]knowledgeCleanupJob, 0, limit)
	for rows.Next() {
		var job knowledgeCleanupJob
		if err := rows.Scan(&job.TenantID, &job.ItemID); err != nil {
			return nil, err
		}
		jobs = append(jobs, job)
	}
	return jobs, nil
}

func (s *KnowledgeService) startCleanupJob(tenantID, itemID int64) bool {
	key := fmt.Sprintf("%d:%d", tenantID, itemID)

	s.cleanupMu.Lock()
	defer s.cleanupMu.Unlock()

	if _, exists := s.cleanupInFlight[key]; exists {
		return false
	}
	s.cleanupInFlight[key] = struct{}{}
	return true
}

func (s *KnowledgeService) finishCleanupJob(tenantID, itemID int64) {
	key := fmt.Sprintf("%d:%d", tenantID, itemID)

	s.cleanupMu.Lock()
	delete(s.cleanupInFlight, key)
	s.cleanupMu.Unlock()
}

func (s *KnowledgeService) cleanupDeletedKnowledgeItem(ctx context.Context, tenantID, itemID int64) {
	item, err := s.loadKnowledgeItemForCleanup(ctx, tenantID, itemID)
	if err != nil || item == nil {
		return
	}

	errs := make([]string, 0, 2)

	if strings.TrimSpace(item.StorageKey) != "" {
		if err := s.objectStorage.Delete(ctx, item.StorageKey); err != nil {
			errs = append(errs, err.Error())
		} else {
			_ = s.markKnowledgeStorageDeleted(ctx, tenantID, itemID)
		}
	}

	pointIDs, err := s.listKnowledgePointIDs(ctx, tenantID, itemID)
	if err != nil {
		errs = append(errs, err.Error())
	} else {
		if len(pointIDs) > 0 {
			if err := s.store.DeletePoints(ctx, pointIDs); err != nil {
				errs = append(errs, err.Error())
			}
		}
	}

	if len(errs) > 0 {
		if s.logger != nil {
			s.logger.Error("knowledge cleanup failed",
				zap.Int64("tenant_id", tenantID),
				zap.Int64("knowledge_item_id", itemID),
				zap.Strings("errors", errs),
			)
		}
		_ = s.setKnowledgeCleanupError(ctx, tenantID, itemID, strings.Join(errs, "; "))
		return
	}

	if err := s.deleteKnowledgeItemPermanently(ctx, tenantID, itemID); err != nil {
		if s.logger != nil {
			s.logger.Error("knowledge cleanup finalize failed",
				zap.Int64("tenant_id", tenantID),
				zap.Int64("knowledge_item_id", itemID),
				zap.Error(err),
			)
		}
		_ = s.setKnowledgeCleanupError(ctx, tenantID, itemID, err.Error())
		return
	}

	if s.logger != nil {
		s.logger.Info("knowledge cleanup completed",
			zap.Int64("tenant_id", tenantID),
			zap.Int64("knowledge_item_id", itemID),
		)
	}
	_ = s.clearKnowledgeCleanupError(ctx, tenantID, itemID)
}

func (s *KnowledgeService) loadKnowledgeItemForCleanup(
	ctx context.Context,
	tenantID, itemID int64,
) (*knowledgeItemRecord, error) {
	record := &knowledgeItemRecord{}
	var contentText *string
	var markdownContent *string
	err := s.pool.QueryRow(ctx, `
		SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
		       ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
		FROM knowledge_items ki
		JOIN knowledge_groups kg ON kg.id = ki.group_id
		WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL
	`, itemID, tenantID).Scan(
		&record.ID,
		&record.TenantID,
		&record.GroupID,
		&record.GroupName,
		&record.Name,
		&record.SourceType,
		&record.SourceURI,
		&record.StorageKey,
		&contentText,
		&markdownContent,
		&record.ItemVersion,
	)
	if err != nil {
		return nil, nil
	}
	record.ContentText = contentText
	record.Markdown = markdownContent
	return record, nil
}

func (s *KnowledgeService) markKnowledgeStorageDeleted(ctx context.Context, tenantID, itemID int64) error {
	_, err := s.pool.Exec(ctx, `
		UPDATE knowledge_items
		SET storage_key = '', updated_at = NOW()
		WHERE id = $1 AND tenant_id = $2
	`, itemID, tenantID)
	return err
}

func (s *KnowledgeService) deleteKnowledgeItemPermanently(ctx context.Context, tenantID, itemID int64) error {
	_, err := s.pool.Exec(ctx, `
		DELETE FROM knowledge_items
		WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NOT NULL
	`, itemID, tenantID)
	return err
}

func (s *KnowledgeService) markKnowledgeChunkMetaDeleted(ctx context.Context, tenantID, itemID int64) error {
	_, err := s.pool.Exec(ctx, `
		UPDATE knowledge_chunks_meta
		SET status = 'deleted', updated_at = NOW()
		WHERE knowledge_item_id = $1 AND tenant_id = $2 AND status <> 'deleted'
	`, itemID, tenantID)
	return err
}

func (s *KnowledgeService) setKnowledgeCleanupError(ctx context.Context, tenantID, itemID int64, message string) error {
	_, err := s.pool.Exec(ctx, `
		UPDATE knowledge_items
		SET error_message = $1, updated_at = NOW()
		WHERE id = $2 AND tenant_id = $3
	`, strings.TrimSpace(message), itemID, tenantID)
	return err
}

func (s *KnowledgeService) clearKnowledgeCleanupError(ctx context.Context, tenantID, itemID int64) error {
	_, err := s.pool.Exec(ctx, `
		UPDATE knowledge_items
		SET error_message = NULL, updated_at = NOW()
		WHERE id = $1 AND tenant_id = $2
	`, itemID, tenantID)
	return err
}

func (s *KnowledgeService) listKnowledgeItemsByIDs(ctx context.Context, tenantID int64, itemIDs []int64) ([]KnowledgeItemResponse, error) {
	if len(itemIDs) == 0 {
		return []KnowledgeItemResponse{}, nil
	}
	rows, err := s.pool.Query(ctx, `
		SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status,
		       ki.size_bytes, ki.error_message, ki.created_at, ki.updated_at
		FROM knowledge_items ki
		JOIN knowledge_groups kg ON kg.id = ki.group_id
		WHERE ki.tenant_id = $1 AND ki.id = ANY($2::bigint[]) AND ki.deleted_at IS NULL
		ORDER BY ki.created_at DESC, ki.id DESC
	`, tenantID, itemIDs)
	if err != nil {
		return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to list knowledge items")
	}
	defer rows.Close()

	items := make([]KnowledgeItemResponse, 0)
	for rows.Next() {
		var (
			item                 KnowledgeItemResponse
			sourceURI            *string
			errorMessage         *string
			createdAt, updatedAt time.Time
		)
		if err := rows.Scan(
			&item.ID,
			&item.GroupID,
			&item.GroupName,
			&item.Name,
			&item.SourceType,
			&sourceURI,
			&item.Status,
			&item.SizeBytes,
			&errorMessage,
			&createdAt,
			&updatedAt,
		); err != nil {
			return nil, response.ErrInternal(50051, "knowledge_items_scan_failed", err.Error())
		}
		item.SourceURI = sourceURI
		item.ErrorMessage = errorMessage
		item.CreatedAt = createdAt.Format(time.RFC3339)
		item.UpdatedAt = updatedAt.Format(time.RFC3339)
		items = append(items, item)
	}
	return items, nil
}

func (s *KnowledgeService) listGroupNames(ctx context.Context, tenantID int64, groupIDs []int64) ([]string, error) {
	groupIDs = normalizeInt64IDs(groupIDs)
	if len(groupIDs) == 0 {
		return []string{}, nil
	}

	rows, err := s.pool.Query(ctx, `
		SELECT id, name
		FROM knowledge_groups
		WHERE tenant_id = $1 AND id = ANY($2::bigint[]) AND deleted_at IS NULL
		ORDER BY sort_order, created_at, id
	`, tenantID, groupIDs)
	if err != nil {
		return nil, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge groups")
	}
	defer rows.Close()

	names := make([]string, 0, len(groupIDs))
	for rows.Next() {
		var (
			groupID int64
			name    string
		)
		if err := rows.Scan(&groupID, &name); err != nil {
			return nil, response.ErrInternal(50050, "knowledge_group_query_failed", err.Error())
		}
		if strings.TrimSpace(name) != "" {
			names = append(names, strings.TrimSpace(name))
		}
	}
	return names, nil
}

func normalizeInt64IDs(ids []int64) []int64 {
	if len(ids) == 0 {
		return nil
	}
	set := make(map[int64]struct{}, len(ids))
	result := make([]int64, 0, len(ids))
	for _, id := range ids {
		if id <= 0 {
			continue
		}
		if _, exists := set[id]; exists {
			continue
		}
		set[id] = struct{}{}
		result = append(result, id)
	}
	slices.Sort(result)
	return result
}

func anyString(value any) string {
	switch typed := value.(type) {
	case string:
		return typed
	default:
		return fmt.Sprintf("%v", typed)
	}
}

func anyInt64(value any) int64 {
	switch typed := value.(type) {
	case int64:
		return typed
	case int32:
		return int64(typed)
	case int:
		return int64(typed)
	case float64:
		return int64(typed)
	case float32:
		return int64(typed)
	default:
		return 0
	}
}

func minInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}