feat(ops): add object storage management and site-mapping CSV import/export

Ship the Ops backstage 对象存储 page (browse / upload / move / delete / folder ops) backed by a new object-storage service + handler, with the shared storage client extended with List/Stat/Copy/Usage/DownloadURL so both MinIO and Aliyun providers cover the new surface. Add ForcePathStyle to the object-storage config and treat r2/s3/custom_s3 as external providers so Cloudflare R2 and other S3-compatibles can share the MinIO client path without spinning up the bundled MinIO. Also add CSV import/export + template download to the site-domain mappings page, raise gin MaxMultipartMemory to 8 MiB for the new multipart endpoints, register Ops swagger routes, and extend the descriptions-parity test to cover the ops router. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-26 01:19:01 +08:00
parent 5f6e9f11da
commit 7d8e82c69f
37 changed files with 5111 additions and 66 deletions
@@ -1,10 +1,14 @@
 package app

 import (
+	"bytes"
 	"context"
+	"encoding/csv"
 	"errors"
+	"fmt"
 	"net"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"

@@ -21,6 +25,12 @@ const (
 	ActionSiteDomainMappingEnable  = "site_domain_mapping.enable"
 	ActionSiteDomainMappingDisable = "site_domain_mapping.disable"
 	ActionSiteDomainMappingDelete  = "site_domain_mapping.delete"
+	ActionSiteDomainMappingImport  = "site_domain_mapping.import"
+	ActionSiteDomainMappingExport  = "site_domain_mapping.export"
+
+	siteDomainMappingImportMaxRows   = 5000
+	siteDomainMappingExportMaxRows   = 10000
+	siteDomainMappingImportMaxErrors = 50
 )

 type SiteDomainMappingService struct {
@@ -63,6 +73,32 @@ type SiteDomainMappingInput struct {
 	IsActive          bool
 }

+type SiteDomainMappingImportInput struct {
+	Content []byte
+}
+
+type SiteDomainMappingImportError struct {
+	Row     int    `json:"row"`
+	Message string `json:"message"`
+}
+
+type SiteDomainMappingImportResult struct {
+	Total     int                            `json:"total"`
+	Created   int                            `json:"created"`
+	Updated   int                            `json:"updated"`
+	Skipped   int                            `json:"skipped"`
+	Failed    int                            `json:"failed"`
+	Duplicate int                            `json:"duplicate"`
+	Errors    []SiteDomainMappingImportError `json:"errors"`
+}
+
+type siteDomainMappingImportColumns struct {
+	Domain int
+	Key    int
+	Name   int
+	Active int
+}
+
 func toSiteDomainMappingView(item *domain.SiteDomainMapping) SiteDomainMappingView {
 	return SiteDomainMappingView{
 		ID:                item.ID,
@@ -102,6 +138,56 @@ func (s *SiteDomainMappingService) List(ctx context.Context, in SiteDomainMappin
 	return &SiteDomainMappingListResult{Items: views, Total: total, Page: page, Size: size}, nil
 }

+func (s *SiteDomainMappingService) ExportCSV(ctx context.Context, actor *Actor, in SiteDomainMappingListInput) ([]byte, error) {
+	items, err := s.mappings.Export(ctx, repository.SiteDomainMappingFilter{
+		Keyword:  strings.TrimSpace(in.Keyword),
+		IsActive: in.IsActive,
+		Limit:    siteDomainMappingExportMaxRows + 1,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(items) > siteDomainMappingExportMaxRows {
+		return nil, response.ErrBadRequest(
+			40059,
+			"site_domain_mapping_export_too_large",
+			fmt.Sprintf("单次最多导出 %d 行，请缩小筛选范围后重试", siteDomainMappingExportMaxRows),
+		)
+	}
+
+	var buf bytes.Buffer
+	buf.WriteString("\xEF\xBB\xBF")
+	writer := csv.NewWriter(&buf)
+	if err := writer.Write([]string{"registrable_domain", "site_key", "site_name", "is_active", "created_at", "updated_at"}); err != nil {
+		return nil, err
+	}
+	for _, item := range items {
+		if err := writer.Write([]string{
+			item.RegistrableDomain,
+			item.SiteKey,
+			item.SiteName,
+			strconv.FormatBool(item.IsActive),
+			item.CreatedAt.Format(time.RFC3339),
+			item.UpdatedAt.Format(time.RFC3339),
+		}); err != nil {
+			return nil, err
+		}
+	}
+	writer.Flush()
+	if err := writer.Error(); err != nil {
+		return nil, err
+	}
+
+	if actor != nil && s.audits != nil {
+		_ = s.audits.Append(ctx, actor.audit(ActionSiteDomainMappingExport, "site_domain_mapping", 0, map[string]any{
+			"count":     len(items),
+			"keyword":   strings.TrimSpace(in.Keyword),
+			"is_active": in.IsActive,
+		}))
+	}
+	return buf.Bytes(), nil
+}
+
 func (s *SiteDomainMappingService) Create(ctx context.Context, actor *Actor, in SiteDomainMappingInput) (*SiteDomainMappingView, error) {
 	normalized, err := normalizeSiteDomainMappingInput(in)
 	if err != nil {
@@ -126,6 +212,85 @@ func (s *SiteDomainMappingService) Create(ctx context.Context, actor *Actor, in
 	return &view, nil
 }

+func (s *SiteDomainMappingService) ImportCSV(ctx context.Context, actor *Actor, in SiteDomainMappingImportInput) (*SiteDomainMappingImportResult, error) {
+	records, err := readSiteDomainMappingCSV(in.Content)
+	if err != nil {
+		return nil, err
+	}
+	if len(records) < 2 {
+		return nil, response.ErrBadRequest(40053, "empty_site_domain_mapping_import", "CSV 至少需要表头和一行数据")
+	}
+	if len(records)-1 > siteDomainMappingImportMaxRows {
+		return nil, response.ErrBadRequest(
+			40054,
+			"site_domain_mapping_import_too_large",
+			fmt.Sprintf("单次最多导入 %d 行", siteDomainMappingImportMaxRows),
+		)
+	}
+
+	columns, err := detectSiteDomainMappingImportColumns(records[0])
+	if err != nil {
+		return nil, err
+	}
+
+	result := &SiteDomainMappingImportResult{
+		Total:  len(records) - 1,
+		Errors: make([]SiteDomainMappingImportError, 0),
+	}
+	seen := map[string]int{}
+	for rowIndex, record := range records[1:] {
+		rowNumber := rowIndex + 2
+		if isBlankCSVRecord(record) {
+			result.Skipped++
+			continue
+		}
+		input, rowErr := siteDomainMappingInputFromCSVRecord(record, columns)
+		if rowErr != nil {
+			appendSiteDomainMappingImportError(result, rowNumber, rowErr.Error())
+			continue
+		}
+		normalized, rowErr := normalizeSiteDomainMappingInput(input)
+		if rowErr != nil {
+			appendSiteDomainMappingImportError(result, rowNumber, response.Normalize(rowErr).Detail)
+			continue
+		}
+		if firstRow, ok := seen[normalized.RegistrableDomain]; ok {
+			result.Duplicate++
+			result.Skipped++
+			appendSiteDomainMappingImportWarning(result, rowNumber, fmt.Sprintf("域名与第 %d 行重复，已跳过", firstRow))
+			continue
+		}
+		seen[normalized.RegistrableDomain] = rowNumber
+
+		upserted, rowErr := s.mappings.Upsert(ctx, repository.UpsertSiteDomainMappingInput{
+			RegistrableDomain: normalized.RegistrableDomain,
+			SiteKey:           normalized.SiteKey,
+			SiteName:          normalized.SiteName,
+			IsActive:          normalized.IsActive,
+		})
+		if rowErr != nil {
+			appendSiteDomainMappingImportError(result, rowNumber, rowErr.Error())
+			continue
+		}
+		if upserted.Created {
+			result.Created++
+		} else {
+			result.Updated++
+		}
+	}
+	if actor != nil && s.audits != nil {
+		_ = s.audits.Append(ctx, actor.audit(ActionSiteDomainMappingImport, "site_domain_mapping", 0, map[string]any{
+			"total":     result.Total,
+			"created":   result.Created,
+			"updated":   result.Updated,
+			"skipped":   result.Skipped,
+			"failed":    result.Failed,
+			"duplicate": result.Duplicate,
+		}))
+	}
+	return result, nil
+}
+
 func (s *SiteDomainMappingService) Update(ctx context.Context, actor *Actor, id int64, in SiteDomainMappingInput) (*SiteDomainMappingView, error) {
 	normalized, err := normalizeSiteDomainMappingInput(in)
 	if err != nil {
@@ -240,6 +405,130 @@ func normalizeDomainName(raw string) string {
 	return strings.Trim(strings.TrimSuffix(value, "."), ".")
 }

+func readSiteDomainMappingCSV(content []byte) ([][]string, error) {
+	content = bytes.TrimPrefix(content, []byte("\xEF\xBB\xBF"))
+	reader := csv.NewReader(bytes.NewReader(content))
+	reader.FieldsPerRecord = -1
+	reader.TrimLeadingSpace = true
+	records, err := reader.ReadAll()
+	if err != nil {
+		return nil, response.ErrBadRequest(40055, "invalid_site_domain_mapping_csv", "CSV 文件格式无效："+err.Error())
+	}
+	return records, nil
+}
+
+func detectSiteDomainMappingImportColumns(header []string) (siteDomainMappingImportColumns, error) {
+	columns := siteDomainMappingImportColumns{
+		Domain: -1,
+		Key:    -1,
+		Name:   -1,
+		Active: -1,
+	}
+	for index, raw := range header {
+		switch normalizeSiteDomainMappingCSVHeader(raw) {
+		case "registrable_domain":
+			columns.Domain = index
+		case "site_key":
+			columns.Key = index
+		case "site_name":
+			columns.Name = index
+		case "is_active":
+			columns.Active = index
+		}
+	}
+	if columns.Domain < 0 || columns.Name < 0 {
+		return columns, response.ErrBadRequest(
+			40056,
+			"invalid_site_domain_mapping_csv_header",
+			"CSV 表头至少需要 registrable_domain/site_name，或中文表头：匹配域名/中文名",
+		)
+	}
+	return columns, nil
+}
+
+func normalizeSiteDomainMappingCSVHeader(header string) string {
+	key := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(header, "\ufeff")))
+	key = strings.ReplaceAll(key, " ", "_")
+	key = strings.ReplaceAll(key, "-", "_")
+	switch key {
+	case "registrable_domain", "domain", "matched_domain", "match_domain", "host", "域名", "匹配域名", "站点域名":
+		return "registrable_domain"
+	case "site_key", "key", "sitekey", "站点key", "站点_key":
+		return "site_key"
+	case "site_name", "sitename", "name", "中文名", "站点名", "站点名称", "网站名称":
+		return "site_name"
+	case "is_active", "active", "enabled", "status", "状态", "启用", "是否启用":
+		return "is_active"
+	default:
+		return key
+	}
+}
+
+func siteDomainMappingInputFromCSVRecord(record []string, columns siteDomainMappingImportColumns) (SiteDomainMappingInput, error) {
+	active := true
+	if columns.Active >= 0 {
+		parsed, err := parseSiteDomainMappingActive(csvCell(record, columns.Active))
+		if err != nil {
+			return SiteDomainMappingInput{}, err
+		}
+		active = parsed
+	}
+	return SiteDomainMappingInput{
+		RegistrableDomain: csvCell(record, columns.Domain),
+		SiteKey:           csvCell(record, columns.Key),
+		SiteName:          csvCell(record, columns.Name),
+		IsActive:          active,
+	}, nil
+}
+
+func parseSiteDomainMappingActive(value string) (bool, error) {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	if normalized == "" {
+		return true, nil
+	}
+	switch normalized {
+	case "true", "1", "yes", "y", "enabled", "enable", "active", "启用", "是":
+		return true, nil
+	case "false", "0", "no", "n", "disabled", "disable", "inactive", "停用", "否":
+		return false, nil
+	default:
+		return false, fmt.Errorf("状态只能填写启用/停用、true/false 或 1/0")
+	}
+}
+
+func csvCell(record []string, index int) string {
+	if index < 0 || index >= len(record) {
+		return ""
+	}
+	return strings.TrimSpace(record[index])
+}
+
+func isBlankCSVRecord(record []string) bool {
+	for _, value := range record {
+		if strings.TrimSpace(value) != "" {
+			return false
+		}
+	}
+	return true
+}
+
+func appendSiteDomainMappingImportError(result *SiteDomainMappingImportResult, row int, message string) {
+	if message == "" {
+		message = "导入失败"
+	}
+	result.Failed++
+	appendSiteDomainMappingImportWarning(result, row, message)
+}
+
+func appendSiteDomainMappingImportWarning(result *SiteDomainMappingImportResult, row int, message string) {
+	if message == "" {
+		message = "导入失败"
+	}
+	if len(result.Errors) < siteDomainMappingImportMaxErrors {
+		result.Errors = append(result.Errors, SiteDomainMappingImportError{Row: row, Message: message})
+	}
+}
+
 func isDuplicateSiteDomainMapping(err error) bool {
 	var pgErr *pgconn.PgError
 	return errors.As(err, &pgErr) && pgErr.Code == "23505"