feat(ops): add object storage management and site-mapping CSV import/export

Ship the Ops backstage 对象存储 page (browse / upload / move / delete / folder ops) backed by a new object-storage service + handler, with the shared storage client extended with List/Stat/Copy/Usage/DownloadURL so both MinIO and Aliyun providers cover the new surface. Add ForcePathStyle to the object-storage config and treat r2/s3/custom_s3 as external providers so Cloudflare R2 and other S3-compatibles can share the MinIO client path without spinning up the bundled MinIO. Also add CSV import/export + template download to the site-domain mappings page, raise gin MaxMultipartMemory to 8 MiB for the new multipart endpoints, register Ops swagger routes, and extend the descriptions-parity test to cover the ops router. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-26 01:19:01 +08:00
parent 5f6e9f11da
commit 7d8e82c69f
37 changed files with 5111 additions and 66 deletions
@@ -5,7 +5,9 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"strings"
+	"time"

 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -32,10 +34,17 @@ func NewMinIOClient(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
 		return disabledClient{reason: "missing object_storage minio endpoint/access_key/secret_key/bucket"}
 	}

+	region := normalizeMinIORegion(cfg)
+	bucketLookup := minio.BucketLookupAuto
+	if cfg.ForcePathStyle || isPathStyleProvider(cfg.Provider) {
+		bucketLookup = minio.BucketLookupPath
+	}
+
 	client, err := minio.New(endpoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
-		Secure: cfg.UseSSL,
-		Region: strings.TrimSpace(cfg.Region),
+		Creds:        credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure:       cfg.UseSSL,
+		Region:       region,
+		BucketLookup: bucketLookup,
 	})
 	if err != nil {
 		return disabledClient{reason: fmt.Sprintf("init minio client failed: %v", err)}
@@ -44,12 +53,41 @@ func NewMinIOClient(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
 	return &minioClient{
 		client: client,
 		bucket: bucket,
-		region: strings.TrimSpace(cfg.Region),
+		region: region,
 		logger: logger,
 		cfg:    cfg,
 	}
 }

+func normalizeMinIORegion(cfg config.ObjectStorageConfig) string {
+	region := strings.TrimSpace(cfg.Region)
+	if region != "" {
+		return region
+	}
+	if isR2Provider(cfg.Provider) {
+		return "auto"
+	}
+	return ""
+}
+
+func isR2Provider(provider string) bool {
+	switch strings.ToLower(strings.TrimSpace(provider)) {
+	case "r2", "cloudflare_r2", "cloudflare-r2":
+		return true
+	default:
+		return false
+	}
+}
+
+func isPathStyleProvider(provider string) bool {
+	switch strings.ToLower(strings.TrimSpace(provider)) {
+	case "r2", "cloudflare_r2", "cloudflare-r2":
+		return true
+	default:
+		return false
+	}
+}
+
 func (c *minioClient) Validate() error {
 	if c == nil || c.client == nil {
 		return fmt.Errorf("%w: minio client is not initialized", ErrNotConfigured)
@@ -198,9 +236,167 @@ func (c *minioClient) PublicURL(objectKey string) (string, error) {
 	if err := c.Validate(); err != nil {
 		return "", err
 	}
+	if !c.isPublicReadable() {
+		return c.DownloadURL(objectKey)
+	}
 	return buildPublicURL(c.cfg, objectKey, false)
 }

+func (c *minioClient) PublicReadableURL(objectKey string) (string, error) {
+	if err := c.Validate(); err != nil {
+		return "", err
+	}
+	if !c.isPublicReadable() {
+		return "", ErrObjectNotPublic
+	}
+	return buildPublicURL(c.cfg, objectKey, false)
+}
+
+func (c *minioClient) isPublicReadable() bool {
+	return isPublicReadACL(c.cfg.ObjectACL) || isPublicReadACL(c.cfg.BucketACL)
+}
+
+func (c *minioClient) DownloadURL(objectKey string) (string, error) {
+	if err := c.Validate(); err != nil {
+		return "", err
+	}
+	objectKey = normalizeObjectKeyPath(objectKey)
+	if objectKey == "" {
+		return "", fmt.Errorf("object key is required")
+	}
+	ttl := c.cfg.SignedURLTTL
+	if ttl <= 0 {
+		ttl = 30 * time.Minute
+	}
+	params := url.Values{}
+	params.Set("response-content-disposition", downloadContentDisposition(objectKey))
+	u, err := c.client.PresignedGetObject(context.Background(), c.bucket, objectKey, ttl, params)
+	if err != nil {
+		return "", fmt.Errorf("presign minio download url: %w", err)
+	}
+	return u.String(), nil
+}
+
+func (c *minioClient) List(ctx context.Context, in ListInput) (*ListResult, error) {
+	if err := c.Validate(); err != nil {
+		return nil, err
+	}
+
+	limit := in.Limit
+	if limit <= 0 || limit > 1000 {
+		limit = 100
+	}
+	delimiter := strings.TrimSpace(in.Delimiter)
+	if delimiter == "" && !in.Recursive {
+		delimiter = "/"
+	}
+
+	core := minio.Core{Client: c.client}
+	result, err := core.ListObjectsV2(
+		c.bucket,
+		normalizeObjectKeyPath(in.Prefix),
+		normalizeObjectKeyPath(in.StartAfter),
+		strings.TrimSpace(in.ContinuationToken),
+		delimiter,
+		limit,
+	)
+	if err != nil {
+		c.logError(ctx, "minio list objects failed",
+			zap.String("bucket", c.bucket),
+			zap.String("prefix", in.Prefix),
+			zap.Error(err),
+		)
+		return nil, fmt.Errorf("list minio objects: %w", err)
+	}
+
+	out := &ListResult{
+		Objects:               make([]ObjectInfo, 0, len(result.Contents)),
+		CommonPrefixes:        make([]string, 0, len(result.CommonPrefixes)),
+		ContinuationToken:     result.ContinuationToken,
+		NextContinuationToken: result.NextContinuationToken,
+		IsTruncated:           result.IsTruncated,
+		Prefix:                result.Prefix,
+		Delimiter:             result.Delimiter,
+	}
+	for _, item := range result.Contents {
+		out.Objects = append(out.Objects, ObjectInfo{
+			Key:          item.Key,
+			Size:         item.Size,
+			ETag:         strings.Trim(item.ETag, `"`),
+			LastModified: item.LastModified,
+			ContentType:  item.ContentType,
+			StorageClass: item.StorageClass,
+		})
+	}
+	for _, prefix := range result.CommonPrefixes {
+		if prefix.Prefix != "" {
+			out.CommonPrefixes = append(out.CommonPrefixes, prefix.Prefix)
+		}
+	}
+	return out, nil
+}
+
+func (c *minioClient) Stat(ctx context.Context, objectKey string) (*ObjectInfo, error) {
+	if err := c.Validate(); err != nil {
+		return nil, err
+	}
+	objectKey = normalizeObjectKeyPath(objectKey)
+	if objectKey == "" {
+		return nil, fmt.Errorf("object key is required")
+	}
+	item, err := c.client.StatObject(ctx, c.bucket, objectKey, minio.StatObjectOptions{})
+	if err != nil {
+		c.logError(ctx, "minio stat object failed",
+			zap.String("bucket", c.bucket),
+			zap.String("object_key", objectKey),
+			zap.Error(err),
+		)
+		if isMinIOObjectNotFound(err) {
+			return nil, fmt.Errorf("%w: %s", ErrObjectNotFound, objectKey)
+		}
+		return nil, fmt.Errorf("stat minio object: %w", err)
+	}
+	return &ObjectInfo{
+		Key:          item.Key,
+		Size:         item.Size,
+		ETag:         strings.Trim(item.ETag, `"`),
+		LastModified: item.LastModified,
+		ContentType:  item.ContentType,
+		StorageClass: item.StorageClass,
+	}, nil
+}
+
+func (c *minioClient) Copy(ctx context.Context, sourceKey, destinationKey string) error {
+	if err := c.Validate(); err != nil {
+		return err
+	}
+	sourceKey = normalizeObjectKeyPath(sourceKey)
+	destinationKey = normalizeObjectKeyPath(destinationKey)
+	if sourceKey == "" || destinationKey == "" {
+		return fmt.Errorf("source and destination object keys are required")
+	}
+	if sourceKey == destinationKey {
+		return nil
+	}
+	_, err := c.client.CopyObject(ctx,
+		minio.CopyDestOptions{Bucket: c.bucket, Object: destinationKey},
+		minio.CopySrcOptions{Bucket: c.bucket, Object: sourceKey},
+	)
+	if err != nil {
+		c.logError(ctx, "minio copy object failed",
+			zap.String("bucket", c.bucket),
+			zap.String("source_key", sourceKey),
+			zap.String("destination_key", destinationKey),
+			zap.Error(err),
+		)
+		if isMinIOObjectNotFound(err) {
+			return fmt.Errorf("%w: %s", ErrObjectNotFound, sourceKey)
+		}
+		return fmt.Errorf("copy minio object: %w", err)
+	}
+	return nil
+}
+
 func (c *minioClient) logError(ctx context.Context, msg string, fields ...zap.Field) {
 	if c == nil || c.logger == nil {
 		return