feat(ops): add object storage management and site-mapping CSV import/export

Ship the Ops backstage 对象存储 page (browse / upload / move / delete /
folder ops) backed by a new object-storage service + handler, with the
shared storage client extended with List/Stat/Copy/Usage/DownloadURL so
both MinIO and Aliyun providers cover the new surface. Add
ForcePathStyle to the object-storage config and treat r2/s3/custom_s3
as external providers so Cloudflare R2 and other S3-compatibles can
share the MinIO client path without spinning up the bundled MinIO.

Also add CSV import/export + template download to the site-domain
mappings page, raise gin MaxMultipartMemory to 8 MiB for the new
multipart endpoints, register Ops swagger routes, and extend the
descriptions-parity test to cover the ops router.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-26 01:19:01 +08:00
parent 5f6e9f11da
commit 7d8e82c69f
37 changed files with 5111 additions and 66 deletions
+178 -2
View File
@@ -5,6 +5,8 @@ import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
@@ -296,6 +298,180 @@ func (c *aliyunClient) PublicURL(objectKey string) (string, error) {
return buildPublicURL(c.cfg, objectKey, true)
}
func (c *aliyunClient) PublicReadableURL(objectKey string) (string, error) {
if err := c.Validate(); err != nil {
return "", err
}
publicRead, err := c.isPublicReadable()
if err != nil {
return "", err
}
if !publicRead {
return "", ErrObjectNotPublic
}
return buildPublicURL(c.cfg, objectKey, true)
}
func (c *aliyunClient) DownloadURL(objectKey string) (string, error) {
if err := c.Validate(); err != nil {
return "", err
}
return c.signedURL(objectKey, oss.ResponseContentDisposition(downloadContentDisposition(objectKey)))
}
func (c *aliyunClient) Usage(ctx context.Context) (*UsageInfo, error) {
if err := c.Validate(); err != nil {
return nil, err
}
stat, err := c.client.GetBucketStat(c.bucket)
if err != nil {
c.logError(ctx, "aliyun get bucket stat failed",
zap.String("bucket", c.bucket),
zap.Error(err),
)
return nil, fmt.Errorf("get aliyun bucket stat: %w", err)
}
return &UsageInfo{
Available: true,
Provider: "aliyun",
StorageSize: stat.Storage,
ObjectCount: stat.ObjectCount,
}, nil
}
func (c *aliyunClient) List(ctx context.Context, in ListInput) (*ListResult, error) {
if err := c.Validate(); err != nil {
return nil, err
}
bucket, err := c.client.Bucket(c.bucket)
if err != nil {
return nil, fmt.Errorf("get aliyun bucket: %w", err)
}
limit := in.Limit
if limit <= 0 || limit > 1000 {
limit = 100
}
delimiter := strings.TrimSpace(in.Delimiter)
if delimiter == "" && !in.Recursive {
delimiter = "/"
}
options := []oss.Option{
oss.Prefix(normalizeObjectKeyPath(in.Prefix)),
oss.MaxKeys(limit),
}
if delimiter != "" {
options = append(options, oss.Delimiter(delimiter))
}
if token := strings.TrimSpace(in.ContinuationToken); token != "" {
options = append(options, oss.ContinuationToken(token))
}
if startAfter := normalizeObjectKeyPath(in.StartAfter); startAfter != "" {
options = append(options, oss.StartAfter(startAfter))
}
result, err := bucket.ListObjectsV2(options...)
if err != nil {
c.logError(ctx, "aliyun list objects failed",
zap.String("bucket", c.bucket),
zap.String("prefix", in.Prefix),
zap.Error(err),
)
return nil, fmt.Errorf("list aliyun objects: %w", err)
}
out := &ListResult{
Objects: make([]ObjectInfo, 0, len(result.Objects)),
CommonPrefixes: append([]string(nil), result.CommonPrefixes...),
ContinuationToken: result.ContinuationToken,
NextContinuationToken: result.NextContinuationToken,
IsTruncated: result.IsTruncated,
Prefix: result.Prefix,
Delimiter: result.Delimiter,
}
for _, item := range result.Objects {
out.Objects = append(out.Objects, ObjectInfo{
Key: item.Key,
Size: item.Size,
ETag: strings.Trim(item.ETag, `"`),
LastModified: item.LastModified,
StorageClass: item.StorageClass,
})
}
return out, nil
}
func (c *aliyunClient) Stat(ctx context.Context, objectKey string) (*ObjectInfo, error) {
if err := c.Validate(); err != nil {
return nil, err
}
objectKey = normalizeObjectKeyPath(objectKey)
if objectKey == "" {
return nil, fmt.Errorf("object key is required")
}
bucket, err := c.client.Bucket(c.bucket)
if err != nil {
return nil, fmt.Errorf("get aliyun bucket: %w", err)
}
header, err := bucket.GetObjectDetailedMeta(objectKey)
if err != nil {
c.logError(ctx, "aliyun stat object failed",
zap.String("bucket", c.bucket),
zap.String("object_key", objectKey),
zap.Error(err),
)
if isAliyunObjectNotFound(err) {
return nil, fmt.Errorf("%w: %s", ErrObjectNotFound, objectKey)
}
return nil, fmt.Errorf("stat aliyun object: %w", err)
}
return aliyunHeaderToObjectInfo(objectKey, header), nil
}
func (c *aliyunClient) Copy(ctx context.Context, sourceKey, destinationKey string) error {
if err := c.Validate(); err != nil {
return err
}
sourceKey = normalizeObjectKeyPath(sourceKey)
destinationKey = normalizeObjectKeyPath(destinationKey)
if sourceKey == "" || destinationKey == "" {
return fmt.Errorf("source and destination object keys are required")
}
if sourceKey == destinationKey {
return nil
}
bucket, err := c.client.Bucket(c.bucket)
if err != nil {
return fmt.Errorf("get aliyun bucket: %w", err)
}
if _, err := bucket.CopyObject(sourceKey, destinationKey); err != nil {
c.logError(ctx, "aliyun copy object failed",
zap.String("bucket", c.bucket),
zap.String("source_key", sourceKey),
zap.String("destination_key", destinationKey),
zap.Error(err),
)
if isAliyunObjectNotFound(err) {
return fmt.Errorf("%w: %s", ErrObjectNotFound, sourceKey)
}
return fmt.Errorf("copy aliyun object: %w", err)
}
return nil
}
func aliyunHeaderToObjectInfo(objectKey string, header http.Header) *ObjectInfo {
size, _ := strconv.ParseInt(strings.TrimSpace(header.Get(oss.HTTPHeaderContentLength)), 10, 64)
lastModified, _ := http.ParseTime(header.Get(oss.HTTPHeaderLastModified))
return &ObjectInfo{
Key: objectKey,
Size: size,
ETag: strings.Trim(header.Get(oss.HTTPHeaderEtag), `"`),
LastModified: lastModified,
ContentType: header.Get(oss.HTTPHeaderContentType),
StorageClass: header.Get(oss.HTTPHeaderOssStorageClass),
}
}
func (c *aliyunClient) isPublicReadable() (bool, error) {
objectACL := strings.ToLower(strings.TrimSpace(c.cfg.ObjectACL))
if isPublicReadACL(objectACL) {
@@ -317,7 +493,7 @@ func (c *aliyunClient) isPublicReadable() (bool, error) {
return isPublicReadACL(result.ACL), nil
}
func (c *aliyunClient) signedURL(objectKey string) (string, error) {
func (c *aliyunClient) signedURL(objectKey string, options ...oss.Option) (string, error) {
objectKey = normalizeObjectKeyPath(objectKey)
if objectKey == "" {
return "", fmt.Errorf("object key is required")
@@ -330,7 +506,7 @@ func (c *aliyunClient) signedURL(objectKey string) (string, error) {
if ttl <= 0 {
ttl = 30 * time.Minute
}
return bucket.SignURL(objectKey, oss.HTTPGet, int64(ttl.Seconds()))
return bucket.SignURL(objectKey, oss.HTTPGet, int64(ttl.Seconds()), options...)
}
func isPublicReadACL(acl string) bool {
+75 -1
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"strings"
"time"
"go.uber.org/zap"
@@ -13,6 +14,8 @@ import (
var ErrNotConfigured = errors.New("object storage is not configured")
var ErrObjectNotFound = errors.New("object storage object not found")
var ErrObjectNotPublic = errors.New("object storage object is not publicly readable")
var ErrUsageUnsupported = errors.New("object storage usage stat is not supported")
type Client interface {
Validate() error
@@ -21,6 +24,57 @@ type Client interface {
Exists(ctx context.Context, objectKey string) (bool, error)
Delete(ctx context.Context, objectKey string) error
PublicURL(objectKey string) (string, error)
DownloadURL(objectKey string) (string, error)
}
type ManagementClient interface {
Client
List(ctx context.Context, in ListInput) (*ListResult, error)
Stat(ctx context.Context, objectKey string) (*ObjectInfo, error)
Copy(ctx context.Context, sourceKey, destinationKey string) error
}
type UsageClient interface {
Usage(ctx context.Context) (*UsageInfo, error)
}
type PublicReadableClient interface {
PublicReadableURL(objectKey string) (string, error)
}
type ListInput struct {
Prefix string
ContinuationToken string
StartAfter string
Delimiter string
Limit int
Recursive bool
}
type ListResult struct {
Objects []ObjectInfo `json:"objects"`
CommonPrefixes []string `json:"common_prefixes"`
ContinuationToken string `json:"continuation_token,omitempty"`
NextContinuationToken string `json:"next_continuation_token,omitempty"`
IsTruncated bool `json:"is_truncated"`
Prefix string `json:"prefix"`
Delimiter string `json:"delimiter"`
}
type ObjectInfo struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag string `json:"etag,omitempty"`
LastModified time.Time `json:"last_modified"`
ContentType string `json:"content_type,omitempty"`
StorageClass string `json:"storage_class,omitempty"`
}
type UsageInfo struct {
Available bool `json:"available"`
Provider string `json:"provider"`
StorageSize int64 `json:"storage_size"`
ObjectCount int64 `json:"object_count"`
}
func New(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
@@ -28,7 +82,7 @@ func New(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
switch provider {
case "", "disabled":
return disabledClient{reason: "object storage is disabled"}
case "minio", "mino":
case "minio", "mino", "r2", "cloudflare_r2", "cloudflare-r2", "s3", "aws_s3", "aws-s3", "custom", "custom_s3", "custom-s3":
return NewMinIOClient(cfg, logger)
case "aliyun", "aliyun_oss", "aliyun-oss", "oss":
return NewAliyunClient(cfg, logger)
@@ -67,3 +121,23 @@ func (c disabledClient) Delete(context.Context, string) error {
func (c disabledClient) PublicURL(string) (string, error) {
return "", c.Validate()
}
func (c disabledClient) DownloadURL(string) (string, error) {
return "", c.Validate()
}
func (c disabledClient) List(context.Context, ListInput) (*ListResult, error) {
return nil, c.Validate()
}
func (c disabledClient) Stat(context.Context, string) (*ObjectInfo, error) {
return nil, c.Validate()
}
func (c disabledClient) Copy(context.Context, string, string) error {
return c.Validate()
}
func (c disabledClient) Usage(context.Context) (*UsageInfo, error) {
return nil, c.Validate()
}
+200 -4
View File
@@ -5,7 +5,9 @@ import (
"context"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -32,10 +34,17 @@ func NewMinIOClient(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
return disabledClient{reason: "missing object_storage minio endpoint/access_key/secret_key/bucket"}
}
region := normalizeMinIORegion(cfg)
bucketLookup := minio.BucketLookupAuto
if cfg.ForcePathStyle || isPathStyleProvider(cfg.Provider) {
bucketLookup = minio.BucketLookupPath
}
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: cfg.UseSSL,
Region: strings.TrimSpace(cfg.Region),
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: cfg.UseSSL,
Region: region,
BucketLookup: bucketLookup,
})
if err != nil {
return disabledClient{reason: fmt.Sprintf("init minio client failed: %v", err)}
@@ -44,12 +53,41 @@ func NewMinIOClient(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
return &minioClient{
client: client,
bucket: bucket,
region: strings.TrimSpace(cfg.Region),
region: region,
logger: logger,
cfg: cfg,
}
}
func normalizeMinIORegion(cfg config.ObjectStorageConfig) string {
region := strings.TrimSpace(cfg.Region)
if region != "" {
return region
}
if isR2Provider(cfg.Provider) {
return "auto"
}
return ""
}
func isR2Provider(provider string) bool {
switch strings.ToLower(strings.TrimSpace(provider)) {
case "r2", "cloudflare_r2", "cloudflare-r2":
return true
default:
return false
}
}
func isPathStyleProvider(provider string) bool {
switch strings.ToLower(strings.TrimSpace(provider)) {
case "r2", "cloudflare_r2", "cloudflare-r2":
return true
default:
return false
}
}
func (c *minioClient) Validate() error {
if c == nil || c.client == nil {
return fmt.Errorf("%w: minio client is not initialized", ErrNotConfigured)
@@ -198,9 +236,167 @@ func (c *minioClient) PublicURL(objectKey string) (string, error) {
if err := c.Validate(); err != nil {
return "", err
}
if !c.isPublicReadable() {
return c.DownloadURL(objectKey)
}
return buildPublicURL(c.cfg, objectKey, false)
}
func (c *minioClient) PublicReadableURL(objectKey string) (string, error) {
if err := c.Validate(); err != nil {
return "", err
}
if !c.isPublicReadable() {
return "", ErrObjectNotPublic
}
return buildPublicURL(c.cfg, objectKey, false)
}
func (c *minioClient) isPublicReadable() bool {
return isPublicReadACL(c.cfg.ObjectACL) || isPublicReadACL(c.cfg.BucketACL)
}
func (c *minioClient) DownloadURL(objectKey string) (string, error) {
if err := c.Validate(); err != nil {
return "", err
}
objectKey = normalizeObjectKeyPath(objectKey)
if objectKey == "" {
return "", fmt.Errorf("object key is required")
}
ttl := c.cfg.SignedURLTTL
if ttl <= 0 {
ttl = 30 * time.Minute
}
params := url.Values{}
params.Set("response-content-disposition", downloadContentDisposition(objectKey))
u, err := c.client.PresignedGetObject(context.Background(), c.bucket, objectKey, ttl, params)
if err != nil {
return "", fmt.Errorf("presign minio download url: %w", err)
}
return u.String(), nil
}
func (c *minioClient) List(ctx context.Context, in ListInput) (*ListResult, error) {
if err := c.Validate(); err != nil {
return nil, err
}
limit := in.Limit
if limit <= 0 || limit > 1000 {
limit = 100
}
delimiter := strings.TrimSpace(in.Delimiter)
if delimiter == "" && !in.Recursive {
delimiter = "/"
}
core := minio.Core{Client: c.client}
result, err := core.ListObjectsV2(
c.bucket,
normalizeObjectKeyPath(in.Prefix),
normalizeObjectKeyPath(in.StartAfter),
strings.TrimSpace(in.ContinuationToken),
delimiter,
limit,
)
if err != nil {
c.logError(ctx, "minio list objects failed",
zap.String("bucket", c.bucket),
zap.String("prefix", in.Prefix),
zap.Error(err),
)
return nil, fmt.Errorf("list minio objects: %w", err)
}
out := &ListResult{
Objects: make([]ObjectInfo, 0, len(result.Contents)),
CommonPrefixes: make([]string, 0, len(result.CommonPrefixes)),
ContinuationToken: result.ContinuationToken,
NextContinuationToken: result.NextContinuationToken,
IsTruncated: result.IsTruncated,
Prefix: result.Prefix,
Delimiter: result.Delimiter,
}
for _, item := range result.Contents {
out.Objects = append(out.Objects, ObjectInfo{
Key: item.Key,
Size: item.Size,
ETag: strings.Trim(item.ETag, `"`),
LastModified: item.LastModified,
ContentType: item.ContentType,
StorageClass: item.StorageClass,
})
}
for _, prefix := range result.CommonPrefixes {
if prefix.Prefix != "" {
out.CommonPrefixes = append(out.CommonPrefixes, prefix.Prefix)
}
}
return out, nil
}
func (c *minioClient) Stat(ctx context.Context, objectKey string) (*ObjectInfo, error) {
if err := c.Validate(); err != nil {
return nil, err
}
objectKey = normalizeObjectKeyPath(objectKey)
if objectKey == "" {
return nil, fmt.Errorf("object key is required")
}
item, err := c.client.StatObject(ctx, c.bucket, objectKey, minio.StatObjectOptions{})
if err != nil {
c.logError(ctx, "minio stat object failed",
zap.String("bucket", c.bucket),
zap.String("object_key", objectKey),
zap.Error(err),
)
if isMinIOObjectNotFound(err) {
return nil, fmt.Errorf("%w: %s", ErrObjectNotFound, objectKey)
}
return nil, fmt.Errorf("stat minio object: %w", err)
}
return &ObjectInfo{
Key: item.Key,
Size: item.Size,
ETag: strings.Trim(item.ETag, `"`),
LastModified: item.LastModified,
ContentType: item.ContentType,
StorageClass: item.StorageClass,
}, nil
}
func (c *minioClient) Copy(ctx context.Context, sourceKey, destinationKey string) error {
if err := c.Validate(); err != nil {
return err
}
sourceKey = normalizeObjectKeyPath(sourceKey)
destinationKey = normalizeObjectKeyPath(destinationKey)
if sourceKey == "" || destinationKey == "" {
return fmt.Errorf("source and destination object keys are required")
}
if sourceKey == destinationKey {
return nil
}
_, err := c.client.CopyObject(ctx,
minio.CopyDestOptions{Bucket: c.bucket, Object: destinationKey},
minio.CopySrcOptions{Bucket: c.bucket, Object: sourceKey},
)
if err != nil {
c.logError(ctx, "minio copy object failed",
zap.String("bucket", c.bucket),
zap.String("source_key", sourceKey),
zap.String("destination_key", destinationKey),
zap.Error(err),
)
if isMinIOObjectNotFound(err) {
return fmt.Errorf("%w: %s", ErrObjectNotFound, sourceKey)
}
return fmt.Errorf("copy minio object: %w", err)
}
return nil
}
func (c *minioClient) logError(ctx context.Context, msg string, fields ...zap.Field) {
if c == nil || c.logger == nil {
return
@@ -47,6 +47,55 @@ func (c *ReloadableClient) PublicURL(objectKey string) (string, error) {
return c.snapshot().PublicURL(objectKey)
}
func (c *ReloadableClient) DownloadURL(objectKey string) (string, error) {
return c.snapshot().DownloadURL(objectKey)
}
func (c *ReloadableClient) List(ctx context.Context, in ListInput) (*ListResult, error) {
client := c.snapshot()
management, ok := client.(ManagementClient)
if !ok {
return nil, ErrNotConfigured
}
return management.List(ctx, in)
}
func (c *ReloadableClient) Stat(ctx context.Context, objectKey string) (*ObjectInfo, error) {
client := c.snapshot()
management, ok := client.(ManagementClient)
if !ok {
return nil, ErrNotConfigured
}
return management.Stat(ctx, objectKey)
}
func (c *ReloadableClient) Copy(ctx context.Context, sourceKey, destinationKey string) error {
client := c.snapshot()
management, ok := client.(ManagementClient)
if !ok {
return ErrNotConfigured
}
return management.Copy(ctx, sourceKey, destinationKey)
}
func (c *ReloadableClient) Usage(ctx context.Context) (*UsageInfo, error) {
client := c.snapshot()
usageClient, ok := client.(UsageClient)
if !ok {
return nil, ErrUsageUnsupported
}
return usageClient.Usage(ctx)
}
func (c *ReloadableClient) PublicReadableURL(objectKey string) (string, error) {
client := c.snapshot()
publicClient, ok := client.(PublicReadableClient)
if !ok {
return "", ErrObjectNotPublic
}
return publicClient.PublicReadableURL(objectKey)
}
func (c *ReloadableClient) snapshot() Client {
if c == nil {
return disabledClient{reason: "object storage client is nil"}
@@ -3,6 +3,7 @@ package objectstorage
import (
"fmt"
"net/url"
"path/filepath"
"strings"
"github.com/geo-platform/tenant-api/internal/shared/config"
@@ -65,6 +66,66 @@ func buildPublicURL(cfg config.ObjectStorageConfig, objectKey string, bucketInHo
return endpointURL.String(), nil
}
func DownloadContentDisposition(objectKey string) string {
name := strings.TrimSpace(filepath.Base(strings.TrimRight(objectKey, "/")))
if name == "" || name == "." || name == "/" || name == "\\" {
name = "object"
}
name = strings.Map(func(r rune) rune {
if r == '"' || r == '\\' || r == '\r' || r == '\n' {
return -1
}
return r
}, name)
if name == "" {
name = "object"
}
return fmt.Sprintf(
`attachment; filename="%s"; filename*=UTF-8''%s`,
asciiContentDispositionFileName(name),
url.PathEscape(name),
)
}
func downloadContentDisposition(objectKey string) string {
return DownloadContentDisposition(objectKey)
}
func asciiContentDispositionFileName(name string) string {
var builder strings.Builder
for _, r := range name {
if r < 0x20 || r >= 0x7f || r == '"' || r == '\\' {
continue
}
builder.WriteRune(r)
}
fallback := strings.TrimSpace(builder.String())
if fallback == "" || fallback == "." || fallback == ".." || strings.HasPrefix(fallback, ".") {
return "object" + safeASCIIExtension(name)
}
return fallback
}
func safeASCIIExtension(name string) string {
ext := filepath.Ext(name)
if ext == "" || ext == "." {
return ""
}
var builder strings.Builder
for _, r := range ext {
if r < 0x20 || r >= 0x7f || r == '"' || r == '\\' {
continue
}
builder.WriteRune(r)
}
ext = strings.TrimSpace(builder.String())
if !strings.HasPrefix(ext, ".") || ext == "." {
return ""
}
return ext
}
func parseURLWithScheme(raw string, defaultScheme string) (*url.URL, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -51,3 +51,23 @@ func TestNormalizeObjectKeyPathDecodesEncodedSegments(t *testing.T) {
t.Fatalf("object key = %q, want %q", got, want)
}
}
func TestDownloadContentDispositionUsesRFC5987FileName(t *testing.T) {
t.Parallel()
got := DownloadContentDisposition("reports/省心推 安装包.zip")
want := `attachment; filename="object.zip"; filename*=UTF-8''%E7%9C%81%E5%BF%83%E6%8E%A8%20%E5%AE%89%E8%A3%85%E5%8C%85.zip`
if got != want {
t.Fatalf("content disposition = %q, want %q", got, want)
}
}
func TestDownloadContentDispositionKeepsASCIIFileNameFallback(t *testing.T) {
t.Parallel()
got := DownloadContentDisposition(`reports/report "final".csv`)
want := `attachment; filename="report final.csv"; filename*=UTF-8''report%20final.csv`
if got != want {
t.Fatalf("content disposition = %q, want %q", got, want)
}
}