Files
geo/server/internal/ops/app/object_storage.go
T
root 7d8e82c69f feat(ops): add object storage management and site-mapping CSV import/export
Ship the Ops backstage 对象存储 page (browse / upload / move / delete /
folder ops) backed by a new object-storage service + handler, with the
shared storage client extended with List/Stat/Copy/Usage/DownloadURL so
both MinIO and Aliyun providers cover the new surface. Add
ForcePathStyle to the object-storage config and treat r2/s3/custom_s3
as external providers so Cloudflare R2 and other S3-compatibles can
share the MinIO client path without spinning up the bundled MinIO.

Also add CSV import/export + template download to the site-domain
mappings page, raise gin MaxMultipartMemory to 8 MiB for the new
multipart endpoints, register Ops swagger routes, and extend the
descriptions-parity test to cover the ops router.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-26 01:19:01 +08:00

709 lines
22 KiB
Go

package app
import (
"context"
"errors"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/geo-platform/tenant-api/internal/shared/objectstorage"
"github.com/geo-platform/tenant-api/internal/shared/response"
"go.uber.org/zap"
)
const (
ActionObjectStorageUpload = "object_storage.upload"
ActionObjectStorageMove = "object_storage.move"
ActionObjectStorageDelete = "object_storage.delete"
ActionObjectStorageFolderCreate = "object_storage.folder.create"
ActionObjectStorageFolderDelete = "object_storage.folder.delete"
)
type ObjectStorageService struct {
storage objectstorage.ManagementClient
audits *AuditService
logger *zap.Logger
}
func NewObjectStorageService(storage objectstorage.Client, audits *AuditService) *ObjectStorageService {
management, _ := storage.(objectstorage.ManagementClient)
return &ObjectStorageService{storage: management, audits: audits}
}
func (s *ObjectStorageService) WithLogger(logger *zap.Logger) *ObjectStorageService {
if s == nil {
return s
}
s.logger = logger
if logger != nil && s.storage == nil {
logger.Warn("object storage backend does not implement ManagementClient, OSS console disabled")
}
return s
}
type ObjectStorageListInput struct {
Prefix string
Keyword string
ContinuationToken string
StartAfter string
Recursive bool
Size int
}
type ObjectStorageListResult struct {
Objects []objectstorage.ObjectInfo `json:"objects"`
CommonPrefixes []string `json:"common_prefixes"`
ContinuationToken string `json:"continuation_token,omitempty"`
NextContinuationToken string `json:"next_continuation_token,omitempty"`
IsTruncated bool `json:"is_truncated"`
Prefix string `json:"prefix"`
Keyword string `json:"keyword,omitempty"`
Recursive bool `json:"recursive"`
}
type ObjectStorageDetailResult struct {
objectstorage.ObjectInfo
PublicURL *string `json:"public_url,omitempty"`
PreviewURL *string `json:"preview_url,omitempty"`
}
type ObjectStorageUploadInput struct {
ObjectKey string
FileName string
Content []byte
ContentType string
Overwrite bool
}
type ObjectStorageUploadResult struct {
Object objectstorage.ObjectInfo `json:"object"`
}
type ObjectStorageMoveInput struct {
SourceKey string
DestinationKey string
Overwrite bool
DeleteSource bool
}
type ObjectStorageFolderCreateInput struct {
Prefix string
}
type ObjectStorageFolderCreateResult struct {
Prefix string `json:"prefix"`
}
type ObjectStorageFolderDeleteResult struct {
Prefix string `json:"prefix"`
DeletedCount int `json:"deleted_count"`
}
type ObjectStorageUsageResult struct {
Available bool `json:"available"`
Provider string `json:"provider,omitempty"`
StorageSize int64 `json:"storage_size"`
ObjectCount int64 `json:"object_count"`
}
func (s *ObjectStorageService) List(ctx context.Context, in ObjectStorageListInput) (*ObjectStorageListResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
size := in.Size
if size <= 0 || size > 1000 {
size = 100
}
prefix, err := normalizeOSSOptionalPrefix(in.Prefix)
if err != nil {
return nil, err
}
keyword := strings.TrimSpace(in.Keyword)
if keyword != "" && len([]rune(keyword)) > 200 {
return nil, response.ErrBadRequest(40093, "invalid_object_storage_keyword", "搜索关键字不能超过 200 个字")
}
listInput := objectstorage.ListInput{
Prefix: prefix,
ContinuationToken: strings.TrimSpace(in.ContinuationToken),
StartAfter: normalizeOSSOptionalToken(in.StartAfter),
Recursive: in.Recursive || keyword != "",
Limit: size,
}
result, err := s.storage.List(ctx, listInput)
if err != nil {
return nil, mapObjectStorageError(err, "object_storage_list_failed", "读取 OSS 对象列表失败")
}
objects := normalizeOSSListedObjects(prefix, result.Objects, in.Recursive || keyword != "")
commonPrefixes := normalizeOSSCommonPrefixes(prefix, result.CommonPrefixes, result.Objects, in.Recursive || keyword != "")
if shouldFallbackOSSDirectoryList(in, keyword, objects, commonPrefixes) {
listInput.Recursive = true
result, err = s.storage.List(ctx, listInput)
if err != nil {
return nil, mapObjectStorageError(err, "object_storage_list_failed", "读取 OSS 对象列表失败")
}
objects = normalizeOSSListedObjects(prefix, result.Objects, in.Recursive || keyword != "")
commonPrefixes = normalizeOSSCommonPrefixes(prefix, result.CommonPrefixes, result.Objects, in.Recursive || keyword != "")
}
if keyword != "" {
objects = filterOSSObjectsByKeyword(objects, keyword)
}
return &ObjectStorageListResult{
Objects: objects,
CommonPrefixes: commonPrefixes,
ContinuationToken: result.ContinuationToken,
NextContinuationToken: result.NextContinuationToken,
IsTruncated: result.IsTruncated,
Prefix: prefix,
Keyword: keyword,
Recursive: in.Recursive || keyword != "",
}, nil
}
func (s *ObjectStorageService) Usage(ctx context.Context) (*ObjectStorageUsageResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
usageClient, ok := s.storage.(objectstorage.UsageClient)
if !ok {
return &ObjectStorageUsageResult{Available: false}, nil
}
info, err := usageClient.Usage(ctx)
if errors.Is(err, objectstorage.ErrUsageUnsupported) {
return &ObjectStorageUsageResult{Available: false}, nil
}
if err != nil {
return nil, mapObjectStorageError(err, "object_storage_usage_failed", "读取 OSS 存储用量失败")
}
if info == nil || !info.Available {
return &ObjectStorageUsageResult{Available: false}, nil
}
return &ObjectStorageUsageResult{
Available: true,
Provider: info.Provider,
StorageSize: info.StorageSize,
ObjectCount: info.ObjectCount,
}, nil
}
func (s *ObjectStorageService) Get(ctx context.Context, key string) (*ObjectStorageDetailResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
objectKey, err := normalizeOSSObjectKey(key)
if err != nil {
return nil, err
}
info, err := s.storage.Stat(ctx, objectKey)
if err != nil {
return nil, mapObjectStorageError(err, "object_storage_stat_failed", "读取 OSS 对象详情失败")
}
publicURL := s.publicReadableURL(objectKey)
previewURL, err := s.storage.DownloadURL(objectKey)
if err != nil || strings.TrimSpace(previewURL) == "" {
previewURL = ""
}
return &ObjectStorageDetailResult{ObjectInfo: *info, PublicURL: publicURL, PreviewURL: optionalString(previewURL)}, nil
}
func (s *ObjectStorageService) publicReadableURL(objectKey string) *string {
publicClient, ok := s.storage.(objectstorage.PublicReadableClient)
if !ok {
return nil
}
urlValue, err := publicClient.PublicReadableURL(objectKey)
if err != nil || strings.TrimSpace(urlValue) == "" {
return nil
}
return &urlValue
}
func (s *ObjectStorageService) Download(ctx context.Context, key string) ([]byte, *objectstorage.ObjectInfo, error) {
if err := s.ensureReady(); err != nil {
return nil, nil, err
}
objectKey, err := normalizeOSSObjectKey(key)
if err != nil {
return nil, nil, err
}
info, statErr := s.storage.Stat(ctx, objectKey)
if statErr != nil && !errors.Is(statErr, objectstorage.ErrObjectNotFound) {
return nil, nil, mapObjectStorageError(statErr, "object_storage_stat_failed", "读取 OSS 对象详情失败")
}
content, err := s.storage.GetBytes(ctx, objectKey)
if err != nil {
return nil, nil, mapObjectStorageError(err, "object_storage_download_failed", "下载 OSS 对象失败")
}
return content, info, nil
}
func (s *ObjectStorageService) ResolvePublicURL(ctx context.Context, key string) (string, error) {
if err := s.ensureReady(); err != nil {
return "", err
}
objectKey, err := normalizeOSSObjectKey(key)
if err != nil {
return "", err
}
if exists, err := s.storage.Exists(ctx, objectKey); err != nil {
return "", mapObjectStorageError(err, "object_storage_stat_failed", "读取 OSS 对象详情失败")
} else if !exists {
return "", response.ErrNotFound(40441, "object_storage_object_not_found", "OSS 对象不存在")
}
urlValue, err := s.storage.PublicURL(objectKey)
if errors.Is(err, objectstorage.ErrObjectNotPublic) {
return s.ResolveDownloadURL(ctx, objectKey)
}
if err != nil {
return "", mapObjectStorageError(err, "object_storage_url_failed", "生成 OSS 访问地址失败")
}
return urlValue, nil
}
func (s *ObjectStorageService) ResolveDownloadURL(ctx context.Context, key string) (string, error) {
if err := s.ensureReady(); err != nil {
return "", err
}
objectKey, err := normalizeOSSObjectKey(key)
if err != nil {
return "", err
}
if exists, err := s.storage.Exists(ctx, objectKey); err != nil {
return "", mapObjectStorageError(err, "object_storage_stat_failed", "读取 OSS 对象详情失败")
} else if !exists {
return "", response.ErrNotFound(40441, "object_storage_object_not_found", "OSS 对象不存在")
}
urlValue, err := s.storage.DownloadURL(objectKey)
if err != nil {
return "", mapObjectStorageError(err, "object_storage_url_failed", "生成 OSS 下载地址失败")
}
return urlValue, nil
}
func (s *ObjectStorageService) Upload(ctx context.Context, actor *Actor, in ObjectStorageUploadInput) (*ObjectStorageUploadResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
if len(in.Content) == 0 {
return nil, response.ErrBadRequest(40094, "empty_object_storage_file", "上传文件不能为空")
}
if len(in.Content) > 500*1024*1024 {
return nil, response.ErrBadRequest(40095, "object_storage_file_too_large", "单个对象不能超过 500MB")
}
objectKey, err := normalizeUploadObjectKey(in.ObjectKey, in.FileName)
if err != nil {
return nil, err
}
if !in.Overwrite {
if exists, err := s.storage.Exists(ctx, objectKey); err != nil {
return nil, mapObjectStorageError(err, "object_storage_stat_failed", "检查 OSS 对象失败")
} else if exists {
return nil, response.ErrConflict(40941, "object_storage_object_exists", "OSS 对象已存在")
}
}
contentType := strings.TrimSpace(in.ContentType)
if contentType == "" {
contentType = http.DetectContentType(in.Content)
}
if err := s.storage.PutBytes(ctx, objectKey, in.Content, contentType); err != nil {
return nil, mapObjectStorageError(err, "object_storage_upload_failed", "上传 OSS 对象失败")
}
info, err := s.storage.Stat(ctx, objectKey)
if err != nil {
if s.logger != nil {
s.logger.Warn("object storage stat failed after upload; returning synthesized metadata",
zap.String("object_key", objectKey),
zap.Error(err),
)
}
info = &objectstorage.ObjectInfo{
Key: objectKey,
Size: int64(len(in.Content)),
ContentType: contentType,
LastModified: time.Now(),
}
}
s.audit(ctx, actor, ActionObjectStorageUpload, objectKey, map[string]any{
"object_key": objectKey,
"size": len(in.Content),
"overwrite": in.Overwrite,
})
return &ObjectStorageUploadResult{Object: *info}, nil
}
func (s *ObjectStorageService) Move(ctx context.Context, actor *Actor, in ObjectStorageMoveInput) (*ObjectStorageDetailResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
sourceKey, err := normalizeOSSObjectKey(in.SourceKey)
if err != nil {
return nil, err
}
destinationKey, err := normalizeOSSObjectKey(in.DestinationKey)
if err != nil {
return nil, err
}
if sourceKey == destinationKey {
return s.Get(ctx, destinationKey)
}
if exists, err := s.storage.Exists(ctx, sourceKey); err != nil {
return nil, mapObjectStorageError(err, "object_storage_stat_failed", "检查源对象失败")
} else if !exists {
return nil, response.ErrNotFound(40441, "object_storage_object_not_found", "源 OSS 对象不存在")
}
if !in.Overwrite {
if exists, err := s.storage.Exists(ctx, destinationKey); err != nil {
return nil, mapObjectStorageError(err, "object_storage_stat_failed", "检查目标对象失败")
} else if exists {
return nil, response.ErrConflict(40941, "object_storage_object_exists", "目标 OSS 对象已存在")
}
}
if err := s.storage.Copy(ctx, sourceKey, destinationKey); err != nil {
return nil, mapObjectStorageError(err, "object_storage_move_failed", "移动 OSS 对象失败")
}
if in.DeleteSource {
if err := s.storage.Delete(ctx, sourceKey); err != nil {
return nil, mapObjectStorageError(err, "object_storage_delete_failed", "删除源 OSS 对象失败")
}
}
s.audit(ctx, actor, ActionObjectStorageMove, destinationKey, map[string]any{
"source_key": sourceKey,
"destination_key": destinationKey,
"overwrite": in.Overwrite,
"delete_source": in.DeleteSource,
})
return s.Get(ctx, destinationKey)
}
func (s *ObjectStorageService) Delete(ctx context.Context, actor *Actor, key string) error {
if err := s.ensureReady(); err != nil {
return err
}
objectKey, err := normalizeOSSObjectKey(key)
if err != nil {
return err
}
if exists, err := s.storage.Exists(ctx, objectKey); err != nil {
return mapObjectStorageError(err, "object_storage_stat_failed", "检查 OSS 对象失败")
} else if !exists {
return response.ErrNotFound(40441, "object_storage_object_not_found", "OSS 对象不存在")
}
if err := s.storage.Delete(ctx, objectKey); err != nil {
return mapObjectStorageError(err, "object_storage_delete_failed", "删除 OSS 对象失败")
}
s.audit(ctx, actor, ActionObjectStorageDelete, objectKey, map[string]any{"object_key": objectKey})
return nil
}
func (s *ObjectStorageService) CreateFolder(
ctx context.Context,
actor *Actor,
in ObjectStorageFolderCreateInput,
) (*ObjectStorageFolderCreateResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
prefix, err := normalizeOSSFolderPrefix(in.Prefix)
if err != nil {
return nil, err
}
if exists, err := s.folderExists(ctx, prefix); err != nil {
return nil, mapObjectStorageError(err, "object_storage_folder_check_failed", "检查 OSS 目录失败")
} else if exists {
return nil, response.ErrConflict(40942, "object_storage_folder_exists", "OSS 目录已存在")
}
if err := s.storage.PutBytes(ctx, prefix, nil, "application/octet-stream"); err != nil {
return nil, mapObjectStorageError(err, "object_storage_folder_create_failed", "创建 OSS 目录失败")
}
s.audit(ctx, actor, ActionObjectStorageFolderCreate, prefix, map[string]any{"prefix": prefix})
return &ObjectStorageFolderCreateResult{Prefix: prefix}, nil
}
func (s *ObjectStorageService) DeleteFolder(
ctx context.Context,
actor *Actor,
prefixValue string,
) (*ObjectStorageFolderDeleteResult, error) {
if err := s.ensureReady(); err != nil {
return nil, err
}
prefix, err := normalizeOSSFolderPrefix(prefixValue)
if err != nil {
return nil, err
}
keys := make([]string, 0)
continuationToken := ""
for {
result, err := s.storage.List(ctx, objectstorage.ListInput{
Prefix: prefix,
ContinuationToken: continuationToken,
Recursive: true,
Limit: 1000,
})
if err != nil {
return nil, mapObjectStorageError(err, "object_storage_folder_list_failed", "读取 OSS 目录失败")
}
for _, item := range result.Objects {
if strings.TrimSpace(item.Key) == "" || !strings.HasPrefix(item.Key, prefix) {
continue
}
keys = append(keys, item.Key)
}
if !result.IsTruncated || strings.TrimSpace(result.NextContinuationToken) == "" {
break
}
continuationToken = strings.TrimSpace(result.NextContinuationToken)
}
if len(keys) == 0 {
return nil, response.ErrNotFound(40442, "object_storage_folder_not_found", "OSS 目录不存在或目录为空")
}
deleted := 0
for _, key := range keys {
if strings.TrimSpace(key) == "" || !strings.HasPrefix(key, prefix) {
continue
}
if err := s.storage.Delete(ctx, key); err != nil {
return nil, mapObjectStorageError(err, "object_storage_folder_delete_failed", "删除 OSS 目录失败")
}
deleted++
}
s.audit(ctx, actor, ActionObjectStorageFolderDelete, prefix, map[string]any{
"prefix": prefix,
"deleted_count": deleted,
})
return &ObjectStorageFolderDeleteResult{Prefix: prefix, DeletedCount: deleted}, nil
}
func (s *ObjectStorageService) ensureReady() error {
if s == nil || s.storage == nil {
return response.ErrServiceUnavailable(50303, "object_storage_unavailable", "object storage is not configured")
}
if err := s.storage.Validate(); err != nil {
return response.ErrServiceUnavailable(50303, "object_storage_unavailable", "object storage is not configured")
}
return nil
}
func normalizeUploadObjectKey(objectKey string, fileName string) (string, error) {
objectKey = strings.TrimSpace(objectKey)
if objectKey == "" || strings.HasSuffix(objectKey, "/") {
name := sanitizeOSSFileName(fileName)
if name == "" {
return "", response.ErrBadRequest(40096, "invalid_object_storage_key", "Object Key 不能为空")
}
objectKey = strings.TrimRight(objectKey, "/")
if objectKey != "" {
objectKey += "/"
}
objectKey += name
}
return normalizeOSSObjectKey(objectKey)
}
func normalizeOSSObjectKey(value string) (string, error) {
key, err := normalizeOSSKey(value, false)
if err != nil {
return "", err
}
if key == "" || strings.HasSuffix(key, "/") {
return "", response.ErrBadRequest(40096, "invalid_object_storage_key", "Object Key 不能为空且不能是目录")
}
return key, nil
}
func normalizeOSSFolderPrefix(value string) (string, error) {
prefix, err := normalizeOSSKey(value, false)
if err != nil {
return "", err
}
prefix = strings.TrimRight(prefix, "/")
if prefix == "" {
return "", response.ErrBadRequest(40097, "invalid_object_storage_folder", "目录路径不能为空")
}
return prefix + "/", nil
}
func normalizeOSSOptionalPrefix(value string) (string, error) {
return normalizeOSSKey(value, true)
}
func normalizeOSSOptionalToken(value string) string {
key, err := normalizeOSSKey(value, true)
if err != nil {
return ""
}
return key
}
func normalizeOSSKey(value string, allowEmpty bool) (string, error) {
value = strings.TrimSpace(value)
value = strings.TrimLeft(value, "/")
if value == "" {
if allowEmpty {
return "", nil
}
return "", response.ErrBadRequest(40096, "invalid_object_storage_key", "Object Key 不能为空")
}
if len(value) > 1024 || strings.Contains(value, "\\") {
return "", response.ErrBadRequest(40096, "invalid_object_storage_key", "Object Key 长度或格式无效")
}
for _, part := range strings.Split(value, "/") {
if part == "." || part == ".." {
return "", response.ErrBadRequest(40096, "invalid_object_storage_key", "Object Key 不能包含 . 或 .. 路径段")
}
}
return value, nil
}
func sanitizeOSSFileName(value string) string {
name := strings.TrimSpace(filepath.Base(value))
if name == "." || name == "/" || name == "\\" {
return ""
}
return strings.ReplaceAll(name, "\\", "")
}
func filterOSSObjectsByKeyword(items []objectstorage.ObjectInfo, keyword string) []objectstorage.ObjectInfo {
if keyword == "" {
return items
}
normalized := strings.ToLower(keyword)
filtered := make([]objectstorage.ObjectInfo, 0, len(items))
for _, item := range items {
if strings.Contains(strings.ToLower(item.Key), normalized) {
filtered = append(filtered, item)
}
}
return filtered
}
func optionalString(value string) *string {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
return &value
}
func shouldFallbackOSSDirectoryList(
in ObjectStorageListInput,
keyword string,
objects []objectstorage.ObjectInfo,
commonPrefixes []string,
) bool {
if in.Recursive || keyword != "" {
return false
}
return len(objects) == 0 && len(commonPrefixes) == 0
}
func normalizeOSSListedObjects(prefix string, items []objectstorage.ObjectInfo, recursive bool) []objectstorage.ObjectInfo {
if recursive {
filtered := make([]objectstorage.ObjectInfo, 0, len(items))
for _, item := range items {
if strings.HasSuffix(item.Key, "/") {
continue
}
filtered = append(filtered, item)
}
return filtered
}
filtered := make([]objectstorage.ObjectInfo, 0, len(items))
for _, item := range items {
if strings.HasSuffix(item.Key, "/") {
continue
}
if !strings.HasPrefix(item.Key, prefix) {
continue
}
remainder := strings.TrimPrefix(item.Key, prefix)
if remainder == "" || strings.Contains(remainder, "/") {
continue
}
filtered = append(filtered, item)
}
return filtered
}
func normalizeOSSCommonPrefixes(
prefix string,
listedPrefixes []string,
objects []objectstorage.ObjectInfo,
recursive bool,
) []string {
if recursive {
return nil
}
seen := make(map[string]struct{}, len(listedPrefixes)+len(objects))
out := make([]string, 0, len(listedPrefixes))
add := func(value string, requireNested bool) {
value = strings.TrimLeft(strings.TrimSpace(value), "/")
if value == "" || value == prefix || !strings.HasPrefix(value, prefix) {
return
}
remainder := strings.TrimPrefix(value, prefix)
first, _, hasSlash := strings.Cut(remainder, "/")
if first == "" || (requireNested && !hasSlash) {
return
}
next := prefix + first + "/"
if _, ok := seen[next]; ok {
return
}
seen[next] = struct{}{}
out = append(out, next)
}
for _, value := range listedPrefixes {
add(value, false)
}
for _, item := range objects {
add(item.Key, true)
}
return out
}
func (s *ObjectStorageService) folderExists(ctx context.Context, prefix string) (bool, error) {
if exists, err := s.storage.Exists(ctx, prefix); err != nil {
return false, err
} else if exists {
return true, nil
}
result, err := s.storage.List(ctx, objectstorage.ListInput{
Prefix: prefix,
Recursive: true,
Limit: 1,
})
if err != nil {
return false, err
}
return len(result.Objects) > 0, nil
}
func mapObjectStorageError(err error, code string, detail string) error {
if errors.Is(err, objectstorage.ErrNotConfigured) {
return response.ErrServiceUnavailable(50303, "object_storage_unavailable", "object storage is not configured")
}
if errors.Is(err, objectstorage.ErrObjectNotFound) {
return response.ErrNotFound(40441, "object_storage_object_not_found", "OSS 对象不存在")
}
appErr := response.ErrInternal(50097, code, detail)
appErr.Cause = err
return appErr
}
func (s *ObjectStorageService) audit(ctx context.Context, actor *Actor, action string, targetKey string, metadata map[string]any) {
if actor == nil || s.audits == nil {
return
}
event := actor.audit(action, "object_storage_object", 0, metadata)
event.TargetID = targetKey
_ = s.audits.Append(ctx, event)
}