feat: Implement direct upload functionality for desktop client releases
Desktop Client Build / Resolve Build Metadata (push) Successful in 52s
Frontend CI / Frontend (push) Successful in 3m49s
Backend CI / Backend (push) Successful in 16m10s
Desktop Client Build / Build Desktop Client (push) Successful in 23m22s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 31s

- Added new endpoints for initiating and completing direct uploads of desktop client packages.
- Introduced request and response structures for direct upload operations.
- Enhanced the upload process to support SHA256 and Content-MD5 validation.
- Updated the frontend to reflect changes in upload button states and progress indicators.
- Modified object storage clients to support presigned PUT URLs with content type and MD5 headers.
- Added tests for direct upload functionality and ensured existing upload processes remain intact.
This commit is contained in:
2026-06-20 12:44:28 +08:00
parent f26802c76e
commit 62d0b22988
15 changed files with 884 additions and 58 deletions
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
@@ -54,7 +55,9 @@ const (
DefaultDesktopClientReleaseChunkSizeBytes = 16 * 1024 * 1024
MinDesktopClientReleaseChunkSizeBytes = 1 * 1024 * 1024
MaxDesktopClientReleaseChunkSizeBytes = 64 * 1024 * 1024
desktopClientReleaseContentMD5Bytes = 16
desktopClientReleaseUploadSessionTTL = 24 * time.Hour
desktopClientReleaseDirectUploadTTL = 2 * time.Hour
)
type DesktopClientReleaseService struct {
@@ -231,6 +234,42 @@ type DesktopClientReleaseChunkUploadResult struct {
UploadedChunks []int `json:"uploaded_chunks"`
}
type DesktopClientReleaseDirectUploadInitInput struct {
Platform string
Arch string
Channel string
Version string
Kind string
FileName string
FileSizeBytes int64
SHA256 string
ContentMD5 string
}
type DesktopClientReleaseDirectUploadInitResult struct {
UploadURL string `json:"upload_url,omitempty"`
Method string `json:"method"`
Headers map[string]string `json:"headers"`
OSSObjectKey string `json:"oss_object_key"`
FileName string `json:"file_name"`
FileSizeBytes int64 `json:"file_size_bytes"`
SHA256 string `json:"sha256"`
ObjectExists bool `json:"object_exists"`
ExpiresAt time.Time `json:"expires_at"`
}
type DesktopClientReleaseDirectUploadCompleteInput struct {
Platform string
Arch string
Channel string
Version string
Kind string
OSSObjectKey string
FileName string
FileSizeBytes int64
SHA256 string
}
type desktopClientReleaseUploadManifest struct {
UploadID string `json:"upload_id"`
Platform string `json:"platform"`
@@ -529,6 +568,171 @@ func (s *DesktopClientReleaseService) UploadOSS(ctx context.Context, in DesktopC
}, nil
}
func (s *DesktopClientReleaseService) InitiateDirectUpload(
ctx context.Context,
in DesktopClientReleaseDirectUploadInitInput,
) (*DesktopClientReleaseDirectUploadInitResult, error) {
if s.storage == nil {
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
}
if err := s.storage.Validate(); err != nil {
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
}
uploadInput := DesktopClientReleaseUploadInput{
Platform: in.Platform,
Arch: in.Arch,
Channel: in.Channel,
Version: in.Version,
Kind: in.Kind,
FileName: in.FileName,
ContentSizeBytes: in.FileSizeBytes,
}
target, fileName, err := normalizeDesktopClientReleaseUploadInput(uploadInput)
if err != nil {
return nil, err
}
sha256Hex, err := normalizeRequiredDesktopReleaseSHA256(in.SHA256)
if err != nil {
return nil, err
}
kind := normalizeDesktopClientAssetKind(in.Kind)
if kind == DesktopClientAssetKindUpdater {
if err := validateDesktopClientUpdaterPackage(target.Platform, &fileName, ""); err != nil {
return nil, err
}
}
contentMD5, err := normalizeDesktopReleaseContentMD5(in.ContentMD5)
if err != nil {
return nil, err
}
objectKey := buildDesktopClientReleaseObjectKey(sha256Hex, fileName)
exists, err := s.storage.Exists(ctx, objectKey)
if err != nil {
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
appErr.Cause = err
return nil, appErr
}
if exists {
return &DesktopClientReleaseDirectUploadInitResult{
Method: http.MethodPut,
Headers: map[string]string{},
OSSObjectKey: objectKey,
FileName: fileName,
FileSizeBytes: in.FileSizeBytes,
SHA256: sha256Hex,
ObjectExists: true,
ExpiresAt: time.Now().UTC(),
}, nil
}
presigner, ok := s.storage.(objectstorage.PresignedPutClient)
if !ok {
return nil, response.ErrInternal(50091, "object_storage_direct_upload_unsupported", "object storage direct upload is unsupported")
}
presigned, err := presigner.PresignedPutURL(
ctx,
objectKey,
detectDesktopReleaseContentType(fileName, nil),
contentMD5,
desktopClientReleaseDirectUploadTTL,
)
if err != nil {
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
appErr.Cause = err
return nil, appErr
}
return &DesktopClientReleaseDirectUploadInitResult{
UploadURL: presigned.URL,
Method: http.MethodPut,
Headers: flattenHTTPHeaders(presigned.Headers),
OSSObjectKey: objectKey,
FileName: fileName,
FileSizeBytes: in.FileSizeBytes,
SHA256: sha256Hex,
ObjectExists: false,
ExpiresAt: presigned.ExpiresAt,
}, nil
}
func (s *DesktopClientReleaseService) CompleteDirectUpload(
ctx context.Context,
in DesktopClientReleaseDirectUploadCompleteInput,
) (*DesktopClientReleaseUploadResult, error) {
if s.storage == nil {
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
}
if err := s.storage.Validate(); err != nil {
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
}
uploadInput := DesktopClientReleaseUploadInput{
Platform: in.Platform,
Arch: in.Arch,
Channel: in.Channel,
Version: in.Version,
Kind: in.Kind,
FileName: in.FileName,
ContentSizeBytes: in.FileSizeBytes,
}
target, fileName, err := normalizeDesktopClientReleaseUploadInput(uploadInput)
if err != nil {
return nil, err
}
sha256Hex, err := normalizeRequiredDesktopReleaseSHA256(in.SHA256)
if err != nil {
return nil, err
}
kind := normalizeDesktopClientAssetKind(in.Kind)
if kind == DesktopClientAssetKindUpdater {
if err := validateDesktopClientUpdaterPackage(target.Platform, &fileName, ""); err != nil {
return nil, err
}
}
objectKey := strings.TrimSpace(in.OSSObjectKey)
expectedObjectKey := buildDesktopClientReleaseObjectKey(sha256Hex, fileName)
if objectKey == "" {
objectKey = expectedObjectKey
}
if objectKey != expectedObjectKey || !isValidDesktopReleaseObjectKey(objectKey) {
return nil, response.ErrBadRequest(40066, "invalid_oss_object_key", "OSS Object Key 与上传文件校验信息不匹配")
}
if management, ok := s.storage.(objectstorage.ManagementClient); ok {
info, err := management.Stat(ctx, objectKey)
if err != nil {
if errors.Is(err, objectstorage.ErrObjectNotFound) {
return nil, response.ErrBadRequest(40078, "direct_upload_not_found", "OSS 直传文件尚未完成")
}
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
appErr.Cause = err
return nil, appErr
}
if info.Size != in.FileSizeBytes {
return nil, response.ErrBadRequest(40079, "direct_upload_size_mismatch", "OSS 直传文件大小不匹配,请重新上传")
}
} else {
exists, err := s.storage.Exists(ctx, objectKey)
if err != nil {
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
appErr.Cause = err
return nil, appErr
}
if !exists {
return nil, response.ErrBadRequest(40078, "direct_upload_not_found", "OSS 直传文件尚未完成")
}
}
return &DesktopClientReleaseUploadResult{
OSSObjectKey: objectKey,
FileName: fileName,
FileSizeBytes: in.FileSizeBytes,
SHA256: sha256Hex,
}, nil
}
func MaxDesktopClientReleaseMultipartBytes() int64 {
return MaxDesktopClientReleaseUploadBytes + desktopClientReleaseMultipartOverheadBytes
}
@@ -1399,6 +1603,29 @@ func normalizeDesktopReleaseSHA256(value *string) (*string, error) {
return &normalized, nil
}
func normalizeRequiredDesktopReleaseSHA256(value string) (string, error) {
sha256, err := normalizeDesktopReleaseSHA256(&value)
if err != nil {
return "", err
}
if sha256 == nil {
return "", response.ErrBadRequest(40070, "invalid_sha256", "SHA256 必须是 64 位十六进制")
}
return *sha256, nil
}
func normalizeDesktopReleaseContentMD5(value string) (string, error) {
normalized := strings.TrimSpace(value)
if normalized == "" {
return "", nil
}
content, err := base64.StdEncoding.DecodeString(normalized)
if err != nil || len(content) != desktopClientReleaseContentMD5Bytes {
return "", response.ErrBadRequest(40080, "invalid_content_md5", "Content-MD5 必须是标准 base64 编码的 16 字节 MD5")
}
return normalized, nil
}
func normalizeDesktopClientReleaseUploadInput(in DesktopClientReleaseUploadInput) (repository.DesktopClientReleaseTarget, string, error) {
size, err := desktopClientReleaseUploadSize(in)
if err != nil {
@@ -1520,6 +1747,25 @@ func putDesktopClientReleaseReader(
return storage.PutBytes(ctx, objectKey, content, contentType)
}
func flattenHTTPHeaders(headers http.Header) map[string]string {
if len(headers) == 0 {
return map[string]string{}
}
out := make(map[string]string, len(headers))
for key, values := range headers {
key = strings.TrimSpace(key)
if key == "" || len(values) == 0 {
continue
}
value := strings.TrimSpace(values[0])
if value == "" {
continue
}
out[key] = value
}
return out
}
func defaultDesktopClientReleaseUploadDir() string {
return filepath.Join(os.TempDir(), "geo-rankly-desktop-release-uploads")
}
@@ -7,11 +7,13 @@ import (
"encoding/hex"
"errors"
"io"
"net/http"
"os"
"testing"
"time"
"github.com/geo-platform/tenant-api/internal/ops/domain"
"github.com/geo-platform/tenant-api/internal/shared/objectstorage"
"github.com/stretchr/testify/require"
)
@@ -311,11 +313,92 @@ func TestDesktopClientReleaseChunkedUploadCompletesOutOfOrder(t *testing.T) {
require.True(t, errors.Is(err, os.ErrNotExist))
}
func TestDesktopClientReleaseDirectUploadUsesPresignedPutURL(t *testing.T) {
t.Parallel()
content := []byte("release-package")
sum := sha256.Sum256(content)
wantSHA := hex.EncodeToString(sum[:])
storage := &desktopReleaseDownloadURLStorage{
existsByKey: map[string]bool{},
objects: map[string][]byte{},
objectSizes: map[string]int64{},
}
svc := NewDesktopClientReleaseService(nil, storage, nil)
initResult, err := svc.InitiateDirectUpload(context.Background(), DesktopClientReleaseDirectUploadInitInput{
Platform: "darwin",
Arch: "arm64",
Channel: "stable",
Version: "1.2.3",
Kind: DesktopClientAssetKindInstaller,
FileName: "client.dmg",
FileSizeBytes: int64(len(content)),
SHA256: wantSHA,
ContentMD5: "pt74NOtqjIwBJtPUYvJ8AA==",
})
require.NoError(t, err)
require.False(t, initResult.ObjectExists)
require.Equal(t, http.MethodPut, initResult.Method)
require.Equal(t, "desktop-client/releases/blobs/"+wantSHA+".dmg", initResult.OSSObjectKey)
require.Equal(t, "https://oss.example.test/upload/"+initResult.OSSObjectKey, initResult.UploadURL)
require.Equal(t, "application/x-apple-diskimage", initResult.Headers["Content-Type"])
require.Equal(t, "pt74NOtqjIwBJtPUYvJ8AA==", initResult.Headers["Content-Md5"])
require.Len(t, storage.presignedPutCalls, 1)
require.Equal(t, initResult.OSSObjectKey, storage.presignedPutCalls[0])
storage.existsByKey[initResult.OSSObjectKey] = true
storage.objectSizes[initResult.OSSObjectKey] = int64(len(content))
completeResult, err := svc.CompleteDirectUpload(context.Background(), DesktopClientReleaseDirectUploadCompleteInput{
Platform: "darwin",
Arch: "arm64",
Channel: "stable",
Version: "1.2.3",
Kind: DesktopClientAssetKindInstaller,
OSSObjectKey: initResult.OSSObjectKey,
FileName: initResult.FileName,
FileSizeBytes: initResult.FileSizeBytes,
SHA256: initResult.SHA256,
})
require.NoError(t, err)
require.Equal(t, initResult.OSSObjectKey, completeResult.OSSObjectKey)
require.Equal(t, wantSHA, completeResult.SHA256)
}
func TestDesktopClientReleaseDirectUploadRejectsSizeMismatch(t *testing.T) {
t.Parallel()
sum := sha256.Sum256([]byte("release-package"))
sha256Hex := hex.EncodeToString(sum[:])
objectKey := buildDesktopClientReleaseObjectKey(sha256Hex, "client.zip")
storage := &desktopReleaseDownloadURLStorage{
existsByKey: map[string]bool{objectKey: true},
objectSizes: map[string]int64{objectKey: 10},
}
svc := NewDesktopClientReleaseService(nil, storage, nil)
_, err := svc.CompleteDirectUpload(context.Background(), DesktopClientReleaseDirectUploadCompleteInput{
Platform: "darwin",
Arch: "arm64",
Channel: "stable",
Version: "1.2.3",
Kind: DesktopClientAssetKindUpdater,
OSSObjectKey: objectKey,
FileName: "client.zip",
FileSizeBytes: 20,
SHA256: sha256Hex,
})
require.Error(t, err)
require.Contains(t, err.Error(), "direct_upload_size_mismatch")
}
type desktopReleaseDownloadURLStorage struct {
publicURLCalls []string
downloadURLCalls []string
existsByKey map[string]bool
objects map[string][]byte
publicURLCalls []string
downloadURLCalls []string
existsByKey map[string]bool
objects map[string][]byte
objectSizes map[string]int64
presignedPutCalls []string
}
func (s *desktopReleaseDownloadURLStorage) Validate() error {
@@ -335,6 +418,10 @@ func (s *desktopReleaseDownloadURLStorage) PutReader(_ context.Context, objectKe
return err
}
s.objects[objectKey] = content
if s.objectSizes == nil {
s.objectSizes = map[string]int64{}
}
s.objectSizes[objectKey] = int64(len(content))
if s.existsByKey == nil {
s.existsByKey = map[string]bool{}
}
@@ -353,6 +440,26 @@ func (s *desktopReleaseDownloadURLStorage) Exists(_ context.Context, objectKey s
return true, nil
}
func (s *desktopReleaseDownloadURLStorage) PresignedPutURL(
_ context.Context,
objectKey string,
contentType string,
contentMD5Base64 string,
ttl time.Duration,
) (*objectstorage.PresignedPutResult, error) {
s.presignedPutCalls = append(s.presignedPutCalls, objectKey)
headers := http.Header{}
headers.Set("Content-Type", contentType)
if contentMD5Base64 != "" {
headers.Set("Content-MD5", contentMD5Base64)
}
return &objectstorage.PresignedPutResult{
URL: "https://oss.example.test/upload/" + objectKey,
Headers: headers,
ExpiresAt: time.Now().UTC().Add(ttl),
}, nil
}
func (s *desktopReleaseDownloadURLStorage) Delete(context.Context, string) error {
return nil
}
@@ -366,3 +473,25 @@ func (s *desktopReleaseDownloadURLStorage) DownloadURL(objectKey string) (string
s.downloadURLCalls = append(s.downloadURLCalls, objectKey)
return "https://oss.example.test/signed/client.zip?Expires=123", nil
}
func (s *desktopReleaseDownloadURLStorage) List(context.Context, objectstorage.ListInput) (*objectstorage.ListResult, error) {
return &objectstorage.ListResult{}, nil
}
func (s *desktopReleaseDownloadURLStorage) Stat(_ context.Context, objectKey string) (*objectstorage.ObjectInfo, error) {
exists, _ := s.Exists(context.Background(), objectKey)
if !exists {
return nil, objectstorage.ErrObjectNotFound
}
if s.objectSizes != nil {
return &objectstorage.ObjectInfo{Key: objectKey, Size: s.objectSizes[objectKey], LastModified: time.Now()}, nil
}
if s.objects != nil {
return &objectstorage.ObjectInfo{Key: objectKey, Size: int64(len(s.objects[objectKey])), LastModified: time.Now()}, nil
}
return &objectstorage.ObjectInfo{Key: objectKey, LastModified: time.Now()}, nil
}
func (s *desktopReleaseDownloadURLStorage) Copy(context.Context, string, string) error {
return nil
}
@@ -50,6 +50,30 @@ type desktopClientReleaseUploadInitRequest struct {
ChunkSizeBytes int64 `json:"chunk_size_bytes"`
}
type desktopClientReleaseDirectUploadInitRequest struct {
Platform string `json:"platform" binding:"required"`
Arch string `json:"arch"`
Channel string `json:"channel"`
Version string `json:"version" binding:"required"`
Kind string `json:"kind"`
FileName string `json:"file_name" binding:"required"`
FileSizeBytes int64 `json:"file_size_bytes" binding:"required"`
SHA256 string `json:"sha256" binding:"required"`
ContentMD5 string `json:"content_md5"`
}
type desktopClientReleaseDirectUploadCompleteRequest struct {
Platform string `json:"platform" binding:"required"`
Arch string `json:"arch"`
Channel string `json:"channel"`
Version string `json:"version" binding:"required"`
Kind string `json:"kind"`
OSSObjectKey string `json:"oss_object_key" binding:"required"`
FileName string `json:"file_name" binding:"required"`
FileSizeBytes int64 `json:"file_size_bytes" binding:"required"`
SHA256 string `json:"sha256" binding:"required"`
}
func listDesktopClientReleasesHandler(svc *app.DesktopClientReleaseService) gin.HandlerFunc {
return func(c *gin.Context) {
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
@@ -208,6 +232,58 @@ func uploadDesktopClientReleaseHandler(svc *app.DesktopClientReleaseService) gin
}
}
func initiateDesktopClientReleaseDirectUploadHandler(svc *app.DesktopClientReleaseService) gin.HandlerFunc {
return func(c *gin.Context) {
var body desktopClientReleaseDirectUploadInitRequest
if err := c.ShouldBindJSON(&body); err != nil {
response.Error(c, response.ErrBadRequest(40000, "invalid_payload", err.Error()))
return
}
result, err := svc.InitiateDirectUpload(c.Request.Context(), app.DesktopClientReleaseDirectUploadInitInput{
Platform: body.Platform,
Arch: body.Arch,
Channel: body.Channel,
Version: body.Version,
Kind: body.Kind,
FileName: body.FileName,
FileSizeBytes: body.FileSizeBytes,
SHA256: body.SHA256,
ContentMD5: body.ContentMD5,
})
if err != nil {
response.Error(c, err)
return
}
response.Success(c, result)
}
}
func completeDesktopClientReleaseDirectUploadHandler(svc *app.DesktopClientReleaseService) gin.HandlerFunc {
return func(c *gin.Context) {
var body desktopClientReleaseDirectUploadCompleteRequest
if err := c.ShouldBindJSON(&body); err != nil {
response.Error(c, response.ErrBadRequest(40000, "invalid_payload", err.Error()))
return
}
result, err := svc.CompleteDirectUpload(c.Request.Context(), app.DesktopClientReleaseDirectUploadCompleteInput{
Platform: body.Platform,
Arch: body.Arch,
Channel: body.Channel,
Version: body.Version,
Kind: body.Kind,
OSSObjectKey: body.OSSObjectKey,
FileName: body.FileName,
FileSizeBytes: body.FileSizeBytes,
SHA256: body.SHA256,
})
if err != nil {
response.Error(c, err)
return
}
response.Success(c, result)
}
}
func initiateDesktopClientReleaseUploadHandler(svc *app.DesktopClientReleaseService) gin.HandlerFunc {
return func(c *gin.Context) {
var body desktopClientReleaseUploadInitRequest
+2
View File
@@ -173,6 +173,8 @@ func RegisterRoutes(d Deps) {
authed.GET("/desktop-client/releases", listDesktopClientReleasesHandler(d.Releases))
authed.POST("/desktop-client/releases/upload", uploadDesktopClientReleaseHandler(d.Releases))
authed.POST("/desktop-client/releases/direct-upload", initiateDesktopClientReleaseDirectUploadHandler(d.Releases))
authed.POST("/desktop-client/releases/direct-upload/complete", completeDesktopClientReleaseDirectUploadHandler(d.Releases))
authed.POST("/desktop-client/release-uploads", initiateDesktopClientReleaseUploadHandler(d.Releases))
authed.POST("/desktop-client/release-uploads/:upload_id/chunks/:chunk_index", uploadDesktopClientReleaseChunkHandler(d.Releases))
authed.POST("/desktop-client/release-uploads/:upload_id/complete", completeDesktopClientReleaseUploadHandler(d.Releases))