feat: Implement direct upload functionality for desktop client releases
Desktop Client Build / Resolve Build Metadata (push) Successful in 52s
Frontend CI / Frontend (push) Successful in 3m49s
Backend CI / Backend (push) Successful in 16m10s
Desktop Client Build / Build Desktop Client (push) Successful in 23m22s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 31s
Desktop Client Build / Resolve Build Metadata (push) Successful in 52s
Frontend CI / Frontend (push) Successful in 3m49s
Backend CI / Backend (push) Successful in 16m10s
Desktop Client Build / Build Desktop Client (push) Successful in 23m22s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 31s
- Added new endpoints for initiating and completing direct uploads of desktop client packages. - Introduced request and response structures for direct upload operations. - Enhanced the upload process to support SHA256 and Content-MD5 validation. - Updated the frontend to reflect changes in upload button states and progress indicators. - Modified object storage clients to support presigned PUT URLs with content type and MD5 headers. - Added tests for direct upload functionality and ensured existing upload processes remain intact.
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -54,7 +55,9 @@ const (
|
||||
DefaultDesktopClientReleaseChunkSizeBytes = 16 * 1024 * 1024
|
||||
MinDesktopClientReleaseChunkSizeBytes = 1 * 1024 * 1024
|
||||
MaxDesktopClientReleaseChunkSizeBytes = 64 * 1024 * 1024
|
||||
desktopClientReleaseContentMD5Bytes = 16
|
||||
desktopClientReleaseUploadSessionTTL = 24 * time.Hour
|
||||
desktopClientReleaseDirectUploadTTL = 2 * time.Hour
|
||||
)
|
||||
|
||||
type DesktopClientReleaseService struct {
|
||||
@@ -231,6 +234,42 @@ type DesktopClientReleaseChunkUploadResult struct {
|
||||
UploadedChunks []int `json:"uploaded_chunks"`
|
||||
}
|
||||
|
||||
type DesktopClientReleaseDirectUploadInitInput struct {
|
||||
Platform string
|
||||
Arch string
|
||||
Channel string
|
||||
Version string
|
||||
Kind string
|
||||
FileName string
|
||||
FileSizeBytes int64
|
||||
SHA256 string
|
||||
ContentMD5 string
|
||||
}
|
||||
|
||||
type DesktopClientReleaseDirectUploadInitResult struct {
|
||||
UploadURL string `json:"upload_url,omitempty"`
|
||||
Method string `json:"method"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
OSSObjectKey string `json:"oss_object_key"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSizeBytes int64 `json:"file_size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ObjectExists bool `json:"object_exists"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
type DesktopClientReleaseDirectUploadCompleteInput struct {
|
||||
Platform string
|
||||
Arch string
|
||||
Channel string
|
||||
Version string
|
||||
Kind string
|
||||
OSSObjectKey string
|
||||
FileName string
|
||||
FileSizeBytes int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
type desktopClientReleaseUploadManifest struct {
|
||||
UploadID string `json:"upload_id"`
|
||||
Platform string `json:"platform"`
|
||||
@@ -529,6 +568,171 @@ func (s *DesktopClientReleaseService) UploadOSS(ctx context.Context, in DesktopC
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *DesktopClientReleaseService) InitiateDirectUpload(
|
||||
ctx context.Context,
|
||||
in DesktopClientReleaseDirectUploadInitInput,
|
||||
) (*DesktopClientReleaseDirectUploadInitResult, error) {
|
||||
if s.storage == nil {
|
||||
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
|
||||
}
|
||||
if err := s.storage.Validate(); err != nil {
|
||||
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
|
||||
}
|
||||
|
||||
uploadInput := DesktopClientReleaseUploadInput{
|
||||
Platform: in.Platform,
|
||||
Arch: in.Arch,
|
||||
Channel: in.Channel,
|
||||
Version: in.Version,
|
||||
Kind: in.Kind,
|
||||
FileName: in.FileName,
|
||||
ContentSizeBytes: in.FileSizeBytes,
|
||||
}
|
||||
target, fileName, err := normalizeDesktopClientReleaseUploadInput(uploadInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha256Hex, err := normalizeRequiredDesktopReleaseSHA256(in.SHA256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kind := normalizeDesktopClientAssetKind(in.Kind)
|
||||
if kind == DesktopClientAssetKindUpdater {
|
||||
if err := validateDesktopClientUpdaterPackage(target.Platform, &fileName, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
contentMD5, err := normalizeDesktopReleaseContentMD5(in.ContentMD5)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objectKey := buildDesktopClientReleaseObjectKey(sha256Hex, fileName)
|
||||
exists, err := s.storage.Exists(ctx, objectKey)
|
||||
if err != nil {
|
||||
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
|
||||
appErr.Cause = err
|
||||
return nil, appErr
|
||||
}
|
||||
if exists {
|
||||
return &DesktopClientReleaseDirectUploadInitResult{
|
||||
Method: http.MethodPut,
|
||||
Headers: map[string]string{},
|
||||
OSSObjectKey: objectKey,
|
||||
FileName: fileName,
|
||||
FileSizeBytes: in.FileSizeBytes,
|
||||
SHA256: sha256Hex,
|
||||
ObjectExists: true,
|
||||
ExpiresAt: time.Now().UTC(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
presigner, ok := s.storage.(objectstorage.PresignedPutClient)
|
||||
if !ok {
|
||||
return nil, response.ErrInternal(50091, "object_storage_direct_upload_unsupported", "object storage direct upload is unsupported")
|
||||
}
|
||||
presigned, err := presigner.PresignedPutURL(
|
||||
ctx,
|
||||
objectKey,
|
||||
detectDesktopReleaseContentType(fileName, nil),
|
||||
contentMD5,
|
||||
desktopClientReleaseDirectUploadTTL,
|
||||
)
|
||||
if err != nil {
|
||||
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
|
||||
appErr.Cause = err
|
||||
return nil, appErr
|
||||
}
|
||||
return &DesktopClientReleaseDirectUploadInitResult{
|
||||
UploadURL: presigned.URL,
|
||||
Method: http.MethodPut,
|
||||
Headers: flattenHTTPHeaders(presigned.Headers),
|
||||
OSSObjectKey: objectKey,
|
||||
FileName: fileName,
|
||||
FileSizeBytes: in.FileSizeBytes,
|
||||
SHA256: sha256Hex,
|
||||
ObjectExists: false,
|
||||
ExpiresAt: presigned.ExpiresAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *DesktopClientReleaseService) CompleteDirectUpload(
|
||||
ctx context.Context,
|
||||
in DesktopClientReleaseDirectUploadCompleteInput,
|
||||
) (*DesktopClientReleaseUploadResult, error) {
|
||||
if s.storage == nil {
|
||||
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
|
||||
}
|
||||
if err := s.storage.Validate(); err != nil {
|
||||
return nil, response.ErrInternal(50091, "object_storage_unavailable", "object storage is unavailable")
|
||||
}
|
||||
|
||||
uploadInput := DesktopClientReleaseUploadInput{
|
||||
Platform: in.Platform,
|
||||
Arch: in.Arch,
|
||||
Channel: in.Channel,
|
||||
Version: in.Version,
|
||||
Kind: in.Kind,
|
||||
FileName: in.FileName,
|
||||
ContentSizeBytes: in.FileSizeBytes,
|
||||
}
|
||||
target, fileName, err := normalizeDesktopClientReleaseUploadInput(uploadInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha256Hex, err := normalizeRequiredDesktopReleaseSHA256(in.SHA256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kind := normalizeDesktopClientAssetKind(in.Kind)
|
||||
if kind == DesktopClientAssetKindUpdater {
|
||||
if err := validateDesktopClientUpdaterPackage(target.Platform, &fileName, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
objectKey := strings.TrimSpace(in.OSSObjectKey)
|
||||
expectedObjectKey := buildDesktopClientReleaseObjectKey(sha256Hex, fileName)
|
||||
if objectKey == "" {
|
||||
objectKey = expectedObjectKey
|
||||
}
|
||||
if objectKey != expectedObjectKey || !isValidDesktopReleaseObjectKey(objectKey) {
|
||||
return nil, response.ErrBadRequest(40066, "invalid_oss_object_key", "OSS Object Key 与上传文件校验信息不匹配")
|
||||
}
|
||||
|
||||
if management, ok := s.storage.(objectstorage.ManagementClient); ok {
|
||||
info, err := management.Stat(ctx, objectKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, objectstorage.ErrObjectNotFound) {
|
||||
return nil, response.ErrBadRequest(40078, "direct_upload_not_found", "OSS 直传文件尚未完成")
|
||||
}
|
||||
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
|
||||
appErr.Cause = err
|
||||
return nil, appErr
|
||||
}
|
||||
if info.Size != in.FileSizeBytes {
|
||||
return nil, response.ErrBadRequest(40079, "direct_upload_size_mismatch", "OSS 直传文件大小不匹配,请重新上传")
|
||||
}
|
||||
} else {
|
||||
exists, err := s.storage.Exists(ctx, objectKey)
|
||||
if err != nil {
|
||||
appErr := response.ErrInternal(50092, "desktop_client_release_upload_failed", err.Error())
|
||||
appErr.Cause = err
|
||||
return nil, appErr
|
||||
}
|
||||
if !exists {
|
||||
return nil, response.ErrBadRequest(40078, "direct_upload_not_found", "OSS 直传文件尚未完成")
|
||||
}
|
||||
}
|
||||
|
||||
return &DesktopClientReleaseUploadResult{
|
||||
OSSObjectKey: objectKey,
|
||||
FileName: fileName,
|
||||
FileSizeBytes: in.FileSizeBytes,
|
||||
SHA256: sha256Hex,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func MaxDesktopClientReleaseMultipartBytes() int64 {
|
||||
return MaxDesktopClientReleaseUploadBytes + desktopClientReleaseMultipartOverheadBytes
|
||||
}
|
||||
@@ -1399,6 +1603,29 @@ func normalizeDesktopReleaseSHA256(value *string) (*string, error) {
|
||||
return &normalized, nil
|
||||
}
|
||||
|
||||
func normalizeRequiredDesktopReleaseSHA256(value string) (string, error) {
|
||||
sha256, err := normalizeDesktopReleaseSHA256(&value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if sha256 == nil {
|
||||
return "", response.ErrBadRequest(40070, "invalid_sha256", "SHA256 必须是 64 位十六进制")
|
||||
}
|
||||
return *sha256, nil
|
||||
}
|
||||
|
||||
func normalizeDesktopReleaseContentMD5(value string) (string, error) {
|
||||
normalized := strings.TrimSpace(value)
|
||||
if normalized == "" {
|
||||
return "", nil
|
||||
}
|
||||
content, err := base64.StdEncoding.DecodeString(normalized)
|
||||
if err != nil || len(content) != desktopClientReleaseContentMD5Bytes {
|
||||
return "", response.ErrBadRequest(40080, "invalid_content_md5", "Content-MD5 必须是标准 base64 编码的 16 字节 MD5")
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func normalizeDesktopClientReleaseUploadInput(in DesktopClientReleaseUploadInput) (repository.DesktopClientReleaseTarget, string, error) {
|
||||
size, err := desktopClientReleaseUploadSize(in)
|
||||
if err != nil {
|
||||
@@ -1520,6 +1747,25 @@ func putDesktopClientReleaseReader(
|
||||
return storage.PutBytes(ctx, objectKey, content, contentType)
|
||||
}
|
||||
|
||||
func flattenHTTPHeaders(headers http.Header) map[string]string {
|
||||
if len(headers) == 0 {
|
||||
return map[string]string{}
|
||||
}
|
||||
out := make(map[string]string, len(headers))
|
||||
for key, values := range headers {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" || len(values) == 0 {
|
||||
continue
|
||||
}
|
||||
value := strings.TrimSpace(values[0])
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func defaultDesktopClientReleaseUploadDir() string {
|
||||
return filepath.Join(os.TempDir(), "geo-rankly-desktop-release-uploads")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user