Initial commit: img-infinite-canvas AI design workbench MVP
Moteva-style AI design workbench replica. Home prompt creates a project; the project page provides an infinite canvas with node drag, zoom/pan, a design chat panel, history replay, image asset upload, and project save/regenerate. - Backend: go-zero, DDD layering, sqlc/pgx, optional PostgreSQL; memory or Redis cache; asynq job queue; MinIO/S3/R2/OSS object storage; sky-valley/pi agent runtime adapter. - Frontend: Next.js App Router + Vite artifact build, TypeScript, i18n, shadcn/ui components, auth (OTP/Turnstile/Google/WeChat). - Deploy: Docker Compose and k3s manifests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,266 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"img_infinite_canvas/internal/domain/design"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/zeromicro/go-zero/core/logx"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQueueName = "default"
|
||||
defaultConcurrency = 10
|
||||
defaultMaxRetry = 3
|
||||
defaultTimeout = 10 * time.Minute
|
||||
defaultShutdownTimeout = 30 * time.Second
|
||||
defaultCompletedTaskRetention = 24 * time.Hour
|
||||
)
|
||||
|
||||
var ErrMissingProcessor = errors.New("job processor is required")
|
||||
|
||||
type AsynqOptions struct {
|
||||
Addr string
|
||||
Password string
|
||||
DB int
|
||||
Queue string
|
||||
Concurrency int
|
||||
MaxRetry int
|
||||
TimeoutSeconds int
|
||||
ShutdownTimeoutSeconds int
|
||||
}
|
||||
|
||||
type AsynqQueue struct {
|
||||
client *asynq.Client
|
||||
queue string
|
||||
maxRetry int
|
||||
timeout time.Duration
|
||||
retention time.Duration
|
||||
}
|
||||
|
||||
type AsynqWorker struct {
|
||||
server *asynq.Server
|
||||
mux *asynq.ServeMux
|
||||
}
|
||||
|
||||
func NewAsynqQueue(ctx context.Context, opts AsynqOptions) (*AsynqQueue, error) {
|
||||
normalized := normalizeAsynqOptions(opts)
|
||||
if err := pingRedis(ctx, normalized); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &AsynqQueue{
|
||||
client: asynq.NewClient(redisClientOpt(normalized)),
|
||||
queue: normalized.Queue,
|
||||
maxRetry: normalized.MaxRetry,
|
||||
timeout: timeoutDuration(normalized),
|
||||
retention: defaultCompletedTaskRetention,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *AsynqQueue) Enqueue(ctx context.Context, job design.Job) error {
|
||||
if job.Kind == "" {
|
||||
return fmt.Errorf("%w: job kind is required", design.ErrInvalidInput)
|
||||
}
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task := asynq.NewTask(string(job.Kind), payload)
|
||||
_, err = q.client.EnqueueContext(
|
||||
ctx,
|
||||
task,
|
||||
asynq.Queue(q.queue),
|
||||
asynq.MaxRetry(maxRetryForJob(job, q.maxRetry)),
|
||||
asynq.Timeout(q.timeout),
|
||||
asynq.Retention(q.retention),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (q *AsynqQueue) Close() error {
|
||||
if q == nil || q.client == nil {
|
||||
return nil
|
||||
}
|
||||
return q.client.Close()
|
||||
}
|
||||
|
||||
func NewAsynqWorker(opts AsynqOptions, processor design.JobProcessor) (*AsynqWorker, error) {
|
||||
if processor == nil {
|
||||
return nil, ErrMissingProcessor
|
||||
}
|
||||
normalized := normalizeAsynqOptions(opts)
|
||||
server := asynq.NewServer(redisClientOpt(normalized), asynq.Config{
|
||||
Concurrency: normalized.Concurrency,
|
||||
Queues: map[string]int{normalized.Queue: 1},
|
||||
ShutdownTimeout: shutdownTimeoutDuration(normalized),
|
||||
ErrorHandler: asynq.ErrorHandlerFunc(func(ctx context.Context, task *asynq.Task, err error) {
|
||||
handleAsynqTaskFailure(ctx, task, err, processor)
|
||||
}),
|
||||
})
|
||||
mux := asynq.NewServeMux()
|
||||
worker := &AsynqWorker{server: server, mux: mux}
|
||||
for _, kind := range knownJobKinds() {
|
||||
mux.HandleFunc(string(kind), worker.handle(processor))
|
||||
}
|
||||
return worker, nil
|
||||
}
|
||||
|
||||
func (w *AsynqWorker) Start() error {
|
||||
if w == nil || w.server == nil || w.mux == nil {
|
||||
return nil
|
||||
}
|
||||
return w.server.Start(w.mux)
|
||||
}
|
||||
|
||||
func (w *AsynqWorker) Run() error {
|
||||
if w == nil || w.server == nil || w.mux == nil {
|
||||
return nil
|
||||
}
|
||||
return w.server.Run(w.mux)
|
||||
}
|
||||
|
||||
func (w *AsynqWorker) Shutdown() {
|
||||
if w == nil || w.server == nil {
|
||||
return
|
||||
}
|
||||
w.server.Shutdown()
|
||||
}
|
||||
|
||||
func (w *AsynqWorker) handle(processor design.JobProcessor) func(context.Context, *asynq.Task) error {
|
||||
return func(ctx context.Context, task *asynq.Task) error {
|
||||
job, err := decodeTaskJob(task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return processor.ProcessJob(ctx, job)
|
||||
}
|
||||
}
|
||||
|
||||
func handleAsynqTaskFailure(ctx context.Context, task *asynq.Task, failure error, processor design.JobProcessor) {
|
||||
retried, retryCountOK := asynq.GetRetryCount(ctx)
|
||||
maxRetry, maxRetryOK := asynq.GetMaxRetry(ctx)
|
||||
logx.Errorf("asynq task %s failed retry=%d/%d: %v", task.Type(), retried, maxRetry, failure)
|
||||
handler, ok := processor.(design.JobFailureHandler)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
job, err := decodeTaskJob(task)
|
||||
if err != nil {
|
||||
logx.Errorf("decode failed asynq task %s payload: %v", task.Type(), err)
|
||||
return
|
||||
}
|
||||
if !shouldNotifyJobFailure(job, retried, maxRetry, retryCountOK && maxRetryOK) {
|
||||
return
|
||||
}
|
||||
if err := handler.HandleJobFailure(context.Background(), job, failure); err != nil {
|
||||
logx.Errorf("handle failed asynq task %s failure: %v", task.Type(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeTaskJob(task *asynq.Task) (design.Job, error) {
|
||||
var job design.Job
|
||||
if err := json.Unmarshal(task.Payload(), &job); err != nil {
|
||||
return design.Job{}, err
|
||||
}
|
||||
taskKind := design.JobKind(task.Type())
|
||||
if job.Kind == "" {
|
||||
job.Kind = taskKind
|
||||
}
|
||||
if job.Kind != taskKind {
|
||||
return design.Job{}, fmt.Errorf("%w: payload kind %s does not match task type %s", design.ErrInvalidInput, job.Kind, taskKind)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func shouldNotifyJobFailure(job design.Job, retried int, maxRetry int, retryMetadataOK bool) bool {
|
||||
if isVisualGenerationJob(job.Kind) {
|
||||
return true
|
||||
}
|
||||
return retryMetadataOK && maxRetry >= 0 && retried >= maxRetry
|
||||
}
|
||||
|
||||
func maxRetryForJob(job design.Job, fallback int) int {
|
||||
if isVisualGenerationJob(job.Kind) {
|
||||
return 0
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func isVisualGenerationJob(kind design.JobKind) bool {
|
||||
switch kind {
|
||||
case design.JobCreateProjectGeneration, design.JobFollowupGeneration, design.JobNodeActionGeneration, design.JobMockupModelGeneration:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func knownJobKinds() []design.JobKind {
|
||||
return []design.JobKind{
|
||||
design.JobCreateProjectGeneration,
|
||||
design.JobFollowupGeneration,
|
||||
design.JobNodeActionGeneration,
|
||||
design.JobAgentConversation,
|
||||
design.JobTextExtraction,
|
||||
design.JobMockupModelGeneration,
|
||||
design.JobLayerMergeGeneration,
|
||||
design.JobAssetCleanup,
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeAsynqOptions(opts AsynqOptions) AsynqOptions {
|
||||
opts.Addr = strings.TrimSpace(opts.Addr)
|
||||
opts.Queue = strings.TrimSpace(opts.Queue)
|
||||
if opts.Queue == "" {
|
||||
opts.Queue = defaultQueueName
|
||||
}
|
||||
if opts.Concurrency <= 0 {
|
||||
opts.Concurrency = defaultConcurrency
|
||||
}
|
||||
if opts.MaxRetry < 0 {
|
||||
opts.MaxRetry = 0
|
||||
}
|
||||
if opts.MaxRetry == 0 {
|
||||
opts.MaxRetry = defaultMaxRetry
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func redisClientOpt(opts AsynqOptions) asynq.RedisClientOpt {
|
||||
return asynq.RedisClientOpt{
|
||||
Addr: opts.Addr,
|
||||
Password: opts.Password,
|
||||
DB: opts.DB,
|
||||
}
|
||||
}
|
||||
|
||||
func timeoutDuration(opts AsynqOptions) time.Duration {
|
||||
if opts.TimeoutSeconds <= 0 {
|
||||
return defaultTimeout
|
||||
}
|
||||
return time.Duration(opts.TimeoutSeconds) * time.Second
|
||||
}
|
||||
|
||||
func shutdownTimeoutDuration(opts AsynqOptions) time.Duration {
|
||||
if opts.ShutdownTimeoutSeconds <= 0 {
|
||||
return defaultShutdownTimeout
|
||||
}
|
||||
return time.Duration(opts.ShutdownTimeoutSeconds) * time.Second
|
||||
}
|
||||
|
||||
func pingRedis(ctx context.Context, opts AsynqOptions) error {
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: opts.Addr,
|
||||
Password: opts.Password,
|
||||
DB: opts.DB,
|
||||
})
|
||||
defer client.Close()
|
||||
return client.Ping(ctx).Err()
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"img_infinite_canvas/internal/domain/design"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
type fakeJobProcessor struct {
|
||||
job design.Job
|
||||
failedJob design.Job
|
||||
failure error
|
||||
}
|
||||
|
||||
func (p *fakeJobProcessor) ProcessJob(ctx context.Context, job design.Job) error {
|
||||
p.job = job
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *fakeJobProcessor) HandleJobFailure(ctx context.Context, job design.Job, err error) error {
|
||||
p.failedJob = job
|
||||
p.failure = err
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAsynqWorkerDispatchesJobPayload(t *testing.T) {
|
||||
processor := &fakeJobProcessor{}
|
||||
worker, err := NewAsynqWorker(AsynqOptions{}, processor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
task := asynq.NewTask(string(design.JobTextExtraction), []byte(`{"Kind":"canvas:text-extraction","ProjectID":"project-1","ThreadID":"thread-1","Feature":"edit-text"}`))
|
||||
if err := worker.handle(processor)(context.Background(), task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if processor.job.Kind != design.JobTextExtraction || processor.job.ProjectID != "project-1" || processor.job.Feature != "edit-text" {
|
||||
t.Fatalf("unexpected dispatched job: %#v", processor.job)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsynqFailureHandlerReportsVisualGenerationFailures(t *testing.T) {
|
||||
processor := &fakeJobProcessor{}
|
||||
task := asynq.NewTask(string(design.JobCreateProjectGeneration), []byte(`{"Kind":"canvas:create-project-generation","ProjectID":"project-1","ThreadID":"thread-1"}`))
|
||||
failure := context.DeadlineExceeded
|
||||
|
||||
handleAsynqTaskFailure(context.Background(), task, failure, processor)
|
||||
|
||||
if processor.failure == nil {
|
||||
t.Fatal("expected visual generation failure to be reported")
|
||||
}
|
||||
if processor.failedJob.Kind != design.JobCreateProjectGeneration || processor.failedJob.ProjectID != "project-1" || processor.failedJob.ThreadID != "thread-1" {
|
||||
t.Fatalf("unexpected failed job: %#v", processor.failedJob)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxRetryForVisualGenerationJobsDisablesQueueRetry(t *testing.T) {
|
||||
if got := maxRetryForJob(design.Job{Kind: design.JobCreateProjectGeneration}, 3); got != 0 {
|
||||
t.Fatalf("expected visual generation max retry 0, got %d", got)
|
||||
}
|
||||
if got := maxRetryForJob(design.Job{Kind: design.JobMockupModelGeneration}, 3); got != 0 {
|
||||
t.Fatalf("expected mockup model generation max retry 0, got %d", got)
|
||||
}
|
||||
if got := maxRetryForJob(design.Job{Kind: design.JobTextExtraction}, 3); got != 3 {
|
||||
t.Fatalf("expected non visual generation max retry passthrough, got %d", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user