feat(objectstorage): add Aliyun OSS support and auto-detect provider in deploy workflows
Introduces Aliyun OSS as an alternative to MinIO; deploy scripts and CI workflows now read object_storage.provider from config and conditionally include or skip MinIO resources in both Docker Compose and k3s paths. Also adds ops scheduler domain and its migration tables. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -156,14 +156,14 @@ qdrant:
|
||||
timeout: 15s
|
||||
|
||||
object_storage:
|
||||
provider: minio
|
||||
endpoint: localhost:9000
|
||||
access_key: minioadmin
|
||||
secret_key: minioadmin
|
||||
bucket: geo-private
|
||||
use_ssl: false
|
||||
provider: aliyun
|
||||
endpoint: https://oss-cn-shanghai.aliyuncs.com
|
||||
access_key: LTAI5tAEj8euR8B1gXzMoG84
|
||||
secret_key: lJPrKo9ViyHJE8UWIMmMCY0B6Je8v2
|
||||
bucket: shengxintui
|
||||
use_ssl: true
|
||||
public_base_url: ""
|
||||
region: ""
|
||||
region: cn-shanghai
|
||||
|
||||
cache:
|
||||
driver: redis
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
SchedulerTriggerAuto = "auto"
|
||||
SchedulerTriggerManual = "manual"
|
||||
SchedulerTriggerDryRun = "dry_run"
|
||||
|
||||
SchedulerRunRunning = "running"
|
||||
SchedulerRunSuccess = "success"
|
||||
SchedulerRunFailed = "failed"
|
||||
SchedulerRunSkipped = "skipped"
|
||||
)
|
||||
|
||||
type SchedulerJob struct {
|
||||
JobKey string `json:"job_key"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Category string `json:"category"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
ScheduleType string `json:"schedule_type"`
|
||||
IntervalSeconds int `json:"interval_seconds"`
|
||||
CronExpr *string `json:"cron_expr,omitempty"`
|
||||
Timezone string `json:"timezone"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
BatchSize *int `json:"batch_size,omitempty"`
|
||||
MaxConcurrency int `json:"max_concurrency"`
|
||||
Config map[string]any `json:"config"`
|
||||
Version int `json:"version"`
|
||||
UpdatedBy *int64 `json:"updated_by,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
LastRun *SchedulerRun `json:"last_run,omitempty"`
|
||||
RunningRun *SchedulerRun `json:"running_run,omitempty"`
|
||||
PendingTriggers int `json:"pending_triggers"`
|
||||
}
|
||||
|
||||
type SchedulerRun struct {
|
||||
ID int64 `json:"id"`
|
||||
JobKey string `json:"job_key"`
|
||||
TriggerID *int64 `json:"trigger_id,omitempty"`
|
||||
SchedulerInstanceID *string `json:"scheduler_instance_id,omitempty"`
|
||||
TriggerType string `json:"trigger_type"`
|
||||
Status string `json:"status"`
|
||||
ConfigVersion int `json:"config_version"`
|
||||
ConfigSnapshot map[string]any `json:"config_snapshot"`
|
||||
Stats map[string]any `json:"stats"`
|
||||
ErrorMessage *string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
DurationMilliseconds *int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
type SchedulerInstance struct {
|
||||
InstanceID string `json:"instance_id"`
|
||||
Hostname string `json:"hostname"`
|
||||
Process string `json:"process"`
|
||||
BuildVersion *string `json:"build_version,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LastSeenAt time.Time `json:"last_seen_at"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
}
|
||||
|
||||
type SchedulerTrigger struct {
|
||||
ID int64 `json:"id"`
|
||||
JobKey string `json:"job_key"`
|
||||
TriggerType string `json:"trigger_type"`
|
||||
Status string `json:"status"`
|
||||
RequestedBy *int64 `json:"requested_by,omitempty"`
|
||||
Reason *string `json:"reason,omitempty"`
|
||||
ConfigOverride map[string]any `json:"config_override"`
|
||||
SchedulerInstanceID *string `json:"scheduler_instance_id,omitempty"`
|
||||
RunID *int64 `json:"run_id,omitempty"`
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
ClaimedAt *time.Time `json:"claimed_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
ErrorMessage *string `json:"error_message,omitempty"`
|
||||
}
|
||||
@@ -27,7 +27,7 @@ func New(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
|
||||
switch provider {
|
||||
case "", "disabled":
|
||||
return disabledClient{reason: "object storage is disabled"}
|
||||
case "minio":
|
||||
case "minio", "mino":
|
||||
return NewMinIOClient(cfg, logger)
|
||||
case "aliyun", "aliyun_oss", "aliyun-oss", "oss":
|
||||
return NewAliyunClient(cfg, logger)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
DROP TABLE IF EXISTS ops.scheduler_job_triggers;
|
||||
DROP TABLE IF EXISTS ops.scheduler_job_runs;
|
||||
DROP TABLE IF EXISTS ops.scheduler_instances;
|
||||
DROP TABLE IF EXISTS ops.scheduler_jobs;
|
||||
@@ -0,0 +1,104 @@
|
||||
CREATE TABLE IF NOT EXISTS ops.scheduler_jobs (
|
||||
job_key VARCHAR(96) PRIMARY KEY,
|
||||
display_name VARCHAR(120) NOT NULL,
|
||||
category VARCHAR(64) NOT NULL,
|
||||
description TEXT,
|
||||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||||
schedule_type VARCHAR(20) NOT NULL DEFAULT 'interval',
|
||||
interval_seconds INT NOT NULL DEFAULT 300,
|
||||
cron_expr VARCHAR(120),
|
||||
timezone VARCHAR(64) NOT NULL DEFAULT 'Asia/Shanghai',
|
||||
timeout_seconds INT NOT NULL DEFAULT 30,
|
||||
batch_size INT,
|
||||
max_concurrency INT NOT NULL DEFAULT 1,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
version INT NOT NULL DEFAULT 1,
|
||||
updated_by BIGINT,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
CHECK (job_key ~ '^[a-z0-9_:-]+$'),
|
||||
CHECK (schedule_type IN ('interval', 'cron', 'manual')),
|
||||
CHECK (interval_seconds > 0),
|
||||
CHECK (timeout_seconds > 0),
|
||||
CHECK (max_concurrency > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_jobs_category
|
||||
ON ops.scheduler_jobs (category, job_key);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops.scheduler_instances (
|
||||
instance_id VARCHAR(160) PRIMARY KEY,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
process VARCHAR(80) NOT NULL,
|
||||
build_version VARCHAR(120),
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
metadata JSONB NOT NULL DEFAULT '{}'::jsonb
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_instances_seen
|
||||
ON ops.scheduler_instances (last_seen_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops.scheduler_job_runs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_key VARCHAR(96) NOT NULL REFERENCES ops.scheduler_jobs(job_key),
|
||||
trigger_id BIGINT,
|
||||
scheduler_instance_id VARCHAR(160),
|
||||
trigger_type VARCHAR(30) NOT NULL DEFAULT 'auto',
|
||||
status VARCHAR(30) NOT NULL DEFAULT 'running',
|
||||
config_version INT NOT NULL DEFAULT 1,
|
||||
config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
stats JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
error_message TEXT,
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
finished_at TIMESTAMPTZ,
|
||||
duration_ms BIGINT,
|
||||
CHECK (trigger_type IN ('auto', 'manual', 'dry_run')),
|
||||
CHECK (status IN ('running', 'success', 'failed', 'skipped'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_job_started
|
||||
ON ops.scheduler_job_runs (job_key, started_at DESC, id DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_status_started
|
||||
ON ops.scheduler_job_runs (status, started_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops.scheduler_job_triggers (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_key VARCHAR(96) NOT NULL REFERENCES ops.scheduler_jobs(job_key),
|
||||
trigger_type VARCHAR(30) NOT NULL DEFAULT 'manual',
|
||||
status VARCHAR(30) NOT NULL DEFAULT 'pending',
|
||||
requested_by BIGINT,
|
||||
reason TEXT,
|
||||
config_override JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
scheduler_instance_id VARCHAR(160),
|
||||
run_id BIGINT REFERENCES ops.scheduler_job_runs(id),
|
||||
requested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
claimed_at TIMESTAMPTZ,
|
||||
finished_at TIMESTAMPTZ,
|
||||
error_message TEXT,
|
||||
CHECK (trigger_type IN ('manual', 'dry_run')),
|
||||
CHECK (status IN ('pending', 'claimed', 'success', 'failed', 'skipped'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_job_triggers_pending
|
||||
ON ops.scheduler_job_triggers (job_key, requested_at ASC, id ASC)
|
||||
WHERE status = 'pending';
|
||||
|
||||
INSERT INTO ops.scheduler_jobs
|
||||
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
|
||||
VALUES
|
||||
('schedule_dispatch', '内容定时任务派发', 'content', '扫描内容定时任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '为品牌问题按业务日生成 AI 监控采集任务。', true, 'interval', 300, 'Asia/Shanghai', 45, 64, '{}'::jsonb),
|
||||
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '恢复已回调但未成功进入结果队列的监控采集结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '回收超过租约时间仍未回调的监控采集任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, NULL, '{}'::jsonb),
|
||||
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '检查长时间停留在 received 状态的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||
('knowledge_deleted_cleanup', '知识库删除清理', 'knowledge', '清理已删除知识库条目的异步对象与向量数据。', true, 'interval', 15, 'Asia/Shanghai', 30, NULL, '{}'::jsonb),
|
||||
('generation_state_check', '生成任务状态巡检', 'generation', '低频巡检文章生成任务与文章状态一致性。', true, 'interval', 300, 'Asia/Shanghai', 10, 100, '{}'::jsonb),
|
||||
('monitoring_retention_cleanup', '监控历史保留清理', 'monitoring', '按业务日期清理最近 30 天窗口以前的监控历史数据。', true, 'interval', 86400, 'Asia/Shanghai', 900, 5000,
|
||||
'{"retention_days":30,"batch_size":5000,"max_batches_per_run":200,"dry_run":false,"statement_timeout":"5s","run_at_local":"02:30"}'::jsonb)
|
||||
ON CONFLICT (job_key) DO UPDATE
|
||||
SET display_name = EXCLUDED.display_name,
|
||||
category = EXCLUDED.category,
|
||||
description = EXCLUDED.description,
|
||||
updated_at = NOW();
|
||||
Reference in New Issue
Block a user