root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(migrations): Harden task audit tracking and optimize article templates

Browse Source 
- Added migration to harden task audit tracking by modifying audit_logs and related tables.
- Introduced operator_id to several tables for better tracking of actions.
- Updated article_templates with new prompt templates for various article types, enhancing content generation.
- Created prompt_rules and schedule_tasks tables to manage content generation rules and scheduling.
- Added foreign key constraints to articles for better data integrity.
This commit is contained in:
liangxu
2026-04-02 00:31:28 +08:00
parent de30497f59
commit b31d8d0096
101 changed files with 16671 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/shared/auditlog/async.go
+278
View File
@@ -0,0 +1,278 @@
package auditlog
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.uber.org/zap"
)
const (
defaultBatchSize = 64
defaultFlushInterval = time.Second
defaultFlushTimeout = 5 * time.Second
)
type Entry struct {
OperatorID int64
TenantID *int64
Module string
Action string
ResourceType *string
ResourceID *int64
RequestID *string
BeforeJSON []byte
AfterJSON []byte
Result *string
ErrorMessage *string
}
type AsyncWriter struct {
pool *pgxpool.Pool
logger *zap.Logger
batchSize int
flushInterval time.Duration
flushTimeout time.Duration
mu sync.Mutex
pending []Entry
readPos int
signal chan struct{}
closed bool
wg sync.WaitGroup
}
func NewAsyncWriter(pool *pgxpool.Pool, logger *zap.Logger) *AsyncWriter {
writer := &AsyncWriter{
pool: pool,
logger: logger,
batchSize: defaultBatchSize,
flushInterval: defaultFlushInterval,
flushTimeout: defaultFlushTimeout,
signal: make(chan struct{}, 1),
}
writer.wg.Add(1)
go writer.run()
return writer
}
func (w *AsyncWriter) Log(entry Entry) bool {
if w == nil {
return false
}
w.mu.Lock()
if w.closed {
w.mu.Unlock()
return false
}
w.pending = append(w.pending, entry)
w.mu.Unlock()
select {
case w.signal <- struct{}{}:
default:
}
return true
}
func (w *AsyncWriter) Close(ctx context.Context) error {
if w == nil {
return nil
}
w.mu.Lock()
if !w.closed {
w.closed = true
}
w.mu.Unlock()
select {
case w.signal <- struct{}{}:
default:
}
done := make(chan struct{})
go func() {
defer close(done)
w.wg.Wait()
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (w *AsyncWriter) run() {
defer w.wg.Done()
ticker := time.NewTicker(w.flushInterval)
defer ticker.Stop()
for {
select {
case <-w.signal:
case <-ticker.C:
}
for {
batch, closed := w.nextBatch()
if len(batch) == 0 {
if closed {
return
}
break
}
if err := w.flush(batch); err != nil {
w.logger.Warn("audit log batch flush failed; keeping backlog for retry",
zap.Int("count", len(batch)),
zap.Error(err),
)
w.requeueFront(batch)
time.Sleep(500 * time.Millisecond)
break
}
}
}
}
func (w *AsyncWriter) flush(entries []Entry) error {
ctx, cancel := context.WithTimeout(context.Background(), w.flushTimeout)
defer cancel()
query, args := buildInsert(entries)
if _, err := w.pool.Exec(ctx, query, args...); err != nil {
return err
}
return nil
}
func (w *AsyncWriter) nextBatch() ([]Entry, bool) {
w.mu.Lock()
defer w.mu.Unlock()
available := len(w.pending) - w.readPos
if available == 0 {
if w.readPos > 0 {
w.pending = nil
w.readPos = 0
}
return nil, w.closed
}
size := w.batchSize
if available < size {
size = available
}
batch := append([]Entry(nil), w.pending[w.readPos:w.readPos+size]...)
w.readPos += size
if w.readPos == len(w.pending) {
w.pending = nil
w.readPos = 0
} else if w.readPos >= len(w.pending)/2 {
remaining := append([]Entry(nil), w.pending[w.readPos:]...)
w.pending = remaining
w.readPos = 0
}
return batch, w.closed
}
func (w *AsyncWriter) requeueFront(entries []Entry) {
w.mu.Lock()
defer w.mu.Unlock()
if len(entries) == 0 {
return
}
if w.readPos > 0 {
remaining := append([]Entry(nil), w.pending[w.readPos:]...)
w.pending = remaining
w.readPos = 0
}
w.pending = append(append([]Entry(nil), entries...), w.pending...)
}
func buildInsert(entries []Entry) (string, []interface{}) {
var builder strings.Builder
builder.WriteString(`INSERT INTO audit_logs (
operator_id,
tenant_id,
module,
action,
resource_type,
resource_id,
request_id,
before_json,
after_json,
result,
error_message
) VALUES `)
args := make([]interface{}, 0, len(entries)*11)
placeholder := 1
for i, entry := range entries {
if i > 0 {
builder.WriteString(",")
}
builder.WriteString("(")
for col := 0; col < 11; col++ {
if col > 0 {
builder.WriteString(",")
}
builder.WriteString(fmt.Sprintf("$%d", placeholder))
placeholder++
}
builder.WriteString(")")
args = append(args,
entry.OperatorID,
derefInt64(entry.TenantID),
entry.Module,
entry.Action,
derefString(entry.ResourceType),
derefInt64(entry.ResourceID),
derefString(entry.RequestID),
nilIfEmptyBytes(entry.BeforeJSON),
nilIfEmptyBytes(entry.AfterJSON),
derefString(entry.Result),
derefString(entry.ErrorMessage),
)
}
return builder.String(), args
}
func derefString(value *string) interface{} {
if value == nil {
return nil
}
return *value
}
func derefInt64(value *int64) interface{} {
if value == nil {
return nil
}
return *value
}
func nilIfEmptyBytes(value []byte) interface{} {
if len(value) == 0 {
return nil
}
return value
}
server/internal/shared/cache/cache.go
Vendored
+15
View File
@@ -0,0 +1,15 @@
package cache
import (
"context"
"errors"
"time"
)
var ErrNotFound = errors.New("cache: key not found")
type Cache interface {
Get(ctx context.Context, key string) ([]byte, error)
Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
Delete(ctx context.Context, key string) error
}
server/internal/shared/cache/factory.go
Vendored
+12
View File
@@ -0,0 +1,12 @@
package cache
import goredis "github.com/redis/go-redis/v9"
func New(driver string, rdb *goredis.Client) Cache {
switch driver {
case "redis":
return newRedisCache(rdb)
default:
return newMemoryCache()
}
}
server/internal/shared/cache/memory.go
Vendored
+74
View File
@@ -0,0 +1,74 @@
package cache
import (
"context"
"sync"
"time"
)
type memoryEntry struct {
data []byte
expiresAt time.Time
}
type memoryCache struct {
mu sync.RWMutex
entries map[string]memoryEntry
}
func newMemoryCache() Cache {
c := &memoryCache{entries: make(map[string]memoryEntry)}
go c.evictLoop()
return c
}
func (c *memoryCache) Get(_ context.Context, key string) ([]byte, error) {
c.mu.RLock()
entry, ok := c.entries[key]
c.mu.RUnlock()
if !ok || time.Now().After(entry.expiresAt) {
if ok {
c.mu.Lock()
delete(c.entries, key)
c.mu.Unlock()
}
return nil, ErrNotFound
}
dst := make([]byte, len(entry.data))
copy(dst, entry.data)
return dst, nil
}
func (c *memoryCache) Set(_ context.Context, key string, value []byte, ttl time.Duration) error {
dst := make([]byte, len(value))
copy(dst, value)
c.mu.Lock()
c.entries[key] = memoryEntry{data: dst, expiresAt: time.Now().Add(ttl)}
c.mu.Unlock()
return nil
}
func (c *memoryCache) Delete(_ context.Context, key string) error {
c.mu.Lock()
delete(c.entries, key)
c.mu.Unlock()
return nil
}
func (c *memoryCache) evictLoop() {
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()
for range ticker.C {
now := time.Now()
c.mu.Lock()
for k, v := range c.entries {
if now.After(v.expiresAt) {
delete(c.entries, k)
}
}
c.mu.Unlock()
}
}
server/internal/shared/cache/redis.go
Vendored
+33
View File
@@ -0,0 +1,33 @@
package cache
import (
"context"
"errors"
"time"
goredis "github.com/redis/go-redis/v9"
)
type redisCache struct {
client *goredis.Client
}
func newRedisCache(client *goredis.Client) Cache {
return &redisCache{client: client}
}
func (c *redisCache) Get(ctx context.Context, key string) ([]byte, error) {
val, err := c.client.Get(ctx, key).Bytes()
if errors.Is(err, goredis.Nil) {
return nil, ErrNotFound
}
return val, err
}
func (c *redisCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
return c.client.Set(ctx, key, value, ttl).Err()
}
func (c *redisCache) Delete(ctx context.Context, key string) error {
return c.client.Del(ctx, key).Err()
}
server/internal/shared/config/config.go
+12 -3
View File
@@ -13,6 +13,7 @@ type Config struct {
Server ServerConfig `mapstructure:"server"`
Database DatabaseConfig `mapstructure:"database"`
Redis RedisConfig `mapstructure:"redis"`
Cache CacheConfig `mapstructure:"cache"`
JWT JWTConfig `mapstructure:"jwt"`
Log LogConfig `mapstructure:"log"`
LLM LLMConfig `mapstructure:"llm"`
@@ -64,12 +65,20 @@ type LLMConfig struct {
Timeout time.Duration `mapstructure:"timeout"`
MaxOutputTokens int64 `mapstructure:"max_output_tokens"`
Temperature float64 `mapstructure:"temperature"`
TopP float64 `mapstructure:"top_p"`
ReasoningEffort string `mapstructure:"reasoning_effort"`
WebSearchLimit int32 `mapstructure:"web_search_limit"`
}
type CacheConfig struct {
Driver string `mapstructure:"driver"` // "redis" or "memory"
}
type GenerationConfig struct {
QueueSize int `mapstructure:"queue_size"`
WorkerConcurrency int `mapstructure:"worker_concurrency"`
StreamEnabled bool `mapstructure:"stream_enabled"`
QueueSize int `mapstructure:"queue_size"`
WorkerConcurrency int `mapstructure:"worker_concurrency"`
StreamEnabled bool `mapstructure:"stream_enabled"`
ArticleTimeout time.Duration `mapstructure:"article_timeout"`
}
func Load(configPath string) (*Config, error) {
server/internal/shared/contentstats/word_count.go
+36
View File
@@ -0,0 +1,36 @@
package contentstats
import (
"regexp"
"strings"
"unicode"
)
var latinTokenPattern = regexp.MustCompile(`[A-Za-z0-9]+(?:['’-][A-Za-z0-9]+)?`)
func CountWords(input string) int {
input = strings.TrimSpace(input)
if input == "" {
return 0
}
var builder strings.Builder
hanCount := 0
for _, r := range input {
switch {
case unicode.Is(unicode.Han, r):
hanCount++
builder.WriteRune(' ')
case unicode.IsLetter(r) || unicode.IsDigit(r):
builder.WriteRune(r)
case unicode.IsSpace(r):
builder.WriteRune(' ')
default:
builder.WriteRune(' ')
}
}
latinCount := len(latinTokenPattern.FindAllString(builder.String(), -1))
return hanCount + latinCount
}
server/internal/shared/llm/ark.go
+75 -2
View File
@@ -25,7 +25,10 @@ type arkClient struct {
model string
timeout time.Duration
maxOutputTokens int64
webSearchLimit int64
temperature float64
topP *float64
reasoning *responses.ResponsesReasoning
}
func NewArkClient(cfg config.LLMConfig) Client {
@@ -53,17 +56,30 @@ func NewArkClient(cfg config.LLMConfig) Client {
maxOutputTokens = 4000
}
webSearchLimit := int64(cfg.WebSearchLimit)
if webSearchLimit <= 0 {
webSearchLimit = 3
}
temperature := cfg.Temperature
if temperature <= 0 {
temperature = 0.7
}
var topP *float64
if cfg.TopP > 0 {
topP = &cfg.TopP
}
return &arkClient{
client: arkruntime.NewClientWithApiKey(cfg.APIKey, arkruntime.WithBaseUrl(baseURL)),
model: model,
timeout: timeout,
maxOutputTokens: maxOutputTokens,
webSearchLimit: webSearchLimit,
temperature: temperature,
topP: topP,
reasoning: resolveArkReasoning(cfg.ReasoningEffort),
}
}
@@ -87,10 +103,21 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
return nil, errors.New("prompt is required")
}
callCtx, cancel := context.WithTimeout(ctx, c.timeout)
timeout := c.timeout
if req.Timeout > 0 {
timeout = req.Timeout
}
maxOutputTokens := c.maxOutputTokens
if req.MaxOutputTokens > 0 {
maxOutputTokens = req.MaxOutputTokens
}
callCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
store := false
streamValue := true
inputMessage := &responses.ItemInputMessage{
Role: responses.MessageRole_user,
Content: []*responses.ContentItem{
@@ -105,11 +132,34 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
},
}
var tools []*responses.ResponsesTool
if req.WebSearch != nil && req.WebSearch.Enabled {
webSearchLimit := c.webSearchLimit
if req.WebSearch.Limit > 0 {
webSearchLimit = int64(req.WebSearch.Limit)
}
tools = []*responses.ResponsesTool{
{
Union: &responses.ResponsesTool_ToolWebSearch{
ToolWebSearch: &responses.ToolWebSearch{
Type: responses.ToolType_web_search,
Limit: &webSearchLimit,
},
},
},
}
}
stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{
Model: c.model,
MaxOutputTokens: &c.maxOutputTokens,
MaxOutputTokens: &maxOutputTokens,
Store: &store,
Stream: &streamValue,
Temperature: &c.temperature,
TopP: c.topP,
Tools: tools,
Reasoning: c.reasoning,
Input: &responses.ResponsesInput{
Union: &responses.ResponsesInput_ListValue{
ListValue: &responses.InputItemList{
@@ -125,6 +175,9 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
},
})
if err != nil {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(callCtx.Err(), context.DeadlineExceeded) {
return nil, fmt.Errorf("ark response stream timeout after %s: %w", timeout, err)
}
return nil, fmt.Errorf("create ark response stream: %w", err)
}
defer stream.Close()
@@ -137,6 +190,9 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
if errors.Is(err, io.EOF) {
break
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(callCtx.Err(), context.DeadlineExceeded) {
return nil, fmt.Errorf("ark response stream timeout after %s: %w", timeout, err)
}
return nil, fmt.Errorf("receive ark stream event: %w", err)
}
@@ -176,3 +232,20 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
Model: c.model,
}, nil
}
func resolveArkReasoning(value string) *responses.ResponsesReasoning {
switch strings.ToLower(strings.TrimSpace(value)) {
case "":
return nil
case "minimal":
return &responses.ResponsesReasoning{Effort: responses.ReasoningEffort_minimal}
case "low":
return &responses.ResponsesReasoning{Effort: responses.ReasoningEffort_low}
case "medium":
return &responses.ResponsesReasoning{Effort: responses.ReasoningEffort_medium}
case "high":
return &responses.ResponsesReasoning{Effort: responses.ReasoningEffort_high}
default:
return nil
}
}
server/internal/shared/llm/client.go
+10 -1
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"strings"
"time"
"github.com/geo-platform/tenant-api/internal/shared/config"
)
@@ -12,7 +13,10 @@ import (
var ErrNotConfigured = errors.New("llm provider is not configured")
type GenerateRequest struct {
Prompt string
Prompt string
Timeout time.Duration
MaxOutputTokens int64
WebSearch *WebSearchOptions
}
type GenerateResult struct {
@@ -20,6 +24,11 @@ type GenerateResult struct {
Model string
}
type WebSearchOptions struct {
Enabled bool
Limit int32
}
type Client interface {
Validate() error
Generate(ctx context.Context, req GenerateRequest, onDelta func(string)) (*GenerateResult, error)
server/internal/shared/middleware/request_id.go
+12
View File
@@ -1,6 +1,8 @@
package middleware
import (
"context"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
@@ -8,6 +10,8 @@ import (
const RequestIDHeader = "X-Request-ID"
const requestIDKey = "request_id"
type requestIDContextKey struct{}
func RequestID() gin.HandlerFunc {
return func(c *gin.Context) {
rid := c.GetHeader(RequestIDHeader)
@@ -15,6 +19,7 @@ func RequestID() gin.HandlerFunc {
rid = uuid.New().String()
}
c.Set(requestIDKey, rid)
c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), requestIDContextKey{}, rid))
c.Header(RequestIDHeader, rid)
c.Next()
}
@@ -26,3 +31,10 @@ func RequestIDFromGin(c *gin.Context) string {
}
return ""
}
func RequestIDFromContext(ctx context.Context) string {
if rid, ok := ctx.Value(requestIDContextKey{}).(string); ok {
return rid
}
return ""
}