diff --git a/server/API.md b/server/API.md
index 03581bc..afd5086 100644
--- a/server/API.md
+++ b/server/API.md
@@ -171,7 +171,7 @@ Returns all project summaries, sorted by `updatedAt` descending.
}
```
-`enableWebSearch` only opens the web-search capability. The agent model still decides per request whether search is actually needed; if it is not needed, no web-search step is recorded.
+`enableWebSearch` only opens the web-search capability. The agent model still decides per request whether search is actually needed; if it is not needed, no web-search step is recorded. When search runs, result titles, URLs, snippets, and cleaned page-content excerpts are stored for the agent.
`GET /api/projects/:id`
@@ -210,7 +210,7 @@ Deletes the project record. PostgreSQL cascades canvas nodes, chat messages, his
}
```
-The harness builds short memory from the current turn, recent messages, and visible canvas state, and long memory from durable project decisions, prior artifacts, and previous research records. Image titles and assistant language are generated from the request context instead of fixed model labels.
+The harness builds short memory from the current turn, recent messages, and visible canvas state, and long memory from durable project decisions, prior artifacts, and previous research records with page-content excerpts. Image titles and assistant language are generated from the request context instead of fixed model labels.
## Generate
diff --git a/server/go.mod b/server/go.mod
index 3d95a3b..2b3d01f 100644
--- a/server/go.mod
+++ b/server/go.mod
@@ -16,6 +16,7 @@ require (
github.com/zeromicro/go-zero v1.10.2
golang.org/x/crypto v0.51.0
golang.org/x/image v0.41.0
+ golang.org/x/net v0.53.0
)
require (
@@ -70,7 +71,6 @@ require (
go.uber.org/automaxprocs v1.6.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
- golang.org/x/net v0.53.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.44.0 // indirect
golang.org/x/text v0.37.0 // indirect
diff --git a/server/internal/application/agent_memory.go b/server/internal/application/agent_memory.go
index efa022f..e901f28 100644
--- a/server/internal/application/agent_memory.go
+++ b/server/internal/application/agent_memory.go
@@ -238,6 +238,7 @@ func researchMemory(messages []design.Message) string {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
+ Content string `json:"content"`
} `json:"results"`
}
if err := json.Unmarshal([]byte(message.Content), &report); err != nil {
@@ -255,6 +256,9 @@ func researchMemory(messages []design.Message) string {
if snippet := strings.TrimSpace(result.Snippet); snippet != "" {
line += " | " + shortenMemoryText(snippet, 160)
}
+ if content := strings.TrimSpace(result.Content); content != "" {
+ line += " | page: " + shortenMemoryText(content, 360)
+ }
lines = append(lines, line)
if len(lines) >= 10 {
break
diff --git a/server/internal/config/config.go b/server/internal/config/config.go
index af4735b..3929889 100644
--- a/server/internal/config/config.go
+++ b/server/internal/config/config.go
@@ -161,9 +161,10 @@ type CreativePlannerConfig struct {
}
type ResearchConfig struct {
- Driver string `json:",default=duckduckgo"`
- TimeoutSeconds int `json:",default=10"`
- MaxResults int `json:",default=5"`
+ Driver string `json:",default=duckduckgo"`
+ TimeoutSeconds int `json:",default=10"`
+ MaxResults int `json:",default=5"`
+ MaxPageContentRunes int `json:",default=1800"`
}
type ObjectStorageConfig struct {
diff --git a/server/internal/domain/design/research.go b/server/internal/domain/design/research.go
index 2a74455..821b15f 100644
--- a/server/internal/domain/design/research.go
+++ b/server/internal/domain/design/research.go
@@ -6,6 +6,7 @@ type ResearchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
+ Content string `json:"content,omitempty"`
}
type ResearchReport struct {
diff --git a/server/internal/infrastructure/websearch/duckduckgo.go b/server/internal/infrastructure/websearch/duckduckgo.go
index f2665ed..73e961d 100644
--- a/server/internal/infrastructure/websearch/duckduckgo.go
+++ b/server/internal/infrastructure/websearch/duckduckgo.go
@@ -1,29 +1,41 @@
package websearch
import (
+ "bytes"
"context"
"fmt"
- "html"
+ stdhtml "html"
"io"
"net/http"
"net/url"
"regexp"
"strings"
+ "sync"
"time"
"img_infinite_canvas/internal/domain/design"
+
+ xhtml "golang.org/x/net/html"
)
type DuckDuckGoOptions struct {
- TimeoutSeconds int
- MaxResults int
+ TimeoutSeconds int
+ MaxResults int
+ MaxPageContentRunes int
}
type DuckDuckGoResearcher struct {
- client *http.Client
- maxResults int
+ client *http.Client
+ maxResults int
+ maxPageContentRunes int
}
+const (
+ defaultMaxPageContentRunes = 1800
+ maxPageContentBytes = 2 << 20
+ maxPageContentConcurrency = 4
+)
+
func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
timeout := time.Duration(opts.TimeoutSeconds) * time.Second
if timeout <= 0 {
@@ -33,9 +45,14 @@ func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
if maxResults <= 0 {
maxResults = 5
}
+ maxPageContentRunes := opts.MaxPageContentRunes
+ if maxPageContentRunes <= 0 {
+ maxPageContentRunes = defaultMaxPageContentRunes
+ }
return &DuckDuckGoResearcher{
- client: &http.Client{Timeout: timeout},
- maxResults: maxResults,
+ client: &http.Client{Timeout: timeout},
+ maxResults: maxResults,
+ maxPageContentRunes: maxPageContentRunes,
}
}
@@ -74,9 +91,74 @@ func (r *DuckDuckGoResearcher) Research(ctx context.Context, prompt string) (des
if len(report.Results) == 0 {
return report, fmt.Errorf("web search returned no results")
}
+ r.hydrateResultContents(ctx, report.Results)
return report, nil
}
+func (r *DuckDuckGoResearcher) hydrateResultContents(ctx context.Context, results []design.ResearchResult) {
+ if r.maxPageContentRunes <= 0 || len(results) == 0 {
+ return
+ }
+
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, maxPageContentConcurrency)
+ for index := range results {
+ if strings.TrimSpace(results[index].URL) == "" {
+ continue
+ }
+ wg.Add(1)
+ go func(index int) {
+ defer wg.Done()
+ select {
+ case sem <- struct{}{}:
+ defer func() { <-sem }()
+ case <-ctx.Done():
+ return
+ }
+ content, err := r.fetchPageContent(ctx, results[index].URL)
+ if err == nil && content != "" {
+ results[index].Content = content
+ }
+ }(index)
+ }
+ wg.Wait()
+}
+
+func (r *DuckDuckGoResearcher) fetchPageContent(ctx context.Context, rawURL string) (string, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+ if err != nil {
+ return "", err
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MotevaDesignAgent/1.0)")
+ req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.3")
+
+ resp, err := r.client.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return "", fmt.Errorf("fetch page failed: %s", resp.Status)
+ }
+
+ contentType := strings.ToLower(resp.Header.Get("Content-Type"))
+ if contentType != "" &&
+ !strings.Contains(contentType, "text/html") &&
+ !strings.Contains(contentType, "application/xhtml+xml") &&
+ !strings.Contains(contentType, "text/plain") {
+ return "", nil
+ }
+
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxPageContentBytes))
+ if err != nil {
+ return "", err
+ }
+ if strings.Contains(contentType, "text/plain") {
+ return cleanPageContentText(string(body), r.maxPageContentRunes), nil
+ }
+ return extractReadableHTMLText(body, r.maxPageContentRunes), nil
+}
+
func BuildVisualSearchQuery(prompt string) string {
prompt = cleanPromptForSearch(prompt)
if prompt == "" {
@@ -286,7 +368,7 @@ func parseDuckDuckGoResults(body string, maxResults int) []design.ResearchResult
}
func resolveDuckDuckGoURL(raw string) string {
- raw = html.UnescapeString(strings.TrimSpace(raw))
+ raw = stdhtml.UnescapeString(strings.TrimSpace(raw))
if strings.HasPrefix(raw, "//") {
raw = "https:" + raw
}
@@ -308,11 +390,87 @@ func resolveDuckDuckGoURL(raw string) string {
func cleanHTML(value string) string {
value = tagRe.ReplaceAllString(value, " ")
- value = html.UnescapeString(value)
+ value = stdhtml.UnescapeString(value)
value = spaceRe.ReplaceAllString(value, " ")
return strings.TrimSpace(value)
}
+func extractReadableHTMLText(body []byte, maxRunes int) string {
+ doc, err := xhtml.Parse(bytes.NewReader(body))
+ if err != nil {
+ return cleanPageContentText(cleanHTML(string(body)), maxRunes)
+ }
+
+ parts := make([]string, 0, 64)
+ runeCount := 0
+ collectReadableText(doc, false, &parts, &runeCount, maxRunes)
+ return cleanPageContentText(strings.Join(parts, " "), maxRunes)
+}
+
+func collectReadableText(node *xhtml.Node, skip bool, parts *[]string, runeCount *int, maxRunes int) bool {
+ if node == nil || *runeCount >= maxRunes {
+ return true
+ }
+ if node.Type == xhtml.ElementNode && skippedPageTextTag(strings.ToLower(node.Data)) {
+ skip = true
+ }
+ if !skip && node.Type == xhtml.TextNode {
+ text := cleanPageContentText(node.Data, maxRunes)
+ if usefulPageText(text) {
+ *parts = append(*parts, text)
+ *runeCount += len([]rune(text)) + 1
+ if *runeCount >= maxRunes {
+ return true
+ }
+ }
+ }
+ for child := node.FirstChild; child != nil; child = child.NextSibling {
+ if collectReadableText(child, skip, parts, runeCount, maxRunes) {
+ return true
+ }
+ }
+ return false
+}
+
+func skippedPageTextTag(tag string) bool {
+ switch tag {
+ case "head", "script", "style", "noscript", "template", "svg", "canvas", "iframe", "form", "button", "nav", "header", "footer":
+ return true
+ default:
+ return false
+ }
+}
+
+func usefulPageText(text string) bool {
+ text = strings.TrimSpace(text)
+ if len([]rune(text)) < 2 {
+ return false
+ }
+ lower := strings.ToLower(text)
+ if len([]rune(text)) < 80 && (strings.Contains(lower, "cookie") || strings.Contains(lower, "privacy policy")) {
+ return false
+ }
+ return true
+}
+
+func cleanPageContentText(value string, maxRunes int) string {
+ value = stdhtml.UnescapeString(value)
+ value = spaceRe.ReplaceAllString(value, " ")
+ value = strings.TrimSpace(value)
+ return clampRunes(value, maxRunes)
+}
+
+func clampRunes(value string, maxRunes int) string {
+ if maxRunes <= 0 {
+ return value
+ }
+ runes := []rune(value)
+ if len(runes) <= maxRunes {
+ return value
+ }
+ return strings.TrimSpace(string(runes[:maxRunes]))
+}
+
func dedupeKey(raw string) string {
parsed, err := url.Parse(raw)
if err != nil {
diff --git a/server/internal/infrastructure/websearch/duckduckgo_test.go b/server/internal/infrastructure/websearch/duckduckgo_test.go
index 94ca8c3..c1bab9d 100644
--- a/server/internal/infrastructure/websearch/duckduckgo_test.go
+++ b/server/internal/infrastructure/websearch/duckduckgo_test.go
@@ -1,8 +1,13 @@
package websearch
import (
+ "context"
+ "net/http"
+ "net/http/httptest"
"strings"
"testing"
+
+ "img_infinite_canvas/internal/domain/design"
)
func TestParseDuckDuckGoResults(t *testing.T) {
@@ -39,3 +44,68 @@ func TestBuildVisualSearchQueryForLogoReferencesAvoidsMakerTools(t *testing.T) {
t.Fatalf("expected query to avoid raw prompt wording, got %q", query)
}
}
+
+func TestFetchPageContentExtractsReadableDetails(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ _, _ = w.Write([]byte(`
+
+ Ignored title
+
+
+
+ Japanese bakery brand identity
+ Warm handmade packaging, hand-drawn bread marks, and soft counter signage.
+ The case study explains reusable color, illustration, and typography choices.
+
+
+
+`))
+ }))
+ defer server.Close()
+
+ researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 220})
+ content, err := researcher.fetchPageContent(context.Background(), server.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, want := range []string{"Japanese bakery brand identity", "Warm handmade packaging", "typography choices"} {
+ if !strings.Contains(content, want) {
+ t.Fatalf("expected page content to contain %q, got %q", want, content)
+ }
+ }
+ for _, unwanted := range []string{"console.log", "Home Pricing Login", "Privacy policy"} {
+ if strings.Contains(content, unwanted) {
+ t.Fatalf("expected page content to omit %q, got %q", unwanted, content)
+ }
+ }
+}
+
+func TestHydrateResultContentsAddsContentForEachResult(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ switch r.URL.Path {
+ case "/one":
+ _, _ = w.Write([]byte(`First reference
Detailed first page content for the agent.
`))
+ case "/two":
+ _, _ = w.Write([]byte(`Second reference
Detailed second page content for the agent.
`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ results := []design.ResearchResult{
+ {Title: "One", URL: server.URL + "/one"},
+ {Title: "Two", URL: server.URL + "/two"},
+ }
+ researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 200})
+ researcher.hydrateResultContents(context.Background(), results)
+
+ if !strings.Contains(results[0].Content, "Detailed first page content") {
+ t.Fatalf("expected first result content, got %#v", results[0])
+ }
+ if !strings.Contains(results[1].Content, "Detailed second page content") {
+ t.Fatalf("expected second result content, got %#v", results[1])
+ }
+}
diff --git a/server/internal/svc/service_context.go b/server/internal/svc/service_context.go
index fdf6ff6..abeb74e 100644
--- a/server/internal/svc/service_context.go
+++ b/server/internal/svc/service_context.go
@@ -256,8 +256,9 @@ func newResearcher(c config.Config) design.Researcher {
switch opts.Driver {
case "", "duckduckgo":
return websearch.NewDuckDuckGoResearcher(websearch.DuckDuckGoOptions{
- TimeoutSeconds: opts.TimeoutSeconds,
- MaxResults: opts.MaxResults,
+ TimeoutSeconds: opts.TimeoutSeconds,
+ MaxResults: opts.MaxResults,
+ MaxPageContentRunes: opts.MaxPageContentRunes,
})
case "none", "disabled":
return nil