feat(research): hydrate web-search results with cleaned page content

Fetch and clean each search result's page body (bounded by size,
concurrency, and a configurable MaxPageContentRunes), store it on the
new ResearchResult.Content field, and surface those excerpts in agent
memory so the model reasons over real page text rather than snippets
alone. Promotes golang.org/x/net to a direct dependency for HTML
parsing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

server/internal/infrastructure/websearch/duckduckgo.go

@@ -1,29 +1,41 @@
 package websearch
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"html"
+	stdhtml "html"
 	"io"
 	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"img_infinite_canvas/internal/domain/design"
+
+	xhtml "golang.org/x/net/html"
 )
 
 type DuckDuckGoOptions struct {
-	TimeoutSeconds int
-	MaxResults     int
+	TimeoutSeconds      int
+	MaxResults          int
+	MaxPageContentRunes int
 }
 
 type DuckDuckGoResearcher struct {
-	client     *http.Client
-	maxResults int
+	client              *http.Client
+	maxResults          int
+	maxPageContentRunes int
 }
 
+const (
+	defaultMaxPageContentRunes = 1800
+	maxPageContentBytes        = 2 << 20
+	maxPageContentConcurrency  = 4
+)
+
 func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
 	timeout := time.Duration(opts.TimeoutSeconds) * time.Second
 	if timeout <= 0 {
@@ -33,9 +45,14 @@ func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
 	if maxResults <= 0 {
 		maxResults = 5
 	}
+	maxPageContentRunes := opts.MaxPageContentRunes
+	if maxPageContentRunes <= 0 {
+		maxPageContentRunes = defaultMaxPageContentRunes
+	}
 	return &DuckDuckGoResearcher{
-		client:     &http.Client{Timeout: timeout},
-		maxResults: maxResults,
+		client:              &http.Client{Timeout: timeout},
+		maxResults:          maxResults,
+		maxPageContentRunes: maxPageContentRunes,
 	}
 }
 
@@ -74,9 +91,74 @@ func (r *DuckDuckGoResearcher) Research(ctx context.Context, prompt string) (des
 	if len(report.Results) == 0 {
 		return report, fmt.Errorf("web search returned no results")
 	}
+	r.hydrateResultContents(ctx, report.Results)
 	return report, nil
 }
 
+func (r *DuckDuckGoResearcher) hydrateResultContents(ctx context.Context, results []design.ResearchResult) {
+	if r.maxPageContentRunes <= 0 || len(results) == 0 {
+		return
+	}
+
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, maxPageContentConcurrency)
+	for index := range results {
+		if strings.TrimSpace(results[index].URL) == "" {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			select {
+			case sem <- struct{}{}:
+				defer func() { <-sem }()
+			case <-ctx.Done():
+				return
+			}
+			content, err := r.fetchPageContent(ctx, results[index].URL)
+			if err == nil && content != "" {
+				results[index].Content = content
+			}
+		}(index)
+	}
+	wg.Wait()
+}
+
+func (r *DuckDuckGoResearcher) fetchPageContent(ctx context.Context, rawURL string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MotevaDesignAgent/1.0)")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.3")
+
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("fetch page failed: %s", resp.Status)
+	}
+
+	contentType := strings.ToLower(resp.Header.Get("Content-Type"))
+	if contentType != "" &&
+		!strings.Contains(contentType, "text/html") &&
+		!strings.Contains(contentType, "application/xhtml+xml") &&
+		!strings.Contains(contentType, "text/plain") {
+		return "", nil
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxPageContentBytes))
+	if err != nil {
+		return "", err
+	}
+	if strings.Contains(contentType, "text/plain") {
+		return cleanPageContentText(string(body), r.maxPageContentRunes), nil
+	}
+	return extractReadableHTMLText(body, r.maxPageContentRunes), nil
+}
+
 func BuildVisualSearchQuery(prompt string) string {
 	prompt = cleanPromptForSearch(prompt)
 	if prompt == "" {
@@ -286,7 +368,7 @@ func parseDuckDuckGoResults(body string, maxResults int) []design.ResearchResult
 }
 
 func resolveDuckDuckGoURL(raw string) string {
-	raw = html.UnescapeString(strings.TrimSpace(raw))
+	raw = stdhtml.UnescapeString(strings.TrimSpace(raw))
 	if strings.HasPrefix(raw, "//") {
 		raw = "https:" + raw
 	}
@@ -308,11 +390,87 @@ func resolveDuckDuckGoURL(raw string) string {
 
 func cleanHTML(value string) string {
 	value = tagRe.ReplaceAllString(value, " ")
-	value = html.UnescapeString(value)
+	value = stdhtml.UnescapeString(value)
 	value = spaceRe.ReplaceAllString(value, " ")
 	return strings.TrimSpace(value)
 }
 
+func extractReadableHTMLText(body []byte, maxRunes int) string {
+	doc, err := xhtml.Parse(bytes.NewReader(body))
+	if err != nil {
+		return cleanPageContentText(cleanHTML(string(body)), maxRunes)
+	}
+
+	parts := make([]string, 0, 64)
+	runeCount := 0
+	collectReadableText(doc, false, &parts, &runeCount, maxRunes)
+	return cleanPageContentText(strings.Join(parts, " "), maxRunes)
+}
+
+func collectReadableText(node *xhtml.Node, skip bool, parts *[]string, runeCount *int, maxRunes int) bool {
+	if node == nil || *runeCount >= maxRunes {
+		return true
+	}
+	if node.Type == xhtml.ElementNode && skippedPageTextTag(strings.ToLower(node.Data)) {
+		skip = true
+	}
+	if !skip && node.Type == xhtml.TextNode {
+		text := cleanPageContentText(node.Data, maxRunes)
+		if usefulPageText(text) {
+			*parts = append(*parts, text)
+			*runeCount += len([]rune(text)) + 1
+			if *runeCount >= maxRunes {
+				return true
+			}
+		}
+	}
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if collectReadableText(child, skip, parts, runeCount, maxRunes) {
+			return true
+		}
+	}
+	return false
+}
+
+func skippedPageTextTag(tag string) bool {
+	switch tag {
+	case "head", "script", "style", "noscript", "template", "svg", "canvas", "iframe", "form", "button", "nav", "header", "footer":
+		return true
+	default:
+		return false
+	}
+}
+
+func usefulPageText(text string) bool {
+	text = strings.TrimSpace(text)
+	if len([]rune(text)) < 2 {
+		return false
+	}
+	lower := strings.ToLower(text)
+	if len([]rune(text)) < 80 && (strings.Contains(lower, "cookie") || strings.Contains(lower, "privacy policy")) {
+		return false
+	}
+	return true
+}
+
+func cleanPageContentText(value string, maxRunes int) string {
+	value = stdhtml.UnescapeString(value)
+	value = spaceRe.ReplaceAllString(value, " ")
+	value = strings.TrimSpace(value)
+	return clampRunes(value, maxRunes)
+}
+
+func clampRunes(value string, maxRunes int) string {
+	if maxRunes <= 0 {
+		return value
+	}
+	runes := []rune(value)
+	if len(runes) <= maxRunes {
+		return value
+	}
+	return strings.TrimSpace(string(runes[:maxRunes]))
+}
+
 func dedupeKey(raw string) string {
 	parsed, err := url.Parse(raw)
 	if err != nil {

server/internal/infrastructure/websearch/duckduckgo_test.go

@@ -1,8 +1,13 @@
 package websearch
 
 import (
+	"context"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"img_infinite_canvas/internal/domain/design"
 )
 
 func TestParseDuckDuckGoResults(t *testing.T) {
@@ -39,3 +44,68 @@ func TestBuildVisualSearchQueryForLogoReferencesAvoidsMakerTools(t *testing.T) {
 		t.Fatalf("expected query to avoid raw prompt wording, got %q", query)
 	}
 }
+
+func TestFetchPageContentExtractsReadableDetails(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`
+<html>
+  <head><title>Ignored title</title><script>console.log("skip")</script></head>
+  <body>
+    <nav>Home Pricing Login</nav>
+    <main>
+      <h1>Japanese bakery brand identity</h1>
+      <p>Warm handmade packaging, hand-drawn bread marks, and soft counter signage.</p>
+      <p>The case study explains reusable color, illustration, and typography choices.</p>
+    </main>
+    <footer>Privacy policy</footer>
+  </body>
+</html>`))
+	}))
+	defer server.Close()
+
+	researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 220})
+	content, err := researcher.fetchPageContent(context.Background(), server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"Japanese bakery brand identity", "Warm handmade packaging", "typography choices"} {
+		if !strings.Contains(content, want) {
+			t.Fatalf("expected page content to contain %q, got %q", want, content)
+		}
+	}
+	for _, unwanted := range []string{"console.log", "Home Pricing Login", "Privacy policy"} {
+		if strings.Contains(content, unwanted) {
+			t.Fatalf("expected page content to omit %q, got %q", unwanted, content)
+		}
+	}
+}
+
+func TestHydrateResultContentsAddsContentForEachResult(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		switch r.URL.Path {
+		case "/one":
+			_, _ = w.Write([]byte(`<article><h1>First reference</h1><p>Detailed first page content for the agent.</p></article>`))
+		case "/two":
+			_, _ = w.Write([]byte(`<article><h1>Second reference</h1><p>Detailed second page content for the agent.</p></article>`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	results := []design.ResearchResult{
+		{Title: "One", URL: server.URL + "/one"},
+		{Title: "Two", URL: server.URL + "/two"},
+	}
+	researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 200})
+	researcher.hydrateResultContents(context.Background(), results)
+
+	if !strings.Contains(results[0].Content, "Detailed first page content") {
+		t.Fatalf("expected first result content, got %#v", results[0])
+	}
+	if !strings.Contains(results[1].Content, "Detailed second page content") {
+		t.Fatalf("expected second result content, got %#v", results[1])
+	}
+}