feat(research): hydrate web-search results with cleaned page content

Fetch and clean each search result's page body (bounded by size, concurrency, and a configurable MaxPageContentRunes), store it on the new ResearchResult.Content field, and surface those excerpts in agent memory so the model reasons over real page text rather than snippets alone. Promotes golang.org/x/net to a direct dependency for HTML parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-08 10:56:31 +08:00
parent 222e5a0219
commit 836de5b597
8 changed files with 252 additions and 17 deletions
@@ -1,8 +1,13 @@
 package websearch

 import (
+	"context"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"img_infinite_canvas/internal/domain/design"
 )

 func TestParseDuckDuckGoResults(t *testing.T) {
@@ -39,3 +44,68 @@ func TestBuildVisualSearchQueryForLogoReferencesAvoidsMakerTools(t *testing.T) {
 		t.Fatalf("expected query to avoid raw prompt wording, got %q", query)
 	}
 }
+
+func TestFetchPageContentExtractsReadableDetails(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(`
+<html>
+  <head><title>Ignored title</title><script>console.log("skip")</script></head>
+  <body>
+    <nav>Home Pricing Login</nav>
+    <main>
+      <h1>Japanese bakery brand identity</h1>
+      <p>Warm handmade packaging, hand-drawn bread marks, and soft counter signage.</p>
+      <p>The case study explains reusable color, illustration, and typography choices.</p>
+    </main>
+    <footer>Privacy policy</footer>
+  </body>
+</html>`))
+	}))
+	defer server.Close()
+
+	researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 220})
+	content, err := researcher.fetchPageContent(context.Background(), server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"Japanese bakery brand identity", "Warm handmade packaging", "typography choices"} {
+		if !strings.Contains(content, want) {
+			t.Fatalf("expected page content to contain %q, got %q", want, content)
+		}
+	}
+	for _, unwanted := range []string{"console.log", "Home Pricing Login", "Privacy policy"} {
+		if strings.Contains(content, unwanted) {
+			t.Fatalf("expected page content to omit %q, got %q", unwanted, content)
+		}
+	}
+}
+
+func TestHydrateResultContentsAddsContentForEachResult(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		switch r.URL.Path {
+		case "/one":
+			_, _ = w.Write([]byte(`<article><h1>First reference</h1><p>Detailed first page content for the agent.</p></article>`))
+		case "/two":
+			_, _ = w.Write([]byte(`<article><h1>Second reference</h1><p>Detailed second page content for the agent.</p></article>`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	results := []design.ResearchResult{
+		{Title: "One", URL: server.URL + "/one"},
+		{Title: "Two", URL: server.URL + "/two"},
+	}
+	researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 200})
+	researcher.hydrateResultContents(context.Background(), results)
+
+	if !strings.Contains(results[0].Content, "Detailed first page content") {
+		t.Fatalf("expected first result content, got %#v", results[0])
+	}
+	if !strings.Contains(results[1].Content, "Detailed second page content") {
+		t.Fatalf("expected second result content, got %#v", results[1])
+	}
+}