root
836de5b597
Fetch and clean each search result's page body (bounded by size, concurrency, and a configurable MaxPageContentRunes), store it on the new ResearchResult.Content field, and surface those excerpts in agent memory so the model reasons over real page text rather than snippets alone. Promotes golang.org/x/net to a direct dependency for HTML parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
112 lines
4.3 KiB
Go
112 lines
4.3 KiB
Go
package websearch
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"img_infinite_canvas/internal/domain/design"
|
|
)
|
|
|
|
func TestParseDuckDuckGoResults(t *testing.T) {
|
|
body := `
|
|
<a rel="nofollow" class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Ffashion&rut=abc">Fashion Website Design</a>
|
|
<a class="result__snippet" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Ffashion&rut=abc">A <b>fashion</b> ecommerce homepage reference.</a>
|
|
<a rel="nofollow" class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fhoodie&rut=def">Hoodie Brand System</a>
|
|
<a class="result__snippet" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fhoodie&rut=def">Brand and hoodie visual language.</a>`
|
|
|
|
results := parseDuckDuckGoResults(body, 5)
|
|
if len(results) != 2 {
|
|
t.Fatalf("expected 2 results, got %d", len(results))
|
|
}
|
|
if results[0].Title != "Fashion Website Design" {
|
|
t.Fatalf("unexpected title: %s", results[0].Title)
|
|
}
|
|
if results[0].URL != "https://example.com/fashion" {
|
|
t.Fatalf("unexpected url: %s", results[0].URL)
|
|
}
|
|
if results[0].Snippet != "A fashion ecommerce homepage reference." {
|
|
t.Fatalf("unexpected snippet: %s", results[0].Snippet)
|
|
}
|
|
}
|
|
|
|
func TestBuildVisualSearchQueryForLogoReferencesAvoidsMakerTools(t *testing.T) {
|
|
query := BuildVisualSearchQuery("Moteva 让设计变简单懂你的设计 Agent，帮我搞定一切设计需求，制作一个 logo")
|
|
|
|
for _, want := range []string{"AI design agent", "logo 设计稿", "设计思路", "brand identity", "-maker", "-generator", "-template"} {
|
|
if !strings.Contains(query, want) {
|
|
t.Fatalf("expected query to contain %q, got %q", want, query)
|
|
}
|
|
}
|
|
if strings.Contains(query, "制作一个 logo") {
|
|
t.Fatalf("expected query to avoid raw prompt wording, got %q", query)
|
|
}
|
|
}
|
|
|
|
func TestFetchPageContentExtractsReadableDetails(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
_, _ = w.Write([]byte(`
|
|
<html>
|
|
<head><title>Ignored title</title><script>console.log("skip")</script></head>
|
|
<body>
|
|
<nav>Home Pricing Login</nav>
|
|
<main>
|
|
<h1>Japanese bakery brand identity</h1>
|
|
<p>Warm handmade packaging, hand-drawn bread marks, and soft counter signage.</p>
|
|
<p>The case study explains reusable color, illustration, and typography choices.</p>
|
|
</main>
|
|
<footer>Privacy policy</footer>
|
|
</body>
|
|
</html>`))
|
|
}))
|
|
defer server.Close()
|
|
|
|
researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 220})
|
|
content, err := researcher.fetchPageContent(context.Background(), server.URL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{"Japanese bakery brand identity", "Warm handmade packaging", "typography choices"} {
|
|
if !strings.Contains(content, want) {
|
|
t.Fatalf("expected page content to contain %q, got %q", want, content)
|
|
}
|
|
}
|
|
for _, unwanted := range []string{"console.log", "Home Pricing Login", "Privacy policy"} {
|
|
if strings.Contains(content, unwanted) {
|
|
t.Fatalf("expected page content to omit %q, got %q", unwanted, content)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHydrateResultContentsAddsContentForEachResult(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
switch r.URL.Path {
|
|
case "/one":
|
|
_, _ = w.Write([]byte(`<article><h1>First reference</h1><p>Detailed first page content for the agent.</p></article>`))
|
|
case "/two":
|
|
_, _ = w.Write([]byte(`<article><h1>Second reference</h1><p>Detailed second page content for the agent.</p></article>`))
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
results := []design.ResearchResult{
|
|
{Title: "One", URL: server.URL + "/one"},
|
|
{Title: "Two", URL: server.URL + "/two"},
|
|
}
|
|
researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 200})
|
|
researcher.hydrateResultContents(context.Background(), results)
|
|
|
|
if !strings.Contains(results[0].Content, "Detailed first page content") {
|
|
t.Fatalf("expected first result content, got %#v", results[0])
|
|
}
|
|
if !strings.Contains(results[1].Content, "Detailed second page content") {
|
|
t.Fatalf("expected second result content, got %#v", results[1])
|
|
}
|
|
}
|