feat(research): hydrate web-search results with cleaned page content
Fetch and clean each search result's page body (bounded by size, concurrency, and a configurable MaxPageContentRunes), store it on the new ResearchResult.Content field, and surface those excerpts in agent memory so the model reasons over real page text rather than snippets alone. Promotes golang.org/x/net to a direct dependency for HTML parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+2
-2
@@ -171,7 +171,7 @@ Returns all project summaries, sorted by `updatedAt` descending.
|
||||
}
|
||||
```
|
||||
|
||||
`enableWebSearch` only opens the web-search capability. The agent model still decides per request whether search is actually needed; if it is not needed, no web-search step is recorded.
|
||||
`enableWebSearch` only opens the web-search capability. The agent model still decides per request whether search is actually needed; if it is not needed, no web-search step is recorded. When search runs, result titles, URLs, snippets, and cleaned page-content excerpts are stored for the agent.
|
||||
|
||||
`GET /api/projects/:id`
|
||||
|
||||
@@ -210,7 +210,7 @@ Deletes the project record. PostgreSQL cascades canvas nodes, chat messages, his
|
||||
}
|
||||
```
|
||||
|
||||
The harness builds short memory from the current turn, recent messages, and visible canvas state, and long memory from durable project decisions, prior artifacts, and previous research records. Image titles and assistant language are generated from the request context instead of fixed model labels.
|
||||
The harness builds short memory from the current turn, recent messages, and visible canvas state, and long memory from durable project decisions, prior artifacts, and previous research records with page-content excerpts. Image titles and assistant language are generated from the request context instead of fixed model labels.
|
||||
|
||||
## Generate
|
||||
|
||||
|
||||
+1
-1
@@ -16,6 +16,7 @@ require (
|
||||
github.com/zeromicro/go-zero v1.10.2
|
||||
golang.org/x/crypto v0.51.0
|
||||
golang.org/x/image v0.41.0
|
||||
golang.org/x/net v0.53.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -70,7 +71,6 @@ require (
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
|
||||
@@ -238,6 +238,7 @@ func researchMemory(messages []design.Message) string {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
Content string `json:"content"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(message.Content), &report); err != nil {
|
||||
@@ -255,6 +256,9 @@ func researchMemory(messages []design.Message) string {
|
||||
if snippet := strings.TrimSpace(result.Snippet); snippet != "" {
|
||||
line += " | " + shortenMemoryText(snippet, 160)
|
||||
}
|
||||
if content := strings.TrimSpace(result.Content); content != "" {
|
||||
line += " | page: " + shortenMemoryText(content, 360)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
if len(lines) >= 10 {
|
||||
break
|
||||
|
||||
@@ -161,9 +161,10 @@ type CreativePlannerConfig struct {
|
||||
}
|
||||
|
||||
type ResearchConfig struct {
|
||||
Driver string `json:",default=duckduckgo"`
|
||||
TimeoutSeconds int `json:",default=10"`
|
||||
MaxResults int `json:",default=5"`
|
||||
Driver string `json:",default=duckduckgo"`
|
||||
TimeoutSeconds int `json:",default=10"`
|
||||
MaxResults int `json:",default=5"`
|
||||
MaxPageContentRunes int `json:",default=1800"`
|
||||
}
|
||||
|
||||
type ObjectStorageConfig struct {
|
||||
|
||||
@@ -6,6 +6,7 @@ type ResearchResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
Content string `json:"content,omitempty"`
|
||||
}
|
||||
|
||||
type ResearchReport struct {
|
||||
|
||||
@@ -1,29 +1,41 @@
|
||||
package websearch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"html"
|
||||
stdhtml "html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"img_infinite_canvas/internal/domain/design"
|
||||
|
||||
xhtml "golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type DuckDuckGoOptions struct {
|
||||
TimeoutSeconds int
|
||||
MaxResults int
|
||||
TimeoutSeconds int
|
||||
MaxResults int
|
||||
MaxPageContentRunes int
|
||||
}
|
||||
|
||||
type DuckDuckGoResearcher struct {
|
||||
client *http.Client
|
||||
maxResults int
|
||||
client *http.Client
|
||||
maxResults int
|
||||
maxPageContentRunes int
|
||||
}
|
||||
|
||||
const (
|
||||
defaultMaxPageContentRunes = 1800
|
||||
maxPageContentBytes = 2 << 20
|
||||
maxPageContentConcurrency = 4
|
||||
)
|
||||
|
||||
func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
|
||||
timeout := time.Duration(opts.TimeoutSeconds) * time.Second
|
||||
if timeout <= 0 {
|
||||
@@ -33,9 +45,14 @@ func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
}
|
||||
maxPageContentRunes := opts.MaxPageContentRunes
|
||||
if maxPageContentRunes <= 0 {
|
||||
maxPageContentRunes = defaultMaxPageContentRunes
|
||||
}
|
||||
return &DuckDuckGoResearcher{
|
||||
client: &http.Client{Timeout: timeout},
|
||||
maxResults: maxResults,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
maxResults: maxResults,
|
||||
maxPageContentRunes: maxPageContentRunes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,9 +91,74 @@ func (r *DuckDuckGoResearcher) Research(ctx context.Context, prompt string) (des
|
||||
if len(report.Results) == 0 {
|
||||
return report, fmt.Errorf("web search returned no results")
|
||||
}
|
||||
r.hydrateResultContents(ctx, report.Results)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *DuckDuckGoResearcher) hydrateResultContents(ctx context.Context, results []design.ResearchResult) {
|
||||
if r.maxPageContentRunes <= 0 || len(results) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, maxPageContentConcurrency)
|
||||
for index := range results {
|
||||
if strings.TrimSpace(results[index].URL) == "" {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
defer func() { <-sem }()
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
content, err := r.fetchPageContent(ctx, results[index].URL)
|
||||
if err == nil && content != "" {
|
||||
results[index].Content = content
|
||||
}
|
||||
}(index)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (r *DuckDuckGoResearcher) fetchPageContent(ctx context.Context, rawURL string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MotevaDesignAgent/1.0)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.3")
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return "", fmt.Errorf("fetch page failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
|
||||
if contentType != "" &&
|
||||
!strings.Contains(contentType, "text/html") &&
|
||||
!strings.Contains(contentType, "application/xhtml+xml") &&
|
||||
!strings.Contains(contentType, "text/plain") {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxPageContentBytes))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.Contains(contentType, "text/plain") {
|
||||
return cleanPageContentText(string(body), r.maxPageContentRunes), nil
|
||||
}
|
||||
return extractReadableHTMLText(body, r.maxPageContentRunes), nil
|
||||
}
|
||||
|
||||
func BuildVisualSearchQuery(prompt string) string {
|
||||
prompt = cleanPromptForSearch(prompt)
|
||||
if prompt == "" {
|
||||
@@ -286,7 +368,7 @@ func parseDuckDuckGoResults(body string, maxResults int) []design.ResearchResult
|
||||
}
|
||||
|
||||
func resolveDuckDuckGoURL(raw string) string {
|
||||
raw = html.UnescapeString(strings.TrimSpace(raw))
|
||||
raw = stdhtml.UnescapeString(strings.TrimSpace(raw))
|
||||
if strings.HasPrefix(raw, "//") {
|
||||
raw = "https:" + raw
|
||||
}
|
||||
@@ -308,11 +390,87 @@ func resolveDuckDuckGoURL(raw string) string {
|
||||
|
||||
func cleanHTML(value string) string {
|
||||
value = tagRe.ReplaceAllString(value, " ")
|
||||
value = html.UnescapeString(value)
|
||||
value = stdhtml.UnescapeString(value)
|
||||
value = spaceRe.ReplaceAllString(value, " ")
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func extractReadableHTMLText(body []byte, maxRunes int) string {
|
||||
doc, err := xhtml.Parse(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return cleanPageContentText(cleanHTML(string(body)), maxRunes)
|
||||
}
|
||||
|
||||
parts := make([]string, 0, 64)
|
||||
runeCount := 0
|
||||
collectReadableText(doc, false, &parts, &runeCount, maxRunes)
|
||||
return cleanPageContentText(strings.Join(parts, " "), maxRunes)
|
||||
}
|
||||
|
||||
func collectReadableText(node *xhtml.Node, skip bool, parts *[]string, runeCount *int, maxRunes int) bool {
|
||||
if node == nil || *runeCount >= maxRunes {
|
||||
return true
|
||||
}
|
||||
if node.Type == xhtml.ElementNode && skippedPageTextTag(strings.ToLower(node.Data)) {
|
||||
skip = true
|
||||
}
|
||||
if !skip && node.Type == xhtml.TextNode {
|
||||
text := cleanPageContentText(node.Data, maxRunes)
|
||||
if usefulPageText(text) {
|
||||
*parts = append(*parts, text)
|
||||
*runeCount += len([]rune(text)) + 1
|
||||
if *runeCount >= maxRunes {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
if collectReadableText(child, skip, parts, runeCount, maxRunes) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func skippedPageTextTag(tag string) bool {
|
||||
switch tag {
|
||||
case "head", "script", "style", "noscript", "template", "svg", "canvas", "iframe", "form", "button", "nav", "header", "footer":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func usefulPageText(text string) bool {
|
||||
text = strings.TrimSpace(text)
|
||||
if len([]rune(text)) < 2 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
if len([]rune(text)) < 80 && (strings.Contains(lower, "cookie") || strings.Contains(lower, "privacy policy")) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func cleanPageContentText(value string, maxRunes int) string {
|
||||
value = stdhtml.UnescapeString(value)
|
||||
value = spaceRe.ReplaceAllString(value, " ")
|
||||
value = strings.TrimSpace(value)
|
||||
return clampRunes(value, maxRunes)
|
||||
}
|
||||
|
||||
func clampRunes(value string, maxRunes int) string {
|
||||
if maxRunes <= 0 {
|
||||
return value
|
||||
}
|
||||
runes := []rune(value)
|
||||
if len(runes) <= maxRunes {
|
||||
return value
|
||||
}
|
||||
return strings.TrimSpace(string(runes[:maxRunes]))
|
||||
}
|
||||
|
||||
func dedupeKey(raw string) string {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
package websearch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"img_infinite_canvas/internal/domain/design"
|
||||
)
|
||||
|
||||
func TestParseDuckDuckGoResults(t *testing.T) {
|
||||
@@ -39,3 +44,68 @@ func TestBuildVisualSearchQueryForLogoReferencesAvoidsMakerTools(t *testing.T) {
|
||||
t.Fatalf("expected query to avoid raw prompt wording, got %q", query)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchPageContentExtractsReadableDetails(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`
|
||||
<html>
|
||||
<head><title>Ignored title</title><script>console.log("skip")</script></head>
|
||||
<body>
|
||||
<nav>Home Pricing Login</nav>
|
||||
<main>
|
||||
<h1>Japanese bakery brand identity</h1>
|
||||
<p>Warm handmade packaging, hand-drawn bread marks, and soft counter signage.</p>
|
||||
<p>The case study explains reusable color, illustration, and typography choices.</p>
|
||||
</main>
|
||||
<footer>Privacy policy</footer>
|
||||
</body>
|
||||
</html>`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 220})
|
||||
content, err := researcher.fetchPageContent(context.Background(), server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"Japanese bakery brand identity", "Warm handmade packaging", "typography choices"} {
|
||||
if !strings.Contains(content, want) {
|
||||
t.Fatalf("expected page content to contain %q, got %q", want, content)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range []string{"console.log", "Home Pricing Login", "Privacy policy"} {
|
||||
if strings.Contains(content, unwanted) {
|
||||
t.Fatalf("expected page content to omit %q, got %q", unwanted, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateResultContentsAddsContentForEachResult(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
switch r.URL.Path {
|
||||
case "/one":
|
||||
_, _ = w.Write([]byte(`<article><h1>First reference</h1><p>Detailed first page content for the agent.</p></article>`))
|
||||
case "/two":
|
||||
_, _ = w.Write([]byte(`<article><h1>Second reference</h1><p>Detailed second page content for the agent.</p></article>`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
results := []design.ResearchResult{
|
||||
{Title: "One", URL: server.URL + "/one"},
|
||||
{Title: "Two", URL: server.URL + "/two"},
|
||||
}
|
||||
researcher := NewDuckDuckGoResearcher(DuckDuckGoOptions{TimeoutSeconds: 2, MaxPageContentRunes: 200})
|
||||
researcher.hydrateResultContents(context.Background(), results)
|
||||
|
||||
if !strings.Contains(results[0].Content, "Detailed first page content") {
|
||||
t.Fatalf("expected first result content, got %#v", results[0])
|
||||
}
|
||||
if !strings.Contains(results[1].Content, "Detailed second page content") {
|
||||
t.Fatalf("expected second result content, got %#v", results[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,8 +256,9 @@ func newResearcher(c config.Config) design.Researcher {
|
||||
switch opts.Driver {
|
||||
case "", "duckduckgo":
|
||||
return websearch.NewDuckDuckGoResearcher(websearch.DuckDuckGoOptions{
|
||||
TimeoutSeconds: opts.TimeoutSeconds,
|
||||
MaxResults: opts.MaxResults,
|
||||
TimeoutSeconds: opts.TimeoutSeconds,
|
||||
MaxResults: opts.MaxResults,
|
||||
MaxPageContentRunes: opts.MaxPageContentRunes,
|
||||
})
|
||||
case "none", "disabled":
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user