package websearch
import (
"bytes"
"context"
"fmt"
stdhtml "html"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
"img_infinite_canvas/internal/domain/design"
xhtml "golang.org/x/net/html"
)
type DuckDuckGoOptions struct {
TimeoutSeconds int
MaxResults int
MaxPageContentRunes int
}
type DuckDuckGoResearcher struct {
client *http.Client
maxResults int
maxPageContentRunes int
}
const (
defaultMaxPageContentRunes = 1800
maxPageContentBytes = 2 << 20
maxPageContentConcurrency = 4
)
func NewDuckDuckGoResearcher(opts DuckDuckGoOptions) *DuckDuckGoResearcher {
timeout := time.Duration(opts.TimeoutSeconds) * time.Second
if timeout <= 0 {
timeout = 10 * time.Second
}
maxResults := opts.MaxResults
if maxResults <= 0 {
maxResults = 5
}
maxPageContentRunes := opts.MaxPageContentRunes
if maxPageContentRunes <= 0 {
maxPageContentRunes = defaultMaxPageContentRunes
}
return &DuckDuckGoResearcher{
client: &http.Client{Timeout: timeout},
maxResults: maxResults,
maxPageContentRunes: maxPageContentRunes,
}
}
func (r *DuckDuckGoResearcher) Research(ctx context.Context, prompt string) (design.ResearchReport, error) {
query := BuildVisualSearchQuery(prompt)
report := design.ResearchReport{
Kind: "web_search",
Source: "duckduckgo",
Query: query,
}
endpoint := "https://html.duckduckgo.com/html/?q=" + url.QueryEscape(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return report, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MotevaDesignAgent/1.0)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := r.client.Do(req)
if err != nil {
return report, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return report, fmt.Errorf("web search failed: %s", resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
if err != nil {
return report, err
}
report.Results = parseDuckDuckGoResults(string(body), r.maxResults)
if len(report.Results) == 0 {
return report, fmt.Errorf("web search returned no results")
}
r.hydrateResultContents(ctx, report.Results)
return report, nil
}
func (r *DuckDuckGoResearcher) hydrateResultContents(ctx context.Context, results []design.ResearchResult) {
if r.maxPageContentRunes <= 0 || len(results) == 0 {
return
}
var wg sync.WaitGroup
sem := make(chan struct{}, maxPageContentConcurrency)
for index := range results {
if strings.TrimSpace(results[index].URL) == "" {
continue
}
wg.Add(1)
go func(index int) {
defer wg.Done()
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-ctx.Done():
return
}
content, err := r.fetchPageContent(ctx, results[index].URL)
if err == nil && content != "" {
results[index].Content = content
}
}(index)
}
wg.Wait()
}
func (r *DuckDuckGoResearcher) fetchPageContent(ctx context.Context, rawURL string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return "", err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MotevaDesignAgent/1.0)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.3")
resp, err := r.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("fetch page failed: %s", resp.Status)
}
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
if contentType != "" &&
!strings.Contains(contentType, "text/html") &&
!strings.Contains(contentType, "application/xhtml+xml") &&
!strings.Contains(contentType, "text/plain") {
return "", nil
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxPageContentBytes))
if err != nil {
return "", err
}
if strings.Contains(contentType, "text/plain") {
return cleanPageContentText(string(body), r.maxPageContentRunes), nil
}
return extractReadableHTMLText(body, r.maxPageContentRunes), nil
}
func BuildVisualSearchQuery(prompt string) string {
prompt = cleanPromptForSearch(prompt)
if prompt == "" {
return "brand visual design website inspiration"
}
lower := strings.ToLower(prompt)
if isLogoReferencePrompt(prompt, lower) {
return buildLogoReferenceQuery(prompt, lower)
}
keywords := []string{"website design reference", "brand visual style", "ecommerce ui"}
if strings.Contains(lower, "hoodie") || strings.Contains(prompt, "服装") || strings.Contains(prompt, "针织") {
keywords = []string{"fashion ecommerce website design", "hoodie brand visual reference", "streetwear UI"}
}
if strings.Contains(prompt, "海报") || strings.Contains(lower, "poster") {
keywords = []string{"poster design reference", "campaign key visual", "layout inspiration"}
}
if strings.Contains(prompt, "包装") || strings.Contains(lower, "packaging") {
keywords = []string{"packaging design reference", "brand identity system", "visual guidelines"}
}
if strings.Contains(lower, "ui") || strings.Contains(prompt, "界面") {
keywords = []string{"UI design reference", "product interface inspiration", "case study"}
}
return strings.Join(append([]string{prompt}, keywords...), " ")
}
func isLogoReferencePrompt(prompt string, lower string) bool {
return strings.Contains(lower, "logo") ||
strings.Contains(prompt, "品牌标识") ||
strings.Contains(prompt, "标志") ||
strings.Contains(prompt, "商标")
}
func buildLogoReferenceQuery(prompt string, lower string) string {
subject := logoReferenceSubject(prompt, lower)
terms := []string{
"logo design case study",
"brand identity",
"visual identity",
"design process",
"moodboard inspiration",
"Behance Dribbble",
"-maker",
"-generator",
"-free",
"-online",
"-template",
"-工具",
"-生成器",
"-制作器",
"-模板",
"-免费",
}
if hasCJK(prompt) {
terms = append([]string{
"logo 设计稿",
"品牌视觉",
"设计思路",
"案例",
"灵感",
}, terms...)
}
if subject != "" {
terms = append([]string{subject}, terms...)
}
return strings.Join(terms, " ")
}
func logoReferenceSubject(prompt string, lower string) string {
if strings.Contains(lower, "design agent") ||
strings.Contains(lower, "design-agent") ||
strings.Contains(prompt, "设计 Agent") ||
strings.Contains(prompt, "设计Agent") ||
strings.Contains(prompt, "设计智能体") ||
strings.Contains(prompt, "设计助手") {
return "AI design agent"
}
if strings.Contains(lower, "ai") && (strings.Contains(lower, "design") || strings.Contains(prompt, "设计")) {
return "AI design product"
}
subject := prompt
replacer := strings.NewReplacer(
"制作一个", " ",
"制作一款", " ",
"设计一个", " ",
"设计一款", " ",
"做一个", " ",
"做一款", " ",
"生成一个", " ",
"生成一款", " ",
"创建一个", " ",
"创建一款", " ",
"logo", " ",
"Logo", " ",
"LOGO", " ",
"品牌标识", " ",
"标志", " ",
"商标", " ",
"让设计变简单", " ",
"帮你搞定一切设计需求", " ",
"一切设计需求", " ",
"create a", " ",
"make a", " ",
"design a", " ",
"generate a", " ",
)
subject = replacer.Replace(subject)
subject = strings.Trim(subject, " ,,。.!!??;;::-—_")
fields := strings.Fields(subject)
if len(fields) > 8 {
fields = fields[:8]
}
return strings.Join(fields, " ")
}
func hasCJK(value string) bool {
for _, r := range value {
if r >= '\u4e00' && r <= '\u9fff' {
return true
}
}
return false
}
func cleanPromptForSearch(prompt string) string {
lines := strings.Split(prompt, "\n")
cleaned := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if strings.HasPrefix(line, "模型选择") || strings.HasPrefix(line, "Selected model:") {
continue
}
if strings.HasPrefix(line, "请先进行联网参考") || strings.HasPrefix(line, "Start with web references") {
continue
}
if strings.HasPrefix(line, "参考图") || strings.HasPrefix(line, "Reference image") {
continue
}
cleaned = append(cleaned, line)
}
value := strings.Join(cleaned, " ")
fields := strings.Fields(value)
if len(fields) > 40 {
fields = fields[:40]
}
return strings.Join(fields, " ")
}
var (
resultLinkRe = regexp.MustCompile(`(?is)]*class="[^"]*\bresult__a\b[^"]*"[^>]*href="([^"]+)"[^>]*>(.*?)`)
snippetRe = regexp.MustCompile(`(?is)]*class="[^"]*\bresult__snippet\b[^"]*"[^>]*>(.*?)`)
tagRe = regexp.MustCompile(`(?is)<[^>]+>`)
spaceRe = regexp.MustCompile(`\s+`)
)
func parseDuckDuckGoResults(body string, maxResults int) []design.ResearchResult {
matches := resultLinkRe.FindAllStringSubmatchIndex(body, -1)
results := make([]design.ResearchResult, 0, maxResults)
seen := map[string]bool{}
for index, match := range matches {
if len(results) >= maxResults {
break
}
rawURL := body[match[2]:match[3]]
titleHTML := body[match[4]:match[5]]
resolvedURL := resolveDuckDuckGoURL(rawURL)
if resolvedURL == "" {
continue
}
key := dedupeKey(resolvedURL)
if key == "" || seen[key] {
continue
}
nextStart := len(body)
if index+1 < len(matches) {
nextStart = matches[index+1][0]
}
segment := body[match[1]:nextStart]
snippet := ""
if snippetMatch := snippetRe.FindStringSubmatch(segment); len(snippetMatch) > 1 {
snippet = cleanHTML(snippetMatch[1])
}
title := cleanHTML(titleHTML)
if title == "" {
continue
}
seen[key] = true
results = append(results, design.ResearchResult{
Title: title,
URL: resolvedURL,
Snippet: snippet,
})
}
return results
}
func resolveDuckDuckGoURL(raw string) string {
raw = stdhtml.UnescapeString(strings.TrimSpace(raw))
if strings.HasPrefix(raw, "//") {
raw = "https:" + raw
}
parsed, err := url.Parse(raw)
if err != nil {
return ""
}
if strings.Contains(parsed.Host, "duckduckgo.com") {
if uddg := parsed.Query().Get("uddg"); uddg != "" {
raw = uddg
}
}
if !strings.HasPrefix(raw, "http://") && !strings.HasPrefix(raw, "https://") {
return ""
}
return raw
}
func cleanHTML(value string) string {
value = tagRe.ReplaceAllString(value, " ")
value = stdhtml.UnescapeString(value)
value = spaceRe.ReplaceAllString(value, " ")
return strings.TrimSpace(value)
}
func extractReadableHTMLText(body []byte, maxRunes int) string {
doc, err := xhtml.Parse(bytes.NewReader(body))
if err != nil {
return cleanPageContentText(cleanHTML(string(body)), maxRunes)
}
parts := make([]string, 0, 64)
runeCount := 0
collectReadableText(doc, false, &parts, &runeCount, maxRunes)
return cleanPageContentText(strings.Join(parts, " "), maxRunes)
}
func collectReadableText(node *xhtml.Node, skip bool, parts *[]string, runeCount *int, maxRunes int) bool {
if node == nil || *runeCount >= maxRunes {
return true
}
if node.Type == xhtml.ElementNode && skippedPageTextTag(strings.ToLower(node.Data)) {
skip = true
}
if !skip && node.Type == xhtml.TextNode {
text := cleanPageContentText(node.Data, maxRunes)
if usefulPageText(text) {
*parts = append(*parts, text)
*runeCount += len([]rune(text)) + 1
if *runeCount >= maxRunes {
return true
}
}
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
if collectReadableText(child, skip, parts, runeCount, maxRunes) {
return true
}
}
return false
}
func skippedPageTextTag(tag string) bool {
switch tag {
case "head", "script", "style", "noscript", "template", "svg", "canvas", "iframe", "form", "button", "nav", "header", "footer":
return true
default:
return false
}
}
func usefulPageText(text string) bool {
text = strings.TrimSpace(text)
if len([]rune(text)) < 2 {
return false
}
lower := strings.ToLower(text)
if len([]rune(text)) < 80 && (strings.Contains(lower, "cookie") || strings.Contains(lower, "privacy policy")) {
return false
}
return true
}
func cleanPageContentText(value string, maxRunes int) string {
value = stdhtml.UnescapeString(value)
value = spaceRe.ReplaceAllString(value, " ")
value = strings.TrimSpace(value)
return clampRunes(value, maxRunes)
}
func clampRunes(value string, maxRunes int) string {
if maxRunes <= 0 {
return value
}
runes := []rune(value)
if len(runes) <= maxRunes {
return value
}
return strings.TrimSpace(string(runes[:maxRunes]))
}
func dedupeKey(raw string) string {
parsed, err := url.Parse(raw)
if err != nil {
return ""
}
host := strings.TrimPrefix(strings.ToLower(parsed.Host), "www.")
return host + strings.TrimRight(parsed.Path, "/")
}