fix(browser-fetch): allow baidu source fallback
This commit is contained in:
@@ -98,7 +98,7 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
|
||||
zap.Error(fallbackErr),
|
||||
)
|
||||
}
|
||||
return nil, classifySourceFetchError(rawURL, err)
|
||||
return nil, classifySourceFetchError(rawURL, fmt.Errorf("%w; browser fetch fallback failed: %v", err, fallbackErr))
|
||||
}
|
||||
|
||||
text := normalizeKnowledgeText(item.Content)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBrowserFetchDomainAllowedMatchesSubdomain(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if !browserFetchDomainAllowed("https://mbd.baidu.com/bh/m/detail/ar_10234112381636647949", []string{"baidu.com"}) {
|
||||
t.Fatalf("expected baidu.com to allow mbd.baidu.com")
|
||||
}
|
||||
if browserFetchDomainAllowed("https://notbaidu.com/article", []string{"baidu.com"}) {
|
||||
t.Fatalf("expected baidu.com not to allow notbaidu.com")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifiedSourceFetchErrorIncludesBrowserFallbackFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := classifySourceFetchError("https://mbd.baidu.com/article", fmt.Errorf(
|
||||
"webpage parser failed: status=%d body=invalid link; browser fetch fallback failed: domain is not allowed",
|
||||
http.StatusInternalServerError,
|
||||
))
|
||||
|
||||
message := err.Error()
|
||||
for _, expected := range []string{
|
||||
"source website fetch restricted status=500",
|
||||
"webpage parser failed",
|
||||
"browser fetch fallback failed: domain is not allowed",
|
||||
} {
|
||||
if !strings.Contains(message, expected) {
|
||||
t.Fatalf("classified error = %q, want %q", message, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user