package app import ( "strings" "unicode" "unicode/utf8" ) var monitoringWindows1252ReverseMap = map[rune]byte{ 0x20AC: 0x80, 0x201A: 0x82, 0x0192: 0x83, 0x201E: 0x84, 0x2026: 0x85, 0x2020: 0x86, 0x2021: 0x87, 0x02C6: 0x88, 0x2030: 0x89, 0x0160: 0x8A, 0x2039: 0x8B, 0x0152: 0x8C, 0x017D: 0x8E, 0x2018: 0x91, 0x2019: 0x92, 0x201C: 0x93, 0x201D: 0x94, 0x2022: 0x95, 0x2013: 0x96, 0x2014: 0x97, 0x02DC: 0x98, 0x2122: 0x99, 0x0161: 0x9A, 0x203A: 0x9B, 0x0153: 0x9C, 0x017E: 0x9E, 0x0178: 0x9F, } func repairMonitoringMojibake(value string) string { trimmed := strings.TrimSpace(value) if trimmed == "" || !looksLikeMonitoringMojibake(trimmed) { return value } repaired, ok := decodeMonitoringLatin1AsUTF8(trimmed) if !ok || strings.ContainsRune(repaired, utf8.RuneError) || !monitoringRepairLooksBetter(trimmed, repaired) { return value } return repaired } func normalizeMonitoringOptionalString(value *string) *string { if value == nil { return nil } repaired := repairMonitoringMojibake(*value) return &repaired } func looksLikeMonitoringMojibake(value string) bool { if value == "" { return false } if strings.Contains(value, "Ã") || strings.Contains(value, "â€") || strings.Contains(value, "ï¼") || strings.Contains(value, "ã€") { return true } hanCount := monitoringHanRuneCount(value) suspiciousCount := monitoringSuspiciousRuneCount(value) if hanCount > 0 { return suspiciousCount >= 8 && suspiciousCount > hanCount*3 } return suspiciousCount >= 4 } func decodeMonitoringLatin1AsUTF8(value string) (string, bool) { buf := make([]byte, 0, len(value)) for _, r := range value { if mapped, ok := monitoringWindows1252ReverseMap[r]; ok { buf = append(buf, mapped) continue } if r > 255 { return "", false } buf = append(buf, byte(r)) } if !utf8.Valid(buf) { return "", false } return string(buf), true } func monitoringRepairLooksBetter(original, repaired string) bool { if strings.TrimSpace(repaired) == "" { return false } originalHan := monitoringHanRuneCount(original) repairedHan := monitoringHanRuneCount(repaired) originalSuspicious := monitoringSuspiciousRuneCount(original) repairedSuspicious := monitoringSuspiciousRuneCount(repaired) if repairedHan > originalHan && repairedHan >= 2 { return true } return repairedHan > 0 && repairedSuspicious*2 < originalSuspicious } func monitoringHanRuneCount(value string) int { count := 0 for _, r := range value { if unicode.Is(unicode.Han, r) { count++ } } return count } func monitoringSuspiciousRuneCount(value string) int { count := 0 for _, r := range value { if (r >= 0x00C0 && r <= 0x017F) || (r >= 0x2000 && r <= 0x20FF) { count++ } } return count } func normalizeMonitoringSourceItemValue(item MonitoringSourceItem) MonitoringSourceItem { item.Title = normalizeMonitoringOptionalString(item.Title) item.SiteName = normalizeMonitoringOptionalString(item.SiteName) item.ResolutionStatus = normalizeMonitoringOptionalString(item.ResolutionStatus) item.ResolutionConfidence = normalizeMonitoringOptionalString(item.ResolutionConfidence) return item } func normalizeMonitoringTaskResultRequestValue(req MonitoringTaskResultRequest) MonitoringTaskResultRequest { if req.Answer != nil { repaired := repairMonitoringMojibake(*req.Answer) req.Answer = &repaired } if req.ErrorMessage != nil { repaired := repairMonitoringMojibake(*req.ErrorMessage) req.ErrorMessage = &repaired } if len(req.Citations) > 0 { next := make([]MonitoringSourceItem, 0, len(req.Citations)) for _, item := range req.Citations { next = append(next, normalizeMonitoringSourceItemValue(item)) } req.Citations = next } if len(req.SearchResults) > 0 { next := make([]MonitoringSourceItem, 0, len(req.SearchResults)) for _, item := range req.SearchResults { next = append(next, normalizeMonitoringSourceItemValue(item)) } req.SearchResults = next } if len(req.RawResponseJSON) > 0 { next := make(map[string]interface{}, len(req.RawResponseJSON)) for key, value := range req.RawResponseJSON { next[key] = normalizeMonitoringJSONValue(value) } req.RawResponseJSON = next } return req } func normalizeMonitoringJSONValue(value interface{}) interface{} { switch typed := value.(type) { case string: return repairMonitoringMojibake(typed) case []interface{}: next := make([]interface{}, len(typed)) for index, item := range typed { next[index] = normalizeMonitoringJSONValue(item) } return next case map[string]interface{}: next := make(map[string]interface{}, len(typed)) for key, item := range typed { next[key] = normalizeMonitoringJSONValue(item) } return next default: return value } }