geo/server/internal/tenant/app/monitoring_text_encoding.go

package app

import (
	"strings"
	"unicode"
	"unicode/utf8"
)

var monitoringWindows1252ReverseMap = map[rune]byte{
	0x20AC: 0x80,
	0x201A: 0x82,
	0x0192: 0x83,
	0x201E: 0x84,
	0x2026: 0x85,
	0x2020: 0x86,
	0x2021: 0x87,
	0x02C6: 0x88,
	0x2030: 0x89,
	0x0160: 0x8A,
	0x2039: 0x8B,
	0x0152: 0x8C,
	0x017D: 0x8E,
	0x2018: 0x91,
	0x2019: 0x92,
	0x201C: 0x93,
	0x201D: 0x94,
	0x2022: 0x95,
	0x2013: 0x96,
	0x2014: 0x97,
	0x02DC: 0x98,
	0x2122: 0x99,
	0x0161: 0x9A,
	0x203A: 0x9B,
	0x0153: 0x9C,
	0x017E: 0x9E,
	0x0178: 0x9F,
}

func repairMonitoringMojibake(value string) string {
	trimmed := strings.TrimSpace(value)
	if trimmed == "" || !looksLikeMonitoringMojibake(trimmed) {
		return value
	}

	repaired, ok := decodeMonitoringLatin1AsUTF8(trimmed)
	if !ok || strings.ContainsRune(repaired, utf8.RuneError) || !monitoringRepairLooksBetter(trimmed, repaired) {
		return value
	}

	return repaired
}

func normalizeMonitoringOptionalString(value *string) *string {
	if value == nil {
		return nil
	}
	repaired := repairMonitoringMojibake(*value)
	return &repaired
}

func looksLikeMonitoringMojibake(value string) bool {
	if value == "" {
		return false
	}

	if strings.Contains(value, "Ã") || strings.Contains(value, "â€") || strings.Contains(value, "ï¼") || strings.Contains(value, "ã€") {
		return true
	}

	hanCount := monitoringHanRuneCount(value)
	suspiciousCount := monitoringSuspiciousRuneCount(value)
	if hanCount > 0 {
		return suspiciousCount >= 8 && suspiciousCount > hanCount*3
	}
	return suspiciousCount >= 4
}

func decodeMonitoringLatin1AsUTF8(value string) (string, bool) {
	buf := make([]byte, 0, len(value))
	for _, r := range value {
		if mapped, ok := monitoringWindows1252ReverseMap[r]; ok {
			buf = append(buf, mapped)
			continue
		}
		if r > 255 {
			return "", false
		}
		buf = append(buf, byte(r))
	}
	if !utf8.Valid(buf) {
		return "", false
	}
	return string(buf), true
}

func monitoringRepairLooksBetter(original, repaired string) bool {
	if strings.TrimSpace(repaired) == "" {
		return false
	}

	originalHan := monitoringHanRuneCount(original)
	repairedHan := monitoringHanRuneCount(repaired)
	originalSuspicious := monitoringSuspiciousRuneCount(original)
	repairedSuspicious := monitoringSuspiciousRuneCount(repaired)

	if repairedHan > originalHan && repairedHan >= 2 {
		return true
	}

	return repairedHan > 0 && repairedSuspicious*2 < originalSuspicious
}

func monitoringHanRuneCount(value string) int {
	count := 0
	for _, r := range value {
		if unicode.Is(unicode.Han, r) {
			count++
		}
	}
	return count
}

func monitoringSuspiciousRuneCount(value string) int {
	count := 0
	for _, r := range value {
		if (r >= 0x00C0 && r <= 0x017F) || (r >= 0x2000 && r <= 0x20FF) {
			count++
		}
	}
	return count
}

func normalizeMonitoringSourceItemValue(item MonitoringSourceItem) MonitoringSourceItem {
	item.Title = normalizeMonitoringOptionalString(item.Title)
	item.SiteName = normalizeMonitoringOptionalString(item.SiteName)
	item.ResolutionStatus = normalizeMonitoringOptionalString(item.ResolutionStatus)
	item.ResolutionConfidence = normalizeMonitoringOptionalString(item.ResolutionConfidence)
	return item
}

func normalizeMonitoringTaskResultRequestValue(req MonitoringTaskResultRequest) MonitoringTaskResultRequest {
	if req.Answer != nil {
		repaired := repairMonitoringMojibake(*req.Answer)
		req.Answer = &repaired
	}
	if req.ErrorMessage != nil {
		repaired := repairMonitoringMojibake(*req.ErrorMessage)
		req.ErrorMessage = &repaired
	}

	if len(req.Citations) > 0 {
		next := make([]MonitoringSourceItem, 0, len(req.Citations))
		for _, item := range req.Citations {
			next = append(next, normalizeMonitoringSourceItemValue(item))
		}
		req.Citations = next
	}
	if len(req.SearchResults) > 0 {
		next := make([]MonitoringSourceItem, 0, len(req.SearchResults))
		for _, item := range req.SearchResults {
			next = append(next, normalizeMonitoringSourceItemValue(item))
		}
		req.SearchResults = next
	}

	if len(req.RawResponseJSON) > 0 {
		next := make(map[string]interface{}, len(req.RawResponseJSON))
		for key, value := range req.RawResponseJSON {
			next[key] = normalizeMonitoringJSONValue(value)
		}
		req.RawResponseJSON = next
	}

	return req
}

func normalizeMonitoringJSONValue(value interface{}) interface{} {
	switch typed := value.(type) {
	case string:
		return repairMonitoringMojibake(typed)
	case []interface{}:
		next := make([]interface{}, len(typed))
		for index, item := range typed {
			next[index] = normalizeMonitoringJSONValue(item)
		}
		return next
	case map[string]interface{}:
		next := make(map[string]interface{}, len(typed))
		for key, item := range typed {
			next[key] = normalizeMonitoringJSONValue(item)
		}
		return next
	default:
		return value
	}
}