fix: resolve UTF-8 character encoding issues in log search and highlighting (#3557)

* fix: find all match str index

* fix: resolve UTF-8 character encoding issue in log search highlighting

* feat: optimize  search logic

* feat: add test

* fix: filter code logic clean

* remove: garbge test code

* golangci: remove unused colorizeByte func

* feat: add test

* fix: golangci

---------

Co-authored-by: tianbaosha <tianbaosha@tencent.com>

internal/color/colorize.go

@@ -43,19 +43,42 @@ func ANSIColorize(text string, color int) string {
 
 // Highlight colorize bytes at given indices.
 func Highlight(bb []byte, ii []int, c int) []byte {
-	b := make([]byte, 0, len(bb))
+	if len(ii) == 0 {
-	for i, j := 0, 0; i < len(bb); i++ {
+		return bb
-		if j < len(ii) && ii[j] == i {
+	}
-			b = append(b, colorizeByte(bb[i], c)...)
+
-			j++
+	result := make([]byte, 0, len(bb)+len(ii)*20) // Extra space for color codes
+
+	// Create a map of byte positions that should be highlighted
+	highlightMap := make(map[int]bool)
+	for _, pos := range ii {
+		highlightMap[pos] = true
+	}
+
+	// Process each byte
+	for i := 0; i < len(bb); i++ {
+		if highlightMap[i] {
+			// Check if this is the start of a UTF-8 character
+			if (bb[i] & 0xC0) != 0x80 {
+				// This is the start of a character, find the end
+				charStart := i
+				charEnd := i + 1
+				for charEnd < len(bb) && (bb[charEnd]&0xC0) == 0x80 {
+					charEnd++
+				}
+				// Colorize the entire character
+				char := string(bb[charStart:charEnd])
+				colored := ANSIColorize(char, c)
+				result = append(result, []byte(colored)...)
+				i = charEnd - 1 // Skip the rest of the character bytes
+			} else {
+				// This is a continuation byte, skip it (already handled)
+				continue
+			}
 		} else {
-			b = append(b, bb[i])
+			result = append(result, bb[i])
 		}
 	}
 
-	return b
+	return result
-}
-
-func colorizeByte(b byte, color int) []byte {
-	return []byte(ANSIColorize(string(b), color))
 }

internal/dao/log_items.go

@@ -207,7 +207,7 @@ func (l *LogItems) filterLogs(index int, q string, showTime bool) (matches []int
 	ll := make([][]byte, len(l.items[index:]))
 	l.Lines(index, showTime, ll)
 	for i, line := range ll {
-		locs := rx.FindIndex(line)
+		locs := rx.FindAllIndex(line, -1)
 		if locs != nil && invert {
 			continue
 		}
@@ -216,8 +216,8 @@ func (l *LogItems) filterLogs(index int, q string, showTime bool) (matches []int
 		}
 		matches = append(matches, i)
 		ii := make([]int, 0, 10)
-		for i := 0; i < len(locs); i += 2 {
+		for _, loc := range locs {
-			for j := locs[i]; j < locs[i+1]; j++ {
+			for j := loc[0]; j < loc[1]; j++ {
 				ii = append(ii, j)
 			}
 		}

internal/dao/log_items_test.go

@@ -19,10 +19,11 @@ func init() {
 
 func TestLogItemsFilter(t *testing.T) {
 	uu := map[string]struct {
-		q    string
+		q       string
-		opts dao.LogOptions
+		opts    dao.LogOptions
-		e    []int
+		e       []int
-		err  error
+		indices [][]int
+		err     error
 	}{
 		"empty": {
 			opts: dao.LogOptions{},
@@ -41,7 +42,8 @@ func TestLogItemsFilter(t *testing.T) {
 				Path:      "fred/blee",
 				Container: "c1",
 			},
-			e: []int{0, 1, 2},
+			e:       []int{0, 1, 2},
+			indices: [][]int{{26, 27}, {26, 27}, {26, 27}}, // matches container name "c1" at positions 26-27 in rendered format each line
 		},
 		"message": {
 			q: "zorg",
@@ -59,6 +61,15 @@ func TestLogItemsFilter(t *testing.T) {
 			},
 			e: []int{2},
 		},
+		"multi-origin-text-match": {
+			q: "will",
+			opts: dao.LogOptions{
+				Path:      "fred/blee",
+				Container: "c1",
+			},
+			e:       []int{1, 2},
+			indices: [][]int{{45, 46, 47, 48, 59, 60, 61, 62}, {64, 65, 66, 67, 70, 71, 72, 73, 76, 77, 78, 79}},
+		},
 	}
 
 	for k := range uu {
@@ -66,18 +77,21 @@ func TestLogItemsFilter(t *testing.T) {
 		ii := dao.NewLogItems()
 		ii.Add(
 			dao.NewLogItem([]byte(fmt.Sprintf("%s %s\n", "2018-12-14T10:36:43.326972-07:00", "Testing 1,2,3..."))),
-			dao.NewLogItemFromString("Bumble bee tuna"),
+			dao.NewLogItemFromString("Bumble bee tuna. will be back. will win."),
-			dao.NewLogItemFromString("Jean Batiste Emmanuel Zorg"),
+			dao.NewLogItemFromString("Jean Batiste Emmanuel Zorg. wili, will. will, will"),
 		)
 		t.Run(k, func(t *testing.T) {
 			_, n := client.Namespaced(u.opts.Path)
 			for _, i := range ii.Items() {
 				i.Pod, i.Container = n, u.opts.Container
 			}
-			res, _, err := ii.Filter(0, u.q, false)
+			res, indices, err := ii.Filter(0, u.q, false)
 			assert.Equal(t, u.err, err)
 			if err == nil {
 				assert.Equal(t, u.e, res)
+				if u.indices != nil {
+					assert.Equal(t, u.indices, indices)
+				}
 			}
 		})
 	}