1
0
Fork 0
forked from forgejo/forgejo

[BUG] Handle bigger files in git grep

- The parser of `git grep`'s output uses `bufio.Scanner`, which is a good
choice overall, however it does have a limit that's usually not noticed,
it will not read more than `64 * 1024` bytes at once which can be hit in
practical scenarios.
- Use `bufio.Reader` instead which doesn't have this limitation, but is
a bit harder to work with as it's a more lower level primitive.
- Adds unit test.
- Resolves https://codeberg.org/forgejo/forgejo/issues/3149
This commit is contained in:
Gusted 2024-04-11 13:34:53 +02:00
parent 656554e247
commit 668709a33f
No known key found for this signature in database
GPG key ID: FD821B732837125F
2 changed files with 43 additions and 4 deletions

View file

@ -10,6 +10,7 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
@ -80,10 +81,21 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
defer stdoutReader.Close()
isInBlock := false
scanner := bufio.NewScanner(stdoutReader)
scanner := bufio.NewReader(stdoutReader)
var res *GrepResult
for scanner.Scan() {
line := scanner.Text()
for {
line, err := scanner.ReadString('\n')
if err != nil {
if err == io.EOF {
return nil
}
return err
}
// Remove delimiter.
if len(line) > 0 {
line = line[:len(line)-1]
}
if !isInBlock {
if _ /* ref */, filename, ok := strings.Cut(line, ":"); ok {
isInBlock = true
@ -109,7 +121,7 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
res.LineCodes = append(res.LineCodes, lineCode)
}
}
return scanner.Err()
return nil
},
})
// git grep exits by cancel (killed), usually it is caused by the limit of results