forked from forgejo/forgejo
[BUG] Handle bigger files in git grep
- The parser of `git grep`'s output uses `bufio.Scanner`, which is a good choice overall, however it does have a limit that's usually not noticed, it will not read more than `64 * 1024` bytes at once which can be hit in practical scenarios. - Use `bufio.Reader` instead which doesn't have this limitation, but is a bit harder to work with as it's a more lower level primitive. - Adds unit test. - Resolves https://codeberg.org/forgejo/forgejo/issues/3149
This commit is contained in:
parent
656554e247
commit
668709a33f
2 changed files with 43 additions and 4 deletions
|
@ -10,6 +10,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
@ -80,10 +81,21 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
|
|||
defer stdoutReader.Close()
|
||||
|
||||
isInBlock := false
|
||||
scanner := bufio.NewScanner(stdoutReader)
|
||||
scanner := bufio.NewReader(stdoutReader)
|
||||
var res *GrepResult
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
for {
|
||||
line, err := scanner.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Remove delimiter.
|
||||
if len(line) > 0 {
|
||||
line = line[:len(line)-1]
|
||||
}
|
||||
|
||||
if !isInBlock {
|
||||
if _ /* ref */, filename, ok := strings.Cut(line, ":"); ok {
|
||||
isInBlock = true
|
||||
|
@ -109,7 +121,7 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO
|
|||
res.LineCodes = append(res.LineCodes, lineCode)
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
// git grep exits by cancel (killed), usually it is caused by the limit of results
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue