domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Use hostmatcher to replace matchlist, improve security (#17605)

Browse source 
Use hostmacher to replace matchlist.

And we introduce a better DialContext to do a full host/IP check, otherwise the attackers can still bypass the allow/block list by a 302 redirection.
This commit is contained in:
wxiaoguang 2021-11-20 17:34:05 +08:00 • committed by GitHub
parent c96be0cd98
commit 013fb73068
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 377 additions and 293 deletions
Download patch file Download diff file Expand all files Collapse all files

138
modules/hostmatcher/hostmatcher.go
View file

@ -13,15 +13,18 @@ import (
)
// HostMatchList is used to check if a host or IP is in a list.
// If you only need to do wildcard matching, consider to use modules/matchlist
type HostMatchList struct {
hosts []string
SettingKeyHint string
SettingValue string
// builtins networks
builtins []string
// patterns for host names (with wildcard support)
patterns []string
// ipNets is the CIDR network list
ipNets []*net.IPNet
}
// MatchBuiltinAll all hosts are matched
const MatchBuiltinAll = "*"
// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched
const MatchBuiltinExternal = "external"
@ -31,9 +34,13 @@ const MatchBuiltinPrivate = "private"
// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
const MatchBuiltinLoopback = "loopback"
func isBuiltin(s string) bool {
return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback
}
// ParseHostMatchList parses the host list HostMatchList
func ParseHostMatchList(hostList string) *HostMatchList {
hl := &HostMatchList{}
func ParseHostMatchList(settingKeyHint string, hostList string) *HostMatchList {
hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList}
for _, s := range strings.Split(hostList, ",") {
s = strings.ToLower(strings.TrimSpace(s))
if s == "" {
@ -42,53 +49,106 @@ func ParseHostMatchList(hostList string) *HostMatchList {
_, ipNet, err := net.ParseCIDR(s)
if err == nil {
hl.ipNets = append(hl.ipNets, ipNet)
} else if isBuiltin(s) {
hl.builtins = append(hl.builtins, s)
} else {
hl.hosts = append(hl.hosts, s)
hl.patterns = append(hl.patterns, s)
}
}
return hl
}
// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list
func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool {
var matched bool
host = strings.ToLower(host)
ipStr := ip.String()
loop:
for _, hostInList := range hl.hosts {
switch hostInList {
case "":
// ParseSimpleMatchList parse a simple matchlist (no built-in networks, no CIDR support, only wildcard pattern match)
func ParseSimpleMatchList(settingKeyHint string, matchList string) *HostMatchList {
hl := &HostMatchList{
SettingKeyHint: settingKeyHint,
SettingValue: matchList,
}
for _, s := range strings.Split(matchList, ",") {
s = strings.ToLower(strings.TrimSpace(s))
if s == "" {
continue
case MatchBuiltinAll:
matched = true
break loop
}
// we keep the same result as old `matchlist`, so no builtin/CIDR support here, we only match wildcard patterns
hl.patterns = append(hl.patterns, s)
}
return hl
}
// AppendBuiltin appends more builtins to match
func (hl *HostMatchList) AppendBuiltin(builtin string) {
hl.builtins = append(hl.builtins, builtin)
}
// IsEmpty checks if the checklist is empty
func (hl *HostMatchList) IsEmpty() bool {
return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0)
}
func (hl *HostMatchList) checkPattern(host string) bool {
host = strings.ToLower(strings.TrimSpace(host))
for _, pattern := range hl.patterns {
if matched, _ := filepath.Match(pattern, host); matched {
return true
}
}
return false
}
func (hl *HostMatchList) checkIP(ip net.IP) bool {
for _, pattern := range hl.patterns {
if pattern == "*" {
return true
}
}
for _, builtin := range hl.builtins {
switch builtin {
case MatchBuiltinExternal:
if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched {
break loop
if ip.IsGlobalUnicast() && !util.IsIPPrivate(ip) {
return true
}
case MatchBuiltinPrivate:
if matched = util.IsIPPrivate(ip); matched {
break loop
if util.IsIPPrivate(ip) {
return true
}
case MatchBuiltinLoopback:
if matched = ip.IsLoopback(); matched {
break loop
}
default:
if matched, _ = filepath.Match(hostInList, host); matched {
break loop
}
if matched, _ = filepath.Match(hostInList, ipStr); matched {
break loop
if ip.IsLoopback() {
return true
}
}
}
if !matched {
for _, ipNet := range hl.ipNets {
if matched = ipNet.Contains(ip); matched {
break
}
for _, ipNet := range hl.ipNets {
if ipNet.Contains(ip) {
return true
}
}
return matched
return false
}
// MatchHostName checks if the host matches an allow/deny(block) list
func (hl *HostMatchList) MatchHostName(host string) bool {
if hl == nil {
return false
}
if hl.checkPattern(host) {
return true
}
if ip := net.ParseIP(host); ip != nil {
return hl.checkIP(ip)
}
return false
}
// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip`
func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool {
if hl == nil {
return false
}
host := ip.String() // nil-safe, we will get "<nil>" if ip is nil
return hl.checkPattern(host) || hl.checkIP(ip)
}
// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list
func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool {
return hl.MatchHostName(host) || hl.MatchIPAddr(ip)
}

79
modules/hostmatcher/hostmatcher_test.go
View file

@ -20,17 +20,28 @@ func TestHostOrIPMatchesList(t *testing.T) {
// for IPv6: "::1" is loopback, "fd00::/8" is private
hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24")
hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24")
test := func(cases []tc) {
for _, c := range cases {
assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected)
}
}
cases := []tc{
{"", net.IPv4zero, false},
{"", net.IPv6zero, false},
{"", net.ParseIP("127.0.0.1"), false},
{"127.0.0.1", nil, false},
{"", net.ParseIP("::1"), false},
{"", net.ParseIP("10.0.1.1"), true},
{"10.0.1.1", nil, true},
{"", net.ParseIP("192.168.1.1"), true},
{"192.168.1.1", nil, true},
{"", net.ParseIP("fd00::1"), true},
{"fd00::1", nil, true},
{"", net.ParseIP("8.8.8.8"), true},
{"", net.ParseIP("1001::1"), true},
@ -39,13 +50,13 @@ func TestHostOrIPMatchesList(t *testing.T) {
{"sub.mydomain.com", net.IPv4zero, true},
{"", net.ParseIP("169.254.1.1"), true},
{"169.254.1.1", nil, true},
{"", net.ParseIP("169.254.2.2"), false},
{"169.254.2.2", nil, false},
}
for _, c := range cases {
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
}
test(cases)
hl = ParseHostMatchList("loopback")
hl = ParseHostMatchList("", "loopback")
cases = []tc{
{"", net.IPv4zero, false},
{"", net.ParseIP("127.0.0.1"), true},
@ -59,11 +70,9 @@ func TestHostOrIPMatchesList(t *testing.T) {
{"mydomain.com", net.IPv4zero, false},
}
for _, c := range cases {
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
}
test(cases)
hl = ParseHostMatchList("private")
hl = ParseHostMatchList("", "private")
cases = []tc{
{"", net.IPv4zero, false},
{"", net.ParseIP("127.0.0.1"), false},
@ -77,11 +86,9 @@ func TestHostOrIPMatchesList(t *testing.T) {
{"mydomain.com", net.IPv4zero, false},
}
for _, c := range cases {
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
}
test(cases)
hl = ParseHostMatchList("external")
hl = ParseHostMatchList("", "external")
cases = []tc{
{"", net.IPv4zero, false},
{"", net.ParseIP("127.0.0.1"), false},
@ -95,11 +102,9 @@ func TestHostOrIPMatchesList(t *testing.T) {
{"mydomain.com", net.IPv4zero, false},
}
for _, c := range cases {
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
}
test(cases)
hl = ParseHostMatchList("*")
hl = ParseHostMatchList("", "*")
cases = []tc{
{"", net.IPv4zero, true},
{"", net.ParseIP("127.0.0.1"), true},
@ -113,7 +118,43 @@ func TestHostOrIPMatchesList(t *testing.T) {
{"mydomain.com", net.IPv4zero, true},
}
for _, c := range cases {
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
test(cases)
// built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name
// this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users
// a real user should never use loopback/private/external as their host names
hl = ParseHostMatchList("", "loopback, [p]rivate")
cases = []tc{
{"loopback", nil, false},
{"", net.ParseIP("127.0.0.1"), true},
{"private", nil, true},
{"", net.ParseIP("192.168.1.1"), false},
}
test(cases)
hl = ParseSimpleMatchList("", "loopback, *.domain.com")
cases = []tc{
{"loopback", nil, true},
{"", net.ParseIP("127.0.0.1"), false},
{"sub.domain.com", nil, true},
{"other.com", nil, false},
{"", net.ParseIP("1.1.1.1"), false},
}
test(cases)
hl = ParseSimpleMatchList("", "external")
cases = []tc{
{"", net.ParseIP("192.168.1.1"), false},
{"", net.ParseIP("1.1.1.1"), false},
{"external", nil, true},
}
test(cases)
hl = ParseSimpleMatchList("", "")
cases = []tc{
{"", net.ParseIP("192.168.1.1"), false},
{"", net.ParseIP("1.1.1.1"), false},
{"external", nil, false},
}
test(cases)
}

58
modules/hostmatcher/http.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package hostmatcher
import (
"context"
"fmt"
"net"
"syscall"
"time"
)
// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check
func NewDialContext(usage string, allowList *HostMatchList, blockList *HostMatchList) func(ctx context.Context, network, addr string) (net.Conn, error) {
// How Go HTTP Client works with redirection:
// transport.RoundTrip URL=http://domain.com, Host=domain.com
// transport.DialContext addrOrHost=domain.com:80
// dialer.Control tcp4:11.22.33.44:80
// transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field)
// transport.DialContext addrOrHost=domain.com:80
// dialer.Control tcp4:11.22.33.44:80
return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) {
dialer := net.Dialer{
// default values comes from http.DefaultTransport
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
Control: func(network, ipAddr string, c syscall.RawConn) (err error) {
var host string
if host, _, err = net.SplitHostPort(addrOrHost); err != nil {
return err
}
// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here
tcpAddr, err := net.ResolveTCPAddr(network, ipAddr)
if err != nil {
return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", usage, host, network, ipAddr, err)
}
var blockedError error
if blockList.MatchHostOrIP(host, tcpAddr.IP) {
blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr)
}
// if we have an allow-list, check the allow-list first
if !allowList.IsEmpty() {
if !allowList.MatchHostOrIP(host, tcpAddr.IP) {
return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr)
}
}
// otherwise, we always follow the blocked list
return blockedError
},
}
return dialer.DialContext(ctx, network, addrOrHost)
}
}

5
modules/lfs/client.go
View file

@ -7,6 +7,7 @@ package lfs
import (
"context"
"io"
"net/http"
"net/url"
)
@ -24,9 +25,9 @@ type Client interface {
}
// NewClient creates a LFS client
func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
func NewClient(endpoint *url.URL, httpTransport *http.Transport) Client {
if endpoint.Scheme == "file" {
return newFilesystemClient(endpoint)
}
return newHTTPClient(endpoint, skipTLSVerify)
return newHTTPClient(endpoint, httpTransport)
}

4
modules/lfs/client_test.go
View file

@ -13,10 +13,10 @@ import (
func TestNewClient(t *testing.T) {
u, _ := url.Parse("file:///test")
c := NewClient(u, true)
c := NewClient(u, nil)
assert.IsType(t, &FilesystemClient{}, c)
u, _ = url.Parse("https://test.com/lfs")
c = NewClient(u, true)
c = NewClient(u, nil)
assert.IsType(t, &HTTPClient{}, c)
}

14
modules/lfs/http_client.go
View file

@ -7,7 +7,6 @@ package lfs
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"net/http"
@ -34,12 +33,15 @@ func (c *HTTPClient) BatchSize() int {
return batchSize
}
func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient {
if httpTransport == nil {
httpTransport = &http.Transport{
Proxy: proxy.Proxy(),
}
}
hc := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
Proxy: proxy.Proxy(),
},
Transport: httpTransport,
}
client := &HTTPClient{

46
modules/matchlist/matchlist.go
View file

@ -1,46 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package matchlist
import (
"strings"
"github.com/gobwas/glob"
)
// Matchlist represents a block or allow list
type Matchlist struct {
ruleGlobs []glob.Glob
}
// NewMatchlist creates a new block or allow list
func NewMatchlist(rules ...string) (*Matchlist, error) {
for i := range rules {
rules[i] = strings.ToLower(rules[i])
}
list := Matchlist{
ruleGlobs: make([]glob.Glob, 0, len(rules)),
}
for _, rule := range rules {
rg, err := glob.Compile(rule)
if err != nil {
return nil, err
}
list.ruleGlobs = append(list.ruleGlobs, rg)
}
return &list, nil
}
// Match will matches
func (b *Matchlist) Match(u string) bool {
for _, r := range b.ruleGlobs {
if r.Match(u) {
return true
}
}
return false
}

19
modules/repository/repo.go
View file

@ -8,7 +8,7 @@ import (
"context"
"fmt"
"io"
"net/url"
"net/http"
"path"
"strings"
"time"
@ -46,7 +46,10 @@ func WikiRemoteURL(remote string) string {
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) {
func MigrateRepositoryGitData(ctx context.Context, u *models.User,
repo *models.Repository, opts migration.MigrateOptions,
httpTransport *http.Transport,
) (*models.Repository, error) {
repoPath := models.RepoPath(u.Name, opts.RepoName)
if u.IsOrganization() {
@ -141,8 +144,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
}
if opts.LFS {
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
lfsClient := lfs.NewClient(endpoint, httpTransport)
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil {
log.Error("Failed to store missing LFS objects for repository: %v", err)
}
}
@ -336,8 +340,7 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
}
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
client := lfs.NewClient(endpoint, skipTLSVerify)
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error {
contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob)
@ -345,7 +348,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
downloadObjects := func(pointers []lfs.Pointer) error {
err := client.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error {
err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error {
if objectError != nil {
return objectError
}
@ -411,7 +414,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi
}
batch = append(batch, pointerBlob.Pointer)
if len(batch) >= client.BatchSize() {
if len(batch) >= lfsClient.BatchSize() {
if err := downloadObjects(batch); err != nil {
return err
}

19
modules/setting/migrations.go
View file

@ -4,17 +4,13 @@
package setting
import (
"strings"
)
var (
// Migrations settings
Migrations = struct {
MaxAttempts int
RetryBackoff int
AllowedDomains []string
BlockedDomains []string
AllowedDomains string
BlockedDomains string
AllowLocalNetworks bool
SkipTLSVerify bool
}{
@ -28,15 +24,8 @@ func newMigrationsService() {
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",")
for i := range Migrations.AllowedDomains {
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i])
}
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",")
for i := range Migrations.BlockedDomains {
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i])
}
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").MustString("")
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("")
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
}

5
modules/setting/webhook.go
View file

@ -7,7 +7,6 @@ package setting
import (
"net/url"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/log"
)
@ -17,7 +16,7 @@ var (
QueueLength int
DeliverTimeout int
SkipTLSVerify bool
AllowedHostList *hostmatcher.HostMatchList
AllowedHostList string
Types []string
PagingNum int
ProxyURL string
@ -38,7 +37,7 @@ func newWebhookService() {
Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinExternal))
Webhook.AllowedHostList = sec.Key("ALLOWED_HOST_LIST").MustString("")
Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"}
Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10)
Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("")