forked from forgejo/forgejo
Add push to remote mirror repository (#15157)
* Added push mirror model. * Integrated push mirror into queue. * Moved methods into own file. * Added basic implementation. * Mirror wiki too. * Removed duplicated method. * Get url for different remotes. * Added migration. * Unified remote url access. * Add/Remove push mirror remotes. * Prevent hangs with missing credentials. * Moved code between files. * Changed sanitizer interface. * Added push mirror backend methods. * Only update the mirror remote. * Limit refs on push. * Added UI part. * Added missing table. * Delete mirror if repository gets removed. * Changed signature. Handle object errors. * Added upload method. * Added "upload" unit tests. * Added transfer adapter unit tests. * Send correct headers. * Added pushing of LFS objects. * Added more logging. * Simpler body handling. * Process files in batches to reduce HTTP calls. * Added created timestamp. * Fixed invalid column name. * Changed name to prevent xorm auto setting. * Remove table header im empty. * Strip exit code from error message. * Added docs page about mirroring. * Fixed date. * Fixed merge errors. * Moved test to integrations. * Added push mirror test. * Added test.
This commit is contained in:
KN4CK3R
GitHub
parent
5d113bdd19
commit
440039c0cc
39 changed files with 2468 additions and 885 deletions
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2017 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
|
|
@ -9,40 +9,53 @@ import (
|
|||
"strings"
|
||||
)
|
||||
|
||||
// urlSafeError wraps an error whose message may contain a sensitive URL
|
||||
type urlSafeError struct {
|
||||
err error
|
||||
unsanitizedURL string
|
||||
const userPlaceholder = "sanitized-credential"
|
||||
const unparsableURL = "(unparsable url)"
|
||||
|
||||
type sanitizedError struct {
|
||||
err error
|
||||
replacer *strings.Replacer
|
||||
}
|
||||
|
||||
func (err urlSafeError) Error() string {
|
||||
return SanitizeMessage(err.err.Error(), err.unsanitizedURL)
|
||||
func (err sanitizedError) Error() string {
|
||||
return err.replacer.Replace(err.err.Error())
|
||||
}
|
||||
|
||||
// URLSanitizedError returns the sanitized version an error whose message may
|
||||
// contain a sensitive URL
|
||||
func URLSanitizedError(err error, unsanitizedURL string) error {
|
||||
return urlSafeError{err: err, unsanitizedURL: unsanitizedURL}
|
||||
// NewSanitizedError wraps an error and replaces all old, new string pairs in the message text.
|
||||
func NewSanitizedError(err error, oldnew ...string) error {
|
||||
return sanitizedError{err: err, replacer: strings.NewReplacer(oldnew...)}
|
||||
}
|
||||
|
||||
// SanitizeMessage sanitizes a message which may contains a sensitive URL
|
||||
func SanitizeMessage(message, unsanitizedURL string) string {
|
||||
sanitizedURL := SanitizeURLCredentials(unsanitizedURL, true)
|
||||
return strings.ReplaceAll(message, unsanitizedURL, sanitizedURL)
|
||||
// NewURLSanitizedError wraps an error and replaces the url credential or removes them.
|
||||
func NewURLSanitizedError(err error, u *url.URL, usePlaceholder bool) error {
|
||||
return sanitizedError{err: err, replacer: NewURLSanitizer(u, usePlaceholder)}
|
||||
}
|
||||
|
||||
// SanitizeURLCredentials sanitizes a url, either removing user credentials
|
||||
// or replacing them with a placeholder.
|
||||
func SanitizeURLCredentials(unsanitizedURL string, usePlaceholder bool) string {
|
||||
u, err := url.Parse(unsanitizedURL)
|
||||
if err != nil {
|
||||
// don't log the error, since it might contain unsanitized URL.
|
||||
return "(unparsable url)"
|
||||
}
|
||||
// NewStringURLSanitizedError wraps an error and replaces the url credential or removes them.
|
||||
// If the url can't get parsed it gets replaced with a placeholder string.
|
||||
func NewStringURLSanitizedError(err error, unsanitizedURL string, usePlaceholder bool) error {
|
||||
return sanitizedError{err: err, replacer: NewStringURLSanitizer(unsanitizedURL, usePlaceholder)}
|
||||
}
|
||||
|
||||
// NewURLSanitizer creates a replacer for the url with the credential sanitized or removed.
|
||||
func NewURLSanitizer(u *url.URL, usePlaceholder bool) *strings.Replacer {
|
||||
old := u.String()
|
||||
|
||||
if u.User != nil && usePlaceholder {
|
||||
u.User = url.User("<credentials>")
|
||||
u.User = url.User(userPlaceholder)
|
||||
} else {
|
||||
u.User = nil
|
||||
}
|
||||
return u.String()
|
||||
return strings.NewReplacer(old, u.String())
|
||||
}
|
||||
|
||||
// NewStringURLSanitizer creates a replacer for the url with the credential sanitized or removed.
|
||||
// If the url can't get parsed it gets replaced with a placeholder string
|
||||
func NewStringURLSanitizer(unsanitizedURL string, usePlaceholder bool) *strings.Replacer {
|
||||
u, err := url.Parse(unsanitizedURL)
|
||||
if err != nil {
|
||||
// don't log the error, since it might contain unsanitized URL.
|
||||
return strings.NewReplacer(unsanitizedURL, unparsableURL)
|
||||
}
|
||||
return NewURLSanitizer(u, usePlaceholder)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,164 @@
|
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSanitizeURLCredentials(t *testing.T) {
|
||||
var kases = map[string]string{
|
||||
"https://github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git",
|
||||
"https://mytoken@github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git",
|
||||
"http://github.com/go-gitea/test_repo.git": "http://github.com/go-gitea/test_repo.git",
|
||||
"/test/repos/repo1": "/test/repos/repo1",
|
||||
"git@github.com:go-gitea/test_repo.git": "(unparsable url)",
|
||||
func TestNewSanitizedError(t *testing.T) {
|
||||
err := errors.New("error while secret on test")
|
||||
err2 := NewSanitizedError(err)
|
||||
assert.Equal(t, err.Error(), err2.Error())
|
||||
|
||||
var cases = []struct {
|
||||
input error
|
||||
oldnew []string
|
||||
expected string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
errors.New("error while secret on test"),
|
||||
[]string{"secret", "replaced"},
|
||||
"error while replaced on test",
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
errors.New("error while sec-ret on test"),
|
||||
[]string{"secret", "replaced"},
|
||||
"error while sec-ret on test",
|
||||
},
|
||||
}
|
||||
|
||||
for source, value := range kases {
|
||||
assert.EqualValues(t, value, SanitizeURLCredentials(source, false))
|
||||
for n, c := range cases {
|
||||
err := NewSanitizedError(c.input, c.oldnew...)
|
||||
|
||||
assert.Equal(t, c.expected, err.Error(), "case %d: error should match", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewStringURLSanitizer(t *testing.T) {
|
||||
var cases = []struct {
|
||||
input string
|
||||
placeholder bool
|
||||
expected string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
true,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
"https://mytoken@github.com/go-gitea/test_repo.git",
|
||||
true,
|
||||
"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
"https://mytoken@github.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
"https://user:password@github.com/go-gitea/test_repo.git",
|
||||
true,
|
||||
"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
"https://user:password@github.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
"https://gi\nthub.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
unparsableURL,
|
||||
},
|
||||
}
|
||||
|
||||
for n, c := range cases {
|
||||
// uses NewURLSanitizer internally
|
||||
result := NewStringURLSanitizer(c.input, c.placeholder).Replace(c.input)
|
||||
|
||||
assert.Equal(t, c.expected, result, "case %d: error should match", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewStringURLSanitizedError(t *testing.T) {
|
||||
var cases = []struct {
|
||||
input string
|
||||
placeholder bool
|
||||
expected string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
true,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
"https://mytoken@github.com/go-gitea/test_repo.git",
|
||||
true,
|
||||
"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
"https://mytoken@github.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
"https://user:password@github.com/go-gitea/test_repo.git",
|
||||
true,
|
||||
"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
"https://user:password@github.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
"https://github.com/go-gitea/test_repo.git",
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
"https://gi\nthub.com/go-gitea/test_repo.git",
|
||||
false,
|
||||
unparsableURL,
|
||||
},
|
||||
}
|
||||
|
||||
encloseText := func(input string) string {
|
||||
return "test " + input + " test"
|
||||
}
|
||||
|
||||
for n, c := range cases {
|
||||
err := errors.New(encloseText(c.input))
|
||||
|
||||
result := NewStringURLSanitizedError(err, c.input, c.placeholder)
|
||||
|
||||
assert.Equal(t, encloseText(c.expected), result.Error(), "case %d: error should match", n)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue