domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Add push to remote mirror repository (#15157)

Browse source 
* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.
This commit is contained in:
KN4CK3R 2021-06-14 19:20:43 +02:00 • committed by GitHub
parent 5d113bdd19
commit 440039c0cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 2468 additions and 885 deletions
Download patch file Download diff file Expand all files Collapse all files

63
modules/util/sanitize.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
@ -9,40 +9,53 @@ import (
"strings"
)
// urlSafeError wraps an error whose message may contain a sensitive URL
type urlSafeError struct {
err error
unsanitizedURL string
const userPlaceholder = "sanitized-credential"
const unparsableURL = "(unparsable url)"
type sanitizedError struct {
err error
replacer *strings.Replacer
}
func (err urlSafeError) Error() string {
return SanitizeMessage(err.err.Error(), err.unsanitizedURL)
func (err sanitizedError) Error() string {
return err.replacer.Replace(err.err.Error())
}
// URLSanitizedError returns the sanitized version an error whose message may
// contain a sensitive URL
func URLSanitizedError(err error, unsanitizedURL string) error {
return urlSafeError{err: err, unsanitizedURL: unsanitizedURL}
// NewSanitizedError wraps an error and replaces all old, new string pairs in the message text.
func NewSanitizedError(err error, oldnew ...string) error {
return sanitizedError{err: err, replacer: strings.NewReplacer(oldnew...)}
}
// SanitizeMessage sanitizes a message which may contains a sensitive URL
func SanitizeMessage(message, unsanitizedURL string) string {
sanitizedURL := SanitizeURLCredentials(unsanitizedURL, true)
return strings.ReplaceAll(message, unsanitizedURL, sanitizedURL)
// NewURLSanitizedError wraps an error and replaces the url credential or removes them.
func NewURLSanitizedError(err error, u *url.URL, usePlaceholder bool) error {
return sanitizedError{err: err, replacer: NewURLSanitizer(u, usePlaceholder)}
}
// SanitizeURLCredentials sanitizes a url, either removing user credentials
// or replacing them with a placeholder.
func SanitizeURLCredentials(unsanitizedURL string, usePlaceholder bool) string {
u, err := url.Parse(unsanitizedURL)
if err != nil {
// don't log the error, since it might contain unsanitized URL.
return "(unparsable url)"
}
// NewStringURLSanitizedError wraps an error and replaces the url credential or removes them.
// If the url can't get parsed it gets replaced with a placeholder string.
func NewStringURLSanitizedError(err error, unsanitizedURL string, usePlaceholder bool) error {
return sanitizedError{err: err, replacer: NewStringURLSanitizer(unsanitizedURL, usePlaceholder)}
}
// NewURLSanitizer creates a replacer for the url with the credential sanitized or removed.
func NewURLSanitizer(u *url.URL, usePlaceholder bool) *strings.Replacer {
old := u.String()
if u.User != nil && usePlaceholder {
u.User = url.User("<credentials>")
u.User = url.User(userPlaceholder)
} else {
u.User = nil
}
return u.String()
return strings.NewReplacer(old, u.String())
}
// NewStringURLSanitizer creates a replacer for the url with the credential sanitized or removed.
// If the url can't get parsed it gets replaced with a placeholder string
func NewStringURLSanitizer(unsanitizedURL string, usePlaceholder bool) *strings.Replacer {
u, err := url.Parse(unsanitizedURL)
if err != nil {
// don't log the error, since it might contain unsanitized URL.
return strings.NewReplacer(unsanitizedURL, unparsableURL)
}
return NewURLSanitizer(u, usePlaceholder)
}