1
0
Fork 0
forked from forgejo/forgejo

Batch hook pre- and post-receive calls (#8602)

* make notifyWatchers work on multiple actions

* more efficient multiple notifyWatchers

* Make CommitRepoAction take advantage of multiple actions

* Batch post and pre-receive results

* Set batch to 30

* Auto adjust timeout & add logging

* adjust processing message

* Add some messages to pre-receive

* Make any non-200 status code from pre-receive an error

* Add missing hookPrintResults

* Remove shortcut for single action

* mistaken merge fix

* oops

* Move master branch to the front

* If repo was empty and the master branch is pushed ensure that that is set as the default branch

* fixup

* fixup

* Missed HookOptions in setdefaultbranch

* Batch PushUpdateAddTag and PushUpdateDelTag

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
zeripath 2019-12-26 11:29:45 +00:00 committed by Lauris BH
parent 114d474f02
commit 7bfb83e064
10 changed files with 1063 additions and 440 deletions

View file

@ -9,6 +9,7 @@ import (
"fmt"
"net/http"
"net/url"
"time"
"code.gitea.io/gitea/modules/setting"
)
@ -22,9 +23,9 @@ const (
// HookOptions represents the options for the Hook calls
type HookOptions struct {
OldCommitID string
NewCommitID string
RefFullName string
OldCommitIDs []string
NewCommitIDs []string
RefFullNames []string
UserID int64
UserName string
GitObjectDirectory string
@ -34,23 +35,33 @@ type HookOptions struct {
IsDeployKey bool
}
// HookPostReceiveResult represents an individual result from PostReceive
type HookPostReceiveResult struct {
Results []HookPostReceiveBranchResult
RepoWasEmpty bool
Err string
}
// HookPostReceiveBranchResult represents an individual branch result from PostReceive
type HookPostReceiveBranchResult struct {
Message bool
Create bool
Branch string
URL string
}
// HookPreReceive check whether the provided commits are allowed
func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s&gitAlternativeObjectDirectories=%s&gitQuarantinePath=%s&prID=%d&isDeployKey=%t",
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s",
url.PathEscape(ownerName),
url.PathEscape(repoName),
url.QueryEscape(opts.OldCommitID),
url.QueryEscape(opts.NewCommitID),
url.QueryEscape(opts.RefFullName),
opts.UserID,
url.QueryEscape(opts.GitObjectDirectory),
url.QueryEscape(opts.GitAlternativeObjectDirectories),
url.QueryEscape(opts.GitQuarantinePath),
opts.ProtectedBranchID,
opts.IsDeployKey,
)
resp, err := newInternalRequest(reqURL, "GET").Response()
req := newInternalRequest(reqURL, "POST")
req = req.Header("Content-Type", "application/json")
jsonBytes, _ := json.Marshal(opts)
req.Body(jsonBytes)
req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second)
resp, err := req.Response()
if err != nil {
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
}
@ -64,17 +75,18 @@ func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string)
}
// HookPostReceive updates services and users
func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]interface{}, string) {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&username=%s",
func HookPostReceive(ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, string) {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s",
url.PathEscape(ownerName),
url.PathEscape(repoName),
url.QueryEscape(opts.OldCommitID),
url.QueryEscape(opts.NewCommitID),
url.QueryEscape(opts.RefFullName),
opts.UserID,
url.QueryEscape(opts.UserName))
)
resp, err := newInternalRequest(reqURL, "GET").Response()
req := newInternalRequest(reqURL, "POST")
req = req.Header("Content-Type", "application/json")
req.SetTimeout(60*time.Second, time.Duration(60+len(opts.OldCommitIDs))*time.Second)
jsonBytes, _ := json.Marshal(opts)
req.Body(jsonBytes)
resp, err := req.Response()
if err != nil {
return nil, fmt.Sprintf("Unable to contact gitea: %v", err.Error())
}
@ -83,8 +95,30 @@ func HookPostReceive(ownerName, repoName string, opts HookOptions) (map[string]i
if resp.StatusCode != http.StatusOK {
return nil, decodeJSONError(resp).Err
}
res := map[string]interface{}{}
_ = json.NewDecoder(resp.Body).Decode(&res)
res := &HookPostReceiveResult{}
_ = json.NewDecoder(resp.Body).Decode(res)
return res, ""
}
// SetDefaultBranch will set the default branch to the provided branch for the provided repository
func SetDefaultBranch(ownerName, repoName, branch string) error {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/set-default-branch/%s/%s/%s",
url.PathEscape(ownerName),
url.PathEscape(repoName),
url.PathEscape(branch),
)
req := newInternalRequest(reqURL, "POST")
req = req.Header("Content-Type", "application/json")
req.SetTimeout(60*time.Second, 60*time.Second)
resp, err := req.Response()
if err != nil {
return fmt.Errorf("Unable to contact gitea: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Error returned from gitea: %v", decodeJSONError(resp).Err)
}
return nil
}

View file

@ -159,112 +159,132 @@ type CommitRepoActionOptions struct {
// CommitRepoAction adds new commit action to the repository, and prepare
// corresponding webhooks.
func CommitRepoAction(opts CommitRepoActionOptions) error {
pusher, err := models.GetUserByName(opts.PusherName)
if err != nil {
return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err)
}
func CommitRepoAction(optsList ...*CommitRepoActionOptions) error {
var pusher *models.User
var repo *models.Repository
actions := make([]*models.Action, len(optsList))
repo, err := models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName)
if err != nil {
return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err)
}
refName := git.RefEndName(opts.RefFullName)
// Change default branch and empty status only if pushed ref is non-empty branch.
if repo.IsEmpty && opts.NewCommitID != git.EmptySHA && strings.HasPrefix(opts.RefFullName, git.BranchPrefix) {
repo.DefaultBranch = refName
repo.IsEmpty = false
if refName != "master" {
gitRepo, err := git.OpenRepository(repo.RepoPath())
for i, opts := range optsList {
if pusher == nil || pusher.Name != opts.PusherName {
var err error
pusher, err = models.GetUserByName(opts.PusherName)
if err != nil {
return err
return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err)
}
if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
if !git.IsErrUnsupportedVersion(err) {
gitRepo.Close()
return err
}
if repo == nil || repo.OwnerID != opts.RepoOwnerID || repo.Name != opts.RepoName {
var err error
if repo != nil {
// Change repository empty status and update last updated time.
if err := models.UpdateRepository(repo, false); err != nil {
return fmt.Errorf("UpdateRepository: %v", err)
}
}
gitRepo.Close()
repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName)
if err != nil {
return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err)
}
}
}
refName := git.RefEndName(opts.RefFullName)
// Change repository empty status and update last updated time.
if err = models.UpdateRepository(repo, false); err != nil {
return fmt.Errorf("UpdateRepository: %v", err)
}
isNewBranch := false
opType := models.ActionCommitRepo
// Check it's tag push or branch.
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
opType = models.ActionPushTag
if opts.NewCommitID == git.EmptySHA {
opType = models.ActionDeleteTag
// Change default branch and empty status only if pushed ref is non-empty branch.
if repo.IsEmpty && opts.NewCommitID != git.EmptySHA && strings.HasPrefix(opts.RefFullName, git.BranchPrefix) {
repo.DefaultBranch = refName
repo.IsEmpty = false
if refName != "master" {
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return err
}
if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
if !git.IsErrUnsupportedVersion(err) {
gitRepo.Close()
return err
}
}
gitRepo.Close()
}
}
opts.Commits = &models.PushCommits{}
} else if opts.NewCommitID == git.EmptySHA {
opType = models.ActionDeleteBranch
opts.Commits = &models.PushCommits{}
} else {
// if not the first commit, set the compare URL.
if opts.OldCommitID == git.EmptySHA {
isNewBranch = true
isNewBranch := false
opType := models.ActionCommitRepo
// Check it's tag push or branch.
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
opType = models.ActionPushTag
if opts.NewCommitID == git.EmptySHA {
opType = models.ActionDeleteTag
}
opts.Commits = &models.PushCommits{}
} else if opts.NewCommitID == git.EmptySHA {
opType = models.ActionDeleteBranch
opts.Commits = &models.PushCommits{}
} else {
opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID)
// if not the first commit, set the compare URL.
if opts.OldCommitID == git.EmptySHA {
isNewBranch = true
} else {
opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID)
}
if err := UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil {
log.Error("updateIssuesCommit: %v", err)
}
}
if err = UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil {
log.Error("updateIssuesCommit: %v", err)
if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum {
opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum]
}
data, err := json.Marshal(opts.Commits)
if err != nil {
return fmt.Errorf("Marshal: %v", err)
}
actions[i] = &models.Action{
ActUserID: pusher.ID,
ActUser: pusher,
OpType: opType,
Content: string(data),
RepoID: repo.ID,
Repo: repo,
RefName: refName,
IsPrivate: repo.IsPrivate,
}
var isHookEventPush = true
switch opType {
case models.ActionCommitRepo: // Push
if isNewBranch {
notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName)
}
case models.ActionDeleteBranch: // Delete Branch
notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName)
case models.ActionPushTag: // Create
notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName)
case models.ActionDeleteTag: // Delete Tag
notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName)
default:
isHookEventPush = false
}
if isHookEventPush {
notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits)
}
}
if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum {
opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum]
if repo != nil {
// Change repository empty status and update last updated time.
if err := models.UpdateRepository(repo, false); err != nil {
return fmt.Errorf("UpdateRepository: %v", err)
}
}
data, err := json.Marshal(opts.Commits)
if err != nil {
return fmt.Errorf("Marshal: %v", err)
}
if err = models.NotifyWatchers(&models.Action{
ActUserID: pusher.ID,
ActUser: pusher,
OpType: opType,
Content: string(data),
RepoID: repo.ID,
Repo: repo,
RefName: refName,
IsPrivate: repo.IsPrivate,
}); err != nil {
if err := models.NotifyWatchers(actions...); err != nil {
return fmt.Errorf("NotifyWatchers: %v", err)
}
var isHookEventPush = true
switch opType {
case models.ActionCommitRepo: // Push
if isNewBranch {
notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName)
}
case models.ActionDeleteBranch: // Delete Branch
notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName)
case models.ActionPushTag: // Create
notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName)
case models.ActionDeleteTag: // Delete Tag
notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName)
default:
isHookEventPush = false
}
if isHookEventPush {
notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits)
}
return nil
}

View file

@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/assert"
)
func testCorrectRepoAction(t *testing.T, opts CommitRepoActionOptions, actionBean *models.Action) {
func testCorrectRepoAction(t *testing.T, opts *CommitRepoActionOptions, actionBean *models.Action) {
models.AssertNotExistsBean(t, actionBean)
assert.NoError(t, CommitRepoAction(opts))
models.AssertExistsAndLoadBean(t, actionBean)
@ -121,7 +121,7 @@ func TestCommitRepoAction(t *testing.T) {
s.action.Repo = repo
s.action.IsPrivate = repo.IsPrivate
testCorrectRepoAction(t, s.commitRepoActionOptions, &s.action)
testCorrectRepoAction(t, &s.commitRepoActionOptions, &s.action)
}
}

View file

@ -432,6 +432,7 @@ type PushUpdateOptions struct {
RefFullName string
OldCommitID string
NewCommitID string
Branch string
}
// PushUpdate must be called for any push actions in order to
@ -460,60 +461,12 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions)
log.Error("Failed to update size for repository: %v", err)
}
var commits = &models.PushCommits{}
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
// If is tag reference
tagName := opts.RefFullName[len(git.TagPrefix):]
if isDelRef {
err = models.PushUpdateDeleteTag(repo, tagName)
if err != nil {
return fmt.Errorf("PushUpdateDeleteTag: %v", err)
}
} else {
// Clear cache for tag commit count
cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
err = models.PushUpdateAddTag(repo, gitRepo, tagName)
if err != nil {
return fmt.Errorf("PushUpdateAddTag: %v", err)
}
}
} else if !isDelRef {
// If is branch reference
// Clear cache for branch commit count
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true))
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
if err != nil {
return fmt.Errorf("gitRepo.GetCommit: %v", err)
}
// Push new branch.
var l *list.List
if isNewRef {
l, err = newCommit.CommitsBeforeLimit(10)
if err != nil {
return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err)
}
} else {
l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID)
if err != nil {
return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err)
}
}
commits = models.ListToPushCommits(l)
commitRepoActionOptions, err := createCommitRepoActionOption(repo, gitRepo, &opts)
if err != nil {
return err
}
if err := CommitRepoAction(CommitRepoActionOptions{
PusherName: opts.PusherName,
RepoOwnerID: repo.OwnerID,
RepoName: repo.Name,
RefFullName: opts.RefFullName,
OldCommitID: opts.OldCommitID,
NewCommitID: opts.NewCommitID,
Commits: commits,
}); err != nil {
if err := CommitRepoAction(commitRepoActionOptions); err != nil {
return fmt.Errorf("CommitRepoAction: %v", err)
}
@ -532,3 +485,174 @@ func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions)
return nil
}
// PushUpdates generates push action history feeds for push updating multiple refs
func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error {
repoPath := repo.RepoPath()
_, err := git.NewCommand("update-server-info").RunInDir(repoPath)
if err != nil {
return fmt.Errorf("Failed to call 'git update-server-info': %v", err)
}
gitRepo, err := git.OpenRepository(repoPath)
if err != nil {
return fmt.Errorf("OpenRepository: %v", err)
}
if err = repo.UpdateSize(); err != nil {
log.Error("Failed to update size for repository: %v", err)
}
actions, err := createCommitRepoActions(repo, gitRepo, optsList)
if err != nil {
return err
}
if err := CommitRepoAction(actions...); err != nil {
return fmt.Errorf("CommitRepoAction: %v", err)
}
var pusher *models.User
for _, opts := range optsList {
if pusher == nil || pusher.ID != opts.PusherID {
var err error
pusher, err = models.GetUserByID(opts.PusherID)
if err != nil {
return err
}
}
log.Trace("TriggerTask '%s/%s' by %s", repo.Name, opts.Branch, pusher.Name)
go pull_service.AddTestPullRequestTask(pusher, repo.ID, opts.Branch, true)
if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil {
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
}
}
return nil
}
func createCommitRepoActions(repo *models.Repository, gitRepo *git.Repository, optsList []*PushUpdateOptions) ([]*CommitRepoActionOptions, error) {
addTags := make([]string, 0, len(optsList))
delTags := make([]string, 0, len(optsList))
actions := make([]*CommitRepoActionOptions, 0, len(optsList))
for _, opts := range optsList {
isNewRef := opts.OldCommitID == git.EmptySHA
isDelRef := opts.NewCommitID == git.EmptySHA
if isNewRef && isDelRef {
return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA)
}
var commits = &models.PushCommits{}
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
// If is tag reference
tagName := opts.RefFullName[len(git.TagPrefix):]
if isDelRef {
delTags = append(delTags, tagName)
} else {
cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
addTags = append(addTags, tagName)
}
} else if !isDelRef {
// If is branch reference
// Clear cache for branch commit count
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true))
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
if err != nil {
return nil, fmt.Errorf("gitRepo.GetCommit: %v", err)
}
// Push new branch.
var l *list.List
if isNewRef {
l, err = newCommit.CommitsBeforeLimit(10)
if err != nil {
return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err)
}
} else {
l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID)
if err != nil {
return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err)
}
}
commits = models.ListToPushCommits(l)
}
actions = append(actions, &CommitRepoActionOptions{
PusherName: opts.PusherName,
RepoOwnerID: repo.OwnerID,
RepoName: repo.Name,
RefFullName: opts.RefFullName,
OldCommitID: opts.OldCommitID,
NewCommitID: opts.NewCommitID,
Commits: commits,
})
}
if err := models.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil {
return nil, fmt.Errorf("PushUpdateAddDeleteTags: %v", err)
}
return actions, nil
}
func createCommitRepoActionOption(repo *models.Repository, gitRepo *git.Repository, opts *PushUpdateOptions) (*CommitRepoActionOptions, error) {
isNewRef := opts.OldCommitID == git.EmptySHA
isDelRef := opts.NewCommitID == git.EmptySHA
if isNewRef && isDelRef {
return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA)
}
var commits = &models.PushCommits{}
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
// If is tag reference
tagName := opts.RefFullName[len(git.TagPrefix):]
if isDelRef {
if err := models.PushUpdateDeleteTag(repo, tagName); err != nil {
return nil, fmt.Errorf("PushUpdateDeleteTag: %v", err)
}
} else {
// Clear cache for tag commit count
cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil {
return nil, fmt.Errorf("PushUpdateAddTag: %v", err)
}
}
} else if !isDelRef {
// If is branch reference
// Clear cache for branch commit count
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true))
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
if err != nil {
return nil, fmt.Errorf("gitRepo.GetCommit: %v", err)
}
// Push new branch.
var l *list.List
if isNewRef {
l, err = newCommit.CommitsBeforeLimit(10)
if err != nil {
return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err)
}
} else {
l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID)
if err != nil {
return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err)
}
}
commits = models.ListToPushCommits(l)
}
return &CommitRepoActionOptions{
PusherName: opts.PusherName,
RepoOwnerID: repo.OwnerID,
RepoName: repo.Name,
RefFullName: opts.RefFullName,
OldCommitID: opts.OldCommitID,
NewCommitID: opts.NewCommitID,
Commits: commits,
}, nil
}

View file

@ -197,11 +197,11 @@ func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) erro
}
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
if err != nil && !git.IsErrNotExist(err) {
return fmt.Errorf("GetTagCommitID: %v", err)
return fmt.Errorf("GetTagCommitID: %s: %v", rel.TagName, err)
}
if git.IsErrNotExist(err) || commitID != rel.Sha1 {
if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil {
return fmt.Errorf("PushUpdateDeleteTag: %v", err)
return fmt.Errorf("PushUpdateDeleteTag: %s: %v", rel.TagName, err)
}
} else {
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
@ -215,7 +215,7 @@ func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) erro
for _, tagName := range tags {
if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil {
return fmt.Errorf("pushUpdateAddTag: %v", err)
return fmt.Errorf("pushUpdateAddTag: %s: %v", tagName, err)
}
}
}