domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Sleep longer if request speed is over github limitation (#9335)

Browse source 
* Sleep longer if request speed is over github limitation

* improve code

* remove unused code

* fix lint

* Use github's rate limit remain value to determine how long to sleep

* Save reset time when finished github api request

* fix bug

* fix lint

* Add context.Context for sleep

* fix test

* improve code

* fix bug and lint

* fix import order
This commit is contained in:
Lunny Xiao 2019-12-17 12:16:54 +08:00 • committed by techknowlogick
parent d1a49977b0
commit ffc904b1e0
8 changed files with 86 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

64
modules/migrations/github.go
View file

@ -11,6 +11,7 @@ import (
"net/http"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
@ -73,6 +74,7 @@ type GithubDownloaderV3 struct {
repoName string
userName string
password string
rate *github.Rate
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
return &downloader
}
// SetContext set context
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
func (g *GithubDownloaderV3) sleep() {
for g.rate != nil && g.rate.Remaining <= 0 {
timer := time.NewTimer(time.Until(g.rate.Reset.Time))
select {
case <-g.ctx.Done():
timer.Stop()
return
case <-timer.C:
}
rates, _, err := g.client.RateLimits(g.ctx)
if err != nil {
log.Error("g.client.RateLimits: %s", err)
}
g.rate = rates.GetCore()
}
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
g.sleep()
gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.rate = &resp.Rate
// convert github repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
@ -126,8 +155,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
// GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
return r.Topics, err
g.sleep()
r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.rate = &resp.Rate
return r.Topics, nil
}
// GetMilestones returns milestones
@ -135,7 +169,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
g.sleep()
ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
if err != nil {
return nil, err
}
g.rate = &resp.Rate
for _, m := range ms {
var desc string
@ -189,7 +225,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
g.sleep()
ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@ -197,6 +234,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
if err != nil {
return nil, err
}
g.rate = &resp.Rate
for _, label := range ls {
labels = append(labels, convertGithubLabel(label))
@ -260,7 +298,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = 100
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
g.sleep()
ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@ -268,6 +307,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
if err != nil {
return nil, err
}
g.rate = &resp.Rate
for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release))
@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
g.sleep()
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, issue := range issues {
if issue.IsPullRequest() {
continue
@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
},
}
for {
g.sleep()
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, comment := range comments {
var email string
if comment.User.Email != nil {
@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
},
}
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
g.sleep()
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, pr := range prs {
var body string
if pr.Body != nil {