diff options
Diffstat (limited to 'models')
41 files changed, 598 insertions, 289 deletions
diff --git a/models/access.go b/models/access.go index df3cd766..c10c9750 100644 --- a/models/access.go +++ b/models/access.go @@ -9,7 +9,7 @@ import ( log "gopkg.in/clog.v1" - "github.com/gogits/gogs/models/errors" + "github.com/gogs/gogs/models/errors" ) type AccessMode int diff --git a/models/action.go b/models/action.go index 288a1812..61c6afc3 100644 --- a/models/action.go +++ b/models/action.go @@ -5,7 +5,6 @@ package models import ( - "encoding/json" "fmt" "path" "regexp" @@ -15,19 +14,20 @@ import ( "github.com/Unknwon/com" "github.com/go-xorm/xorm" + "github.com/json-iterator/go" log "gopkg.in/clog.v1" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/tool" ) type ActionType int -// To maintain backward compatibility only append to the end of list +// Note: To maintain backward compatibility only append to the end of list const ( ACTION_CREATE_REPO ActionType = iota + 1 // 1 ACTION_RENAME_REPO // 2 @@ -48,6 +48,9 @@ const ( ACTION_DELETE_BRANCH // 17 ACTION_DELETE_TAG // 18 ACTION_FORK_REPO // 19 + ACTION_MIRROR_SYNC_PUSH // 20 + ACTION_MIRROR_SYNC_CREATE // 21 + ACTION_MIRROR_SYNC_DELETE // 22 ) var ( @@ -77,14 +80,14 @@ type Action struct { OpType ActionType ActUserID int64 // Doer user ID ActUserName string // Doer user name - ActAvatar string `xorm:"-"` + ActAvatar string `xorm:"-" json:"-"` RepoID int64 `xorm:"INDEX"` RepoUserName string RepoName string RefName string IsPrivate bool `xorm:"NOT NULL DEFAULT false"` Content string `xorm:"TEXT"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 } @@ -251,7 +254,7 @@ func NewPushCommits() *PushCommits { } } -func (pc *PushCommits) ToApiPayloadCommits(repoPath, repoLink string) ([]*api.PayloadCommit, error) { +func (pc *PushCommits) ToApiPayloadCommits(repoPath, repoURL string) ([]*api.PayloadCommit, error) { commits := make([]*api.PayloadCommit, len(pc.Commits)) for i, commit := range pc.Commits { authorUsername := "" @@ -278,7 +281,7 @@ func (pc *PushCommits) ToApiPayloadCommits(repoPath, repoLink string) ([]*api.Pa commits[i] = &api.PayloadCommit{ ID: commit.Sha1, Message: commit.Message, - URL: fmt.Sprintf("%s/commit/%s", repoLink, commit.Sha1), + URL: fmt.Sprintf("%s/commit/%s", repoURL, commit.Sha1), Author: &api.PayloadUser{ Name: commit.AuthorName, Email: commit.AuthorEmail, @@ -498,7 +501,7 @@ func CommitRepoAction(opts CommitRepoActionOptions) error { opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] } - data, err := json.Marshal(opts.Commits) + data, err := jsoniter.Marshal(opts.Commits) if err != nil { return fmt.Errorf("Marshal: %v", err) } @@ -667,6 +670,71 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error return mergePullRequestAction(x, actUser, repo, pull) } +func mirrorSyncAction(opType ActionType, repo *Repository, refName string, data []byte) error { + return NotifyWatchers(&Action{ + ActUserID: repo.OwnerID, + ActUserName: repo.MustOwner().Name, + OpType: opType, + Content: string(data), + RepoID: repo.ID, + RepoUserName: repo.MustOwner().Name, + RepoName: repo.Name, + RefName: refName, + IsPrivate: repo.IsPrivate, + }) +} + +type MirrorSyncPushActionOptions struct { + RefName string + OldCommitID string + NewCommitID string + Commits *PushCommits +} + +// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits. +func MirrorSyncPushAction(repo *Repository, opts MirrorSyncPushActionOptions) error { + if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { + opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] + } + + apiCommits, err := opts.Commits.ToApiPayloadCommits(repo.RepoPath(), repo.HTMLURL()) + if err != nil { + return fmt.Errorf("ToApiPayloadCommits: %v", err) + } + + opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) + apiPusher := repo.MustOwner().APIFormat() + if err := PrepareWebhooks(repo, HOOK_EVENT_PUSH, &api.PushPayload{ + Ref: opts.RefName, + Before: opts.OldCommitID, + After: opts.NewCommitID, + CompareURL: setting.AppURL + opts.Commits.CompareURL, + Commits: apiCommits, + Repo: repo.APIFormat(nil), + Pusher: apiPusher, + Sender: apiPusher, + }); err != nil { + return fmt.Errorf("PrepareWebhooks: %v", err) + } + + data, err := jsoniter.Marshal(opts.Commits) + if err != nil { + return err + } + + return mirrorSyncAction(ACTION_MIRROR_SYNC_PUSH, repo, opts.RefName, data) +} + +// MirrorSyncCreateAction adds new action for mirror synchronization of new reference. +func MirrorSyncCreateAction(repo *Repository, refName string) error { + return mirrorSyncAction(ACTION_MIRROR_SYNC_CREATE, repo, refName, nil) +} + +// MirrorSyncCreateAction adds new action for mirror synchronization of delete reference. +func MirrorSyncDeleteAction(repo *Repository, refName string) error { + return mirrorSyncAction(ACTION_MIRROR_SYNC_DELETE, repo, refName, nil) +} + // GetFeeds returns action list of given user in given context. // actorID is the user who's requesting, ctxUserID is the user/org that is requested. // actorID can be -1 when isProfile is true or to skip the permission check. diff --git a/models/admin.go b/models/admin.go index 10dd61da..d82dcb32 100644 --- a/models/admin.go +++ b/models/admin.go @@ -15,8 +15,8 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/tool" ) type NoticeType int @@ -30,7 +30,7 @@ type Notice struct { ID int64 Type NoticeType Description string `xorm:"TEXT"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 } diff --git a/models/attachment.go b/models/attachment.go index fb4ec88d..31ca7f3e 100644 --- a/models/attachment.go +++ b/models/attachment.go @@ -15,7 +15,7 @@ import ( "github.com/go-xorm/xorm" gouuid "github.com/satori/go.uuid" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/setting" ) // Attachment represent a attachment of issue/comment/release. @@ -27,7 +27,7 @@ type Attachment struct { ReleaseID int64 `xorm:"INDEX"` Name string - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 } diff --git a/models/comment.go b/models/comment.go index bcea2268..447c15db 100644 --- a/models/comment.go +++ b/models/comment.go @@ -13,10 +13,10 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - api "github.com/gogits/go-gogs-client" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/markup" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/markup" ) // CommentType defines whether a comment is just a simple comment, an action (like close) or a reference. @@ -52,26 +52,26 @@ type Comment struct { ID int64 Type CommentType PosterID int64 - Poster *User `xorm:"-"` + Poster *User `xorm:"-" json:"-"` IssueID int64 `xorm:"INDEX"` - Issue *Issue `xorm:"-"` + Issue *Issue `xorm:"-" json:"-"` CommitID int64 Line int64 Content string `xorm:"TEXT"` - RenderedContent string `xorm:"-"` + RenderedContent string `xorm:"-" json:"-"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` + Updated time.Time `xorm:"-" json:"-"` UpdatedUnix int64 // Reference issue in commit message CommitSHA string `xorm:"VARCHAR(40)"` - Attachments []*Attachment `xorm:"-"` + Attachments []*Attachment `xorm:"-" json:"-"` // For view issue page. - ShowTag CommentTag `xorm:"-"` + ShowTag CommentTag `xorm:"-" json:"-"` } func (c *Comment) BeforeInsert() { diff --git a/models/git_diff.go b/models/git_diff.go index 3a47533f..cfaf5286 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -15,11 +15,11 @@ import ( "golang.org/x/net/html/charset" "golang.org/x/text/transform" - "github.com/gogits/git-module" + "github.com/gogs/git-module" - "github.com/gogits/gogs/pkg/tool" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/template/highlight" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/template/highlight" + "github.com/gogs/gogs/pkg/tool" ) type DiffSection struct { diff --git a/models/git_diff_test.go b/models/git_diff_test.go index 74032371..285e5646 100644 --- a/models/git_diff_test.go +++ b/models/git_diff_test.go @@ -8,7 +8,7 @@ import ( "html/template" "testing" - "github.com/gogits/git-module" + "github.com/gogs/git-module" dmp "github.com/sergi/go-diff/diffmatchpatch" ) diff --git a/models/issue.go b/models/issue.go index c2c3d253..2d0f06ab 100644 --- a/models/issue.go +++ b/models/issue.go @@ -13,11 +13,11 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - api "github.com/gogits/go-gogs-client" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/tool" ) var ( @@ -28,34 +28,34 @@ var ( type Issue struct { ID int64 RepoID int64 `xorm:"INDEX UNIQUE(repo_index)"` - Repo *Repository `xorm:"-"` + Repo *Repository `xorm:"-" json:"-"` Index int64 `xorm:"UNIQUE(repo_index)"` // Index in one repository. PosterID int64 - Poster *User `xorm:"-"` + Poster *User `xorm:"-" json:"-"` Title string `xorm:"name"` Content string `xorm:"TEXT"` - RenderedContent string `xorm:"-"` - Labels []*Label `xorm:"-"` + RenderedContent string `xorm:"-" json:"-"` + Labels []*Label `xorm:"-" json:"-"` MilestoneID int64 - Milestone *Milestone `xorm:"-"` + Milestone *Milestone `xorm:"-" json:"-"` Priority int AssigneeID int64 - Assignee *User `xorm:"-"` + Assignee *User `xorm:"-" json:"-"` IsClosed bool - IsRead bool `xorm:"-"` + IsRead bool `xorm:"-" json:"-"` IsPull bool // Indicates whether is a pull request or not. - PullRequest *PullRequest `xorm:"-"` + PullRequest *PullRequest `xorm:"-" json:"-"` NumComments int - Deadline time.Time `xorm:"-"` + Deadline time.Time `xorm:"-" json:"-"` DeadlineUnix int64 - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` + Updated time.Time `xorm:"-" json:"-"` UpdatedUnix int64 - Attachments []*Attachment `xorm:"-"` - Comments []*Comment `xorm:"-"` + Attachments []*Attachment `xorm:"-" json:"-"` + Comments []*Comment `xorm:"-" json:"-"` } func (issue *Issue) BeforeInsert() { diff --git a/models/issue_label.go b/models/issue_label.go index 7a463a7a..bec1cb1f 100644 --- a/models/issue_label.go +++ b/models/issue_label.go @@ -13,9 +13,9 @@ import ( "github.com/go-xorm/xorm" - api "github.com/gogits/go-gogs-client" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/pkg/tool" ) var labelColorPattern = regexp.MustCompile("#([a-fA-F0-9]{6})") @@ -60,8 +60,8 @@ type Label struct { Color string `xorm:"VARCHAR(7)"` NumIssues int NumClosedIssues int - NumOpenIssues int `xorm:"-"` - IsChecked bool `xorm:"-"` + NumOpenIssues int `xorm:"-" json:"-"` + IsChecked bool `xorm:"-" json:"-"` } func (label *Label) APIFormat() *api.Label { diff --git a/models/issue_mail.go b/models/issue_mail.go index 7ee38e63..b543c750 100644 --- a/models/issue_mail.go +++ b/models/issue_mail.go @@ -10,9 +10,9 @@ import ( "github.com/Unknwon/com" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/pkg/mailer" - "github.com/gogits/gogs/pkg/markup" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/mailer" + "github.com/gogs/gogs/pkg/markup" + "github.com/gogs/gogs/pkg/setting" ) func (issue *Issue) MailSubject() string { diff --git a/models/login_source.go b/models/login_source.go index 3a1162ec..88ad42dc 100644 --- a/models/login_source.go +++ b/models/login_source.go @@ -6,7 +6,6 @@ package models import ( "crypto/tls" - "encoding/json" "fmt" "net/smtp" "net/textproto" @@ -20,13 +19,14 @@ import ( "github.com/go-macaron/binding" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + "github.com/json-iterator/go" log "gopkg.in/clog.v1" "gopkg.in/ini.v1" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/auth/ldap" - "github.com/gogits/gogs/pkg/auth/pam" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/auth/ldap" + "github.com/gogs/gogs/pkg/auth/pam" + "github.com/gogs/gogs/pkg/setting" ) type LoginType int @@ -66,11 +66,11 @@ type LDAPConfig struct { } func (cfg *LDAPConfig) FromDB(bs []byte) error { - return json.Unmarshal(bs, &cfg) + return jsoniter.Unmarshal(bs, &cfg) } func (cfg *LDAPConfig) ToDB() ([]byte, error) { - return json.Marshal(cfg) + return jsoniter.Marshal(cfg) } func (cfg *LDAPConfig) SecurityProtocolName() string { @@ -87,11 +87,11 @@ type SMTPConfig struct { } func (cfg *SMTPConfig) FromDB(bs []byte) error { - return json.Unmarshal(bs, cfg) + return jsoniter.Unmarshal(bs, cfg) } func (cfg *SMTPConfig) ToDB() ([]byte, error) { - return json.Marshal(cfg) + return jsoniter.Marshal(cfg) } type PAMConfig struct { @@ -99,11 +99,11 @@ type PAMConfig struct { } func (cfg *PAMConfig) FromDB(bs []byte) error { - return json.Unmarshal(bs, &cfg) + return jsoniter.Unmarshal(bs, &cfg) } func (cfg *PAMConfig) ToDB() ([]byte, error) { - return json.Marshal(cfg) + return jsoniter.Marshal(cfg) } // AuthSourceFile contains information of an authentication source file. @@ -135,12 +135,12 @@ type LoginSource struct { IsActived bool `xorm:"NOT NULL DEFAULT false"` Cfg core.Conversion `xorm:"TEXT"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` + Updated time.Time `xorm:"-" json:"-"` UpdatedUnix int64 - LocalFile *AuthSourceFile `xorm:"-"` + LocalFile *AuthSourceFile `xorm:"-" json:"-"` } func (s *LoginSource) BeforeInsert() { diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 5192419b..44496a15 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -13,7 +13,7 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/pkg/tool" ) const _MIN_DB_VER = 10 @@ -64,6 +64,8 @@ var migrations = []Migration{ NewMigration("update repository sizes", updateRepositorySizes), // v16 -> v17:v0.10.31 NewMigration("remove invalid protect branch whitelist", removeInvalidProtectBranchWhitelist), + // v17 -> v18:v0.11.48 + NewMigration("store long text in repository description field", updateRepositoryDescriptionField), // v18 -> v19:v0.11.55 NewMigration("clean unlinked webhook and hook_tasks", cleanUnlinkedWebhookAndHookTasks), } @@ -100,9 +102,9 @@ You can migrate your older database using a previous release, then you can upgra Please save following instructions to somewhere and start working: - If you were using below 0.6.0 (e.g. 0.5.x), download last supported archive from following link: - https://github.com/gogits/gogs/releases/tag/v0.7.33 + https://github.com/gogs/gogs/releases/tag/v0.7.33 - If you were using below 0.7.0 (e.g. 0.6.x), download last supported archive from following link: - https://github.com/gogits/gogs/releases/tag/v0.9.141 + https://github.com/gogs/gogs/releases/tag/v0.9.141 Once finished downloading, diff --git a/models/migrations/v13.go b/models/migrations/v13.go index f81271f9..3f7e81df 100644 --- a/models/migrations/v13.go +++ b/models/migrations/v13.go @@ -5,12 +5,12 @@ package migrations import ( - "encoding/json" "fmt" "strings" "github.com/Unknwon/com" "github.com/go-xorm/xorm" + "github.com/json-iterator/go" ) func ldapUseSSLToSecurityProtocol(x *xorm.Engine) error { @@ -30,17 +30,17 @@ func ldapUseSSLToSecurityProtocol(x *xorm.Engine) error { for _, result := range results { cfg := map[string]interface{}{} - if err = json.Unmarshal(result["cfg"], &cfg); err != nil { - return fmt.Errorf("decode JSON config: %v", err) + if err = jsoniter.Unmarshal(result["cfg"], &cfg); err != nil { + return fmt.Errorf("unmarshal JSON config: %v", err) } if com.ToStr(cfg["UseSSL"]) == "true" { cfg["SecurityProtocol"] = 1 // LDAPS } delete(cfg, "UseSSL") - data, err := json.Marshal(&cfg) + data, err := jsoniter.Marshal(&cfg) if err != nil { - return fmt.Errorf("encode JSON config: %v", err) + return fmt.Errorf("marshal JSON config: %v", err) } if _, err = sess.Exec("UPDATE `login_source` SET `cfg`=? WHERE `id`=?", diff --git a/models/migrations/v15.go b/models/migrations/v15.go index ebee4964..d6cccb1d 100644 --- a/models/migrations/v15.go +++ b/models/migrations/v15.go @@ -15,7 +15,7 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/setting" ) func generateAndMigrateGitHooks(x *xorm.Engine) (err error) { diff --git a/models/migrations/v16.go b/models/migrations/v16.go index c5facc81..e95fe10a 100644 --- a/models/migrations/v16.go +++ b/models/migrations/v16.go @@ -12,9 +12,9 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - "github.com/gogits/git-module" + "github.com/gogs/git-module" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/setting" ) func updateRepositorySizes(x *xorm.Engine) (err error) { diff --git a/models/migrations/v18.go b/models/migrations/v18.go new file mode 100644 index 00000000..086cd27a --- /dev/null +++ b/models/migrations/v18.go @@ -0,0 +1,34 @@ +// Copyright 2018 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "github.com/go-xorm/xorm" + + "github.com/gogs/gogs/pkg/setting" +) + +func updateRepositoryDescriptionField(x *xorm.Engine) error { + exist, err := x.IsTableExist("repository") + if err != nil { + return fmt.Errorf("IsTableExist: %v", err) + } else if !exist { + return nil + } + switch { + case setting.UseMySQL: + _, err = x.Exec("ALTER TABLE `repository` MODIFY `description` VARCHAR(512);") + case setting.UseMSSQL: + _, err = x.Exec("ALTER TABLE `repository` ALTER COLUMN `description` VARCHAR(512);") + case setting.UsePostgreSQL: + _, err = x.Exec("ALTER TABLE `repository` ALTER COLUMN `description` TYPE VARCHAR(512);") + case setting.UseSQLite3: + // Sqlite3 uses TEXT type by default for any string type field. + // Keep this comment to mention that we don't missed any option. + } + return err +} diff --git a/models/milestone.go b/models/milestone.go index 72fe5f63..0488e03a 100644 --- a/models/milestone.go +++ b/models/milestone.go @@ -11,9 +11,9 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - api "github.com/gogits/go-gogs-client" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/setting" ) // Milestone represents a milestone of repository. @@ -22,18 +22,18 @@ type Milestone struct { RepoID int64 `xorm:"INDEX"` Name string Content string `xorm:"TEXT"` - RenderedContent string `xorm:"-"` + RenderedContent string `xorm:"-" json:"-"` IsClosed bool NumIssues int NumClosedIssues int - NumOpenIssues int `xorm:"-"` + NumOpenIssues int `xorm:"-" json:"-"` Completeness int // Percentage(1-100). - IsOverDue bool `xorm:"-"` + IsOverDue bool `xorm:"-" json:"-"` - DeadlineString string `xorm:"-"` - Deadline time.Time `xorm:"-"` + DeadlineString string `xorm:"-" json:"-"` + Deadline time.Time `xorm:"-" json:"-"` DeadlineUnix int64 - ClosedDate time.Time `xorm:"-"` + ClosedDate time.Time `xorm:"-" json:"-"` ClosedDateUnix int64 } diff --git a/models/mirror.go b/models/mirror.go index f48e8efa..a2d3d629 100644 --- a/models/mirror.go +++ b/models/mirror.go @@ -15,12 +15,12 @@ import ( log "gopkg.in/clog.v1" "gopkg.in/ini.v1" - "github.com/gogits/git-module" + "github.com/gogs/git-module" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/process" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/sync" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/process" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/sync" ) var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) @@ -29,26 +29,26 @@ var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) type Mirror struct { ID int64 RepoID int64 - Repo *Repository `xorm:"-"` + Repo *Repository `xorm:"-" json:"-"` Interval int // Hour. EnablePrune bool `xorm:"NOT NULL DEFAULT true"` - Updated time.Time `xorm:"-"` - UpdatedUnix int64 - NextUpdate time.Time `xorm:"-"` - NextUpdateUnix int64 + // Last and next sync time of Git data from upstream + LastSync time.Time `xorm:"-" json:"-"` + LastSyncUnix int64 `xorm:"updated_unix"` + NextSync time.Time `xorm:"-" json:"-"` + NextSyncUnix int64 `xorm:"next_update_unix"` - address string `xorm:"-"` + address string `xorm:"-" json:"-"` } func (m *Mirror) BeforeInsert() { - m.UpdatedUnix = time.Now().Unix() - m.NextUpdateUnix = m.NextUpdate.Unix() + m.NextSyncUnix = m.NextSync.Unix() } func (m *Mirror) BeforeUpdate() { - m.UpdatedUnix = time.Now().Unix() - m.NextUpdateUnix = m.NextUpdate.Unix() + m.LastSyncUnix = m.LastSync.Unix() + m.NextSyncUnix = m.NextSync.Unix() } func (m *Mirror) AfterSet(colName string, _ xorm.Cell) { @@ -60,15 +60,15 @@ func (m *Mirror) AfterSet(colName string, _ xorm.Cell) { log.Error(3, "GetRepositoryByID [%d]: %v", m.ID, err) } case "updated_unix": - m.Updated = time.Unix(m.UpdatedUnix, 0).Local() + m.LastSync = time.Unix(m.LastSyncUnix, 0).Local() case "next_update_unix": - m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local() + m.NextSync = time.Unix(m.NextSyncUnix, 0).Local() } } -// ScheduleNextUpdate calculates and sets next update time. -func (m *Mirror) ScheduleNextUpdate() { - m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour) +// ScheduleNextSync calculates and sets next sync time based on repostiroy mirror setting. +func (m *Mirror) ScheduleNextSync() { + m.NextSync = time.Now().Add(time.Duration(m.Interval) * time.Hour) } // findPasswordInMirrorAddress returns start (inclusive) and end index (exclusive) @@ -188,8 +188,67 @@ func (m *Mirror) SaveAddress(addr string) error { return cfg.SaveToIndent(configPath, "\t") } +const GIT_SHORT_EMPTY_SHA = "0000000" + +// mirrorSyncResult contains information of a updated reference. +// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. +// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. +type mirrorSyncResult struct { + refName string + oldCommitID string + newCommitID string +} + +// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. +func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { + results := make([]*mirrorSyncResult, 0, 3) + lines := strings.Split(output, "\n") + for i := range lines { + // Make sure reference name is presented before continue + idx := strings.Index(lines[i], "-> ") + if idx == -1 { + continue + } + + refName := lines[i][idx+3:] + + switch { + case strings.HasPrefix(lines[i], " * "): // New reference + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: GIT_SHORT_EMPTY_SHA, + }) + case strings.HasPrefix(lines[i], " - "): // Delete reference + results = append(results, &mirrorSyncResult{ + refName: refName, + newCommitID: GIT_SHORT_EMPTY_SHA, + }) + case strings.HasPrefix(lines[i], " "): // New commits of a reference + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error(2, "SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "..") + if len(shas) != 2 { + log.Error(2, "Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + + default: + log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) + } + } + return results +} + // runSync returns true if sync finished without error. -func (m *Mirror) runSync() bool { +func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -204,29 +263,32 @@ func (m *Mirror) runSync() bool { if err := CreateRepositoryNotice(desc); err != nil { log.Error(2, "CreateRepositoryNotice: %v", err) } - return false + return nil, false } gitArgs := []string{"remote", "update"} if m.EnablePrune { gitArgs = append(gitArgs, "--prune") } - if _, stderr, err := process.ExecDir( + _, stderr, err := process.ExecDir( timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath), - "git", gitArgs...); err != nil { + "git", gitArgs...) + if err != nil { desc := fmt.Sprintf("Fail to update mirror repository '%s': %s", repoPath, stderr) log.Error(2, desc) if err = CreateRepositoryNotice(desc); err != nil { log.Error(2, "CreateRepositoryNotice: %v", err) } - return false + return nil, false } + output := stderr if err := m.Repo.UpdateSize(); err != nil { log.Error(2, "UpdateSize [repo_id: %d]: %v", m.Repo.ID, err) } if m.Repo.HasWiki() { + // Even if wiki sync failed, we still want results from the main repository if _, stderr, err := process.ExecDir( timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath), "git", "remote", "update", "--prune"); err != nil { @@ -235,11 +297,10 @@ func (m *Mirror) runSync() bool { if err = CreateRepositoryNotice(desc); err != nil { log.Error(2, "CreateRepositoryNotice: %v", err) } - return false } } - return true + return parseRemoteUpdateOutput(output), true } func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) { @@ -301,22 +362,92 @@ func MirrorUpdate() { func SyncMirrors() { // Start listening on new sync requests. for repoID := range MirrorQueue.Queue() { - log.Trace("SyncMirrors [repo_id: %v]", repoID) + log.Trace("SyncMirrors [repo_id: %d]", repoID) MirrorQueue.Remove(repoID) m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64()) if err != nil { - log.Error(2, "GetMirrorByRepoID [%s]: %v", m.RepoID, err) + log.Error(2, "GetMirrorByRepoID [%d]: %v", m.RepoID, err) continue } - if !m.runSync() { + results, ok := m.runSync() + if !ok { continue } - m.ScheduleNextUpdate() + m.ScheduleNextSync() if err = UpdateMirror(m); err != nil { - log.Error(2, "UpdateMirror [%s]: %v", m.RepoID, err) + log.Error(2, "UpdateMirror [%d]: %v", m.RepoID, err) + continue + } + + // TODO: + // - Create "Mirror Sync" webhook event + // - Create mirror sync (create, push and delete) events and trigger the "mirror sync" webhooks + + var gitRepo *git.Repository + if len(results) == 0 { + log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID) + } else { + gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) + if err != nil { + log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err) + continue + } + } + + for _, result := range results { + // Discard GitHub pull requests, i.e. refs/pull/* + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + + // Create reference + if result.oldCommitID == GIT_SHORT_EMPTY_SHA { + if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil { + log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Delete reference + if result.newCommitID == GIT_SHORT_EMPTY_SHA { + if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil { + log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Push commits + oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) + if err != nil { + log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) + if err != nil { + log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) + if err != nil { + log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) + continue + } + if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{ + RefName: result.refName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + Commits: ListToPushCommits(commits), + }); err != nil { + log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err) + continue + } + } + + if _, err = x.Exec("UPDATE mirror SET updated_unix = ? WHERE repo_id = ?", time.Now().Unix(), m.RepoID); err != nil { + log.Error(2, "Update 'mirror.updated_unix' [%d]: %v", m.RepoID, err) continue } @@ -324,14 +455,14 @@ func SyncMirrors() { // update if latest commit date is newer. commitDate, err := git.GetLatestCommitDate(m.Repo.RepoPath(), "") if err != nil { - log.Error(2, "GetLatestCommitDate [%s]: %v", m.RepoID, err) + log.Error(2, "GetLatestCommitDate [%d]: %v", m.RepoID, err) continue } else if commitDate.Before(m.Repo.Updated) { continue } if _, err = x.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", commitDate.Unix(), m.RepoID); err != nil { - log.Error(2, "Update repository 'updated_unix' [%s]: %v", m.RepoID, err) + log.Error(2, "Update 'repository.updated_unix' [%d]: %v", m.RepoID, err) continue } } diff --git a/models/mirror_test.go b/models/mirror_test.go index b4af58ba..d6e86502 100644 --- a/models/mirror_test.go +++ b/models/mirror_test.go @@ -10,6 +10,40 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +func Test_parseRemoteUpdateOutput(t *testing.T) { + Convey("Parse mirror remote update output", t, func() { + testCases := []struct { + output string + results []*mirrorSyncResult + }{ + { + ` +From https://try.gogs.io/unknwon/upsteam + * [new branch] develop -> develop + b0bb24f..1d85a4f master -> master + - [deleted] (none) -> bugfix +`, + []*mirrorSyncResult{ + {"develop", GIT_SHORT_EMPTY_SHA, ""}, + {"master", "b0bb24f", "1d85a4f"}, + {"bugfix", "", GIT_SHORT_EMPTY_SHA}, + }, + }, + } + + for _, tc := range testCases { + results := parseRemoteUpdateOutput(tc.output) + So(len(results), ShouldEqual, len(tc.results)) + + for i := range tc.results { + So(tc.results[i].refName, ShouldEqual, results[i].refName) + So(tc.results[i].oldCommitID, ShouldEqual, results[i].oldCommitID) + So(tc.results[i].newCommitID, ShouldEqual, results[i].newCommitID) + } + } + }) +} + func Test_findPasswordInMirrorAddress(t *testing.T) { Convey("Find password portion in mirror address", t, func() { testCases := []struct { diff --git a/models/models.go b/models/models.go index 6fcd2619..4392bb76 100644 --- a/models/models.go +++ b/models/models.go @@ -7,7 +7,6 @@ package models import ( "bufio" "database/sql" - "encoding/json" "errors" "fmt" "net/url" @@ -20,11 +19,12 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + "github.com/json-iterator/go" _ "github.com/lib/pq" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/models/migrations" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/models/migrations" + "github.com/gogs/gogs/pkg/setting" ) // Engine represents a XORM engine or session. @@ -285,8 +285,7 @@ func DumpDatabase(dirPath string) (err error) { } if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) { - enc := json.NewEncoder(f) - return enc.Encode(bean) + return jsoniter.NewEncoder(f).Encode(bean) }); err != nil { f.Close() return fmt.Errorf("fail to dump table '%s': %v", tableName, err) @@ -300,6 +299,10 @@ func DumpDatabase(dirPath string) (err error) { func ImportDatabase(dirPath string, verbose bool) (err error) { snakeMapper := core.SnakeMapper{} + skipInsertProcessors := map[string]bool{ + "mirror": true, + } + // Purposely create a local variable to not modify global variable tables := append(tables, new(Version)) for _, table := range tables { @@ -314,22 +317,24 @@ func ImportDatabase(dirPath string, verbose bool) (err error) { } if err = x.DropTables(table); err != nil { - return fmt.Errorf("fail to drop table '%s': %v", tableName, err) + return fmt.Errorf("drop table '%s': %v", tableName, err) } else if err = x.Sync2(table); err != nil { - return fmt.Errorf("fail to sync table '%s': %v", tableName, err) + return fmt.Errorf("sync table '%s': %v", tableName, err) } f, err := os.Open(tableFile) if err != nil { - return fmt.Errorf("fail to open JSON file: %v", err) + return fmt.Errorf("open JSON file: %v", err) } + rawTableName := x.TableName(table) + _, isInsertProcessor := table.(xorm.BeforeInsertProcessor) scanner := bufio.NewScanner(f) for scanner.Scan() { switch bean := table.(type) { case *LoginSource: meta := make(map[string]interface{}) - if err = json.Unmarshal(scanner.Bytes(), &meta); err != nil { - return fmt.Errorf("fail to unmarshal to map: %v", err) + if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil { + return fmt.Errorf("unmarshal to map: %v", err) } tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64()) @@ -346,12 +351,23 @@ func ImportDatabase(dirPath string, verbose bool) (err error) { table = bean } - if err = json.Unmarshal(scanner.Bytes(), table); err != nil { - return fmt.Errorf("fail to unmarshal to struct: %v", err) + if err = jsoniter.Unmarshal(scanner.Bytes(), table); err != nil { + return fmt.Errorf("unmarshal to struct: %v", err) } if _, err = x.Insert(table); err != nil { - return fmt.Errorf("fail to insert strcut: %v", err) + return fmt.Errorf("insert strcut: %v", err) + } + + // Reset created_unix back to the date save in archive because Insert method updates its value + if isInsertProcessor && !skipInsertProcessors[rawTableName] { + meta := make(map[string]interface{}) + if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil { + log.Error(2, "Failed to unmarshal to map: %v", err) + } + if _, err = x.Exec("UPDATE "+rawTableName+" SET created_unix=? WHERE id=?", meta["CreatedUnix"], meta["ID"]); err != nil { + log.Error(2, "Failed to reset 'created_unix': %v", err) + } } } @@ -360,7 +376,7 @@ func ImportDatabase(dirPath string, verbose bool) (err error) { rawTableName := snakeMapper.Obj2Table(tableName) seqName := rawTableName + "_id_seq" if _, err = x.Exec(fmt.Sprintf(`SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM "%s"), 1), false);`, seqName, rawTableName)); err != nil { - return fmt.Errorf("fail to reset table '%s' sequence: %v", rawTableName, err) + return fmt.Errorf("reset table '%s' sequence: %v", rawTableName, err) } } } diff --git a/models/org_team.go b/models/org_team.go index 3e8016e8..709db547 100644 --- a/models/org_team.go +++ b/models/org_team.go @@ -22,8 +22,8 @@ type Team struct { Name string Description string Authorize AccessMode - Repos []*Repository `xorm:"-"` - Members []*User `xorm:"-"` + Repos []*Repository `xorm:"-" json:"-"` + Members []*User `xorm:"-" json:"-"` NumRepos int NumMembers int } @@ -31,7 +31,7 @@ type Team struct { func (t *Team) AfterSet(colName string, _ xorm.Cell) { switch colName { case "num_repos": - // LEGACY [1.0]: this is backward compatibility bug fix for https://github.com/gogits/gogs/issues/3671 + // LEGACY [1.0]: this is backward compatibility bug fix for https://github.com/gogs/gogs/issues/3671 if t.NumRepos < 0 { t.NumRepos = 0 } diff --git a/models/pull.go b/models/pull.go index de188c09..43211ed7 100644 --- a/models/pull.go +++ b/models/pull.go @@ -15,13 +15,13 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/process" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/sync" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/process" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/sync" ) var PullRequestQueue = sync.NewUniqueQueue(setting.Repository.PullRequestQueueLength) @@ -48,13 +48,13 @@ type PullRequest struct { Status PullRequestStatus IssueID int64 `xorm:"INDEX"` - Issue *Issue `xorm:"-"` + Issue *Issue `xorm:"-" json:"-"` Index int64 HeadRepoID int64 - HeadRepo *Repository `xorm:"-"` + HeadRepo *Repository `xorm:"-" json:"-"` BaseRepoID int64 - BaseRepo *Repository `xorm:"-"` + BaseRepo *Repository `xorm:"-" json:"-"` HeadUserName string HeadBranch string BaseBranch string @@ -63,8 +63,8 @@ type PullRequest struct { HasMerged bool MergedCommitID string `xorm:"VARCHAR(40)"` MergerID int64 - Merger *User `xorm:"-"` - Merged time.Time `xorm:"-"` + Merger *User `xorm:"-" json:"-"` + Merged time.Time `xorm:"-" json:"-"` MergedUnix int64 } diff --git a/models/release.go b/models/release.go index d8334554..65f85431 100644 --- a/models/release.go +++ b/models/release.go @@ -13,35 +13,35 @@ import ( "github.com/go-xorm/xorm" log "gopkg.in/clog.v1" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/process" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/process" ) // Release represents a release of repository. type Release struct { ID int64 RepoID int64 - Repo *Repository `xorm:"-"` + Repo *Repository `xorm:"-" json:"-"` PublisherID int64 - Publisher *User `xorm:"-"` + Publisher *User `xorm:"-" json:"-"` TagName string LowerTagName string Target string Title string Sha1 string `xorm:"VARCHAR(40)"` NumCommits int64 - NumCommitsBehind int64 `xorm:"-"` + NumCommitsBehind int64 `xorm:"-" json:"-"` Note string `xorm:"TEXT"` IsDraft bool `xorm:"NOT NULL DEFAULT false"` IsPrerelease bool - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Attachments []*Attachment `xorm:"-"` + Attachments []*Attachment `xorm:"-" json:"-"` } func (r *Release) BeforeInsert() { diff --git a/models/repo.go b/models/repo.go index 827c9aaf..909af1d8 100644 --- a/models/repo.go +++ b/models/repo.go @@ -23,15 +23,15 @@ import ( log "gopkg.in/clog.v1" "gopkg.in/ini.v1" - git "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" - - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/bindata" - "github.com/gogits/gogs/pkg/markup" - "github.com/gogits/gogs/pkg/process" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/sync" + git "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" + + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/bindata" + "github.com/gogs/gogs/pkg/markup" + "github.com/gogs/gogs/pkg/process" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/sync" ) var repoWorkingPool = sync.NewExclusivePool() @@ -143,10 +143,10 @@ func NewRepoContext() { type Repository struct { ID int64 OwnerID int64 `xorm:"UNIQUE(s)"` - Owner *User `xorm:"-"` + Owner *User `xorm:"-" json:"-"` LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` Name string `xorm:"INDEX NOT NULL"` - Description string + Description string `xorm:"VARCHAR(512)"` Website string DefaultBranch string Size int64 `xorm:"NOT NULL DEFAULT 0"` @@ -156,20 +156,20 @@ type Repository struct { NumForks int NumIssues int NumClosedIssues int - NumOpenIssues int `xorm:"-"` + NumOpenIssues int `xorm:"-" json:"-"` NumPulls int NumClosedPulls int - NumOpenPulls int `xorm:"-"` + NumOpenPulls int `xorm:"-" json:"-"` NumMilestones int `xorm:"NOT NULL DEFAULT 0"` NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` - NumOpenMilestones int `xorm:"-"` - NumTags int `xorm:"-"` + NumOpenMilestones int `xorm:"-" json:"-"` + NumTags int `xorm:"-" json:"-"` IsPrivate bool IsBare bool IsMirror bool - *Mirror `xorm:"-"` + *Mirror `xorm:"-" json:"-"` // Advanced settings EnableWiki bool `xorm:"NOT NULL DEFAULT true"` @@ -182,18 +182,18 @@ type Repository struct { ExternalTrackerURL string ExternalTrackerFormat string ExternalTrackerStyle string - ExternalMetas map[string]string `xorm:"-"` + ExternalMetas map[string]string `xorm:"-" json:"-"` EnablePulls bool `xorm:"NOT NULL DEFAULT true"` PullsIgnoreWhitespace bool `xorm:"NOT NULL DEFAULT false"` PullsAllowRebase bool `xorm:"NOT NULL DEFAULT false"` IsFork bool `xorm:"NOT NULL DEFAULT false"` ForkID int64 - BaseRepo *Repository `xorm:"-"` + BaseRepo *Repository `xorm:"-" json:"-"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` + Updated time.Time `xorm:"-" json:"-"` UpdatedUnix int64 } @@ -750,7 +750,7 @@ func MigrateRepository(doer, owner *User, opts MigrateRepoOptions) (*Repository, RepoID: repo.ID, Interval: setting.Mirror.DefaultInterval, EnablePrune: true, - NextUpdate: time.Now().Add(time.Duration(setting.Mirror.DefaultInterval) * time.Hour), + NextSync: time.Now().Add(time.Duration(setting.Mirror.DefaultInterval) * time.Hour), }); err != nil { return repo, fmt.Errorf("InsertOne: %v", err) } @@ -1331,8 +1331,8 @@ func GetRepositoriesByForkID(forkID int64) ([]*Repository, error) { func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) { repo.LowerName = strings.ToLower(repo.Name) - if len(repo.Description) > 255 { - repo.Description = repo.Description[:255] + if len(repo.Description) > 512 { + repo.Description = repo.Description[:512] } if len(repo.Website) > 255 { repo.Website = repo.Website[:255] diff --git a/models/repo_branch.go b/models/repo_branch.go index 6439faf0..1d98fd12 100644 --- a/models/repo_branch.go +++ b/models/repo_branch.go @@ -9,10 +9,10 @@ import ( "strings" "github.com/Unknwon/com" - "github.com/gogits/git-module" + "github.com/gogs/git-module" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/tool" ) type Branch struct { diff --git a/models/repo_collaboration.go b/models/repo_collaboration.go index f3babd42..107e1c5a 100644 --- a/models/repo_collaboration.go +++ b/models/repo_collaboration.go @@ -9,7 +9,7 @@ import ( log "gopkg.in/clog.v1" - api "github.com/gogits/go-gogs-client" + api "github.com/gogs/go-gogs-client" ) // Collaboration represent the relation between an individual and a repository. diff --git a/models/repo_editor.go b/models/repo_editor.go index ebf25ecd..27683ade 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -19,11 +19,11 @@ import ( gouuid "github.com/satori/go.uuid" log "gopkg.in/clog.v1" - git "github.com/gogits/git-module" + git "github.com/gogs/git-module" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/process" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/process" + "github.com/gogs/gogs/pkg/setting" ) // ___________ .___.__ __ ___________.__.__ diff --git a/models/repo_test.go b/models/repo_test.go index 76025a4c..2d7f9c38 100644 --- a/models/repo_test.go +++ b/models/repo_test.go @@ -5,8 +5,8 @@ import ( . "github.com/smartystreets/goconvey/convey" - . "github.com/gogits/gogs/models" - "github.com/gogits/gogs/pkg/markup" + . "github.com/gogs/gogs/models" + "github.com/gogs/gogs/pkg/markup" ) func TestRepo(t *testing.T) { diff --git a/models/ssh_key.go b/models/ssh_key.go index 350c5f75..3a8d205f 100644 --- a/models/ssh_key.go +++ b/models/ssh_key.go @@ -23,9 +23,9 @@ import ( "golang.org/x/crypto/ssh" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/pkg/process" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/pkg/process" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/tool" ) const ( @@ -51,12 +51,12 @@ type PublicKey struct { Mode AccessMode `xorm:"NOT NULL DEFAULT 2"` Type KeyType `xorm:"NOT NULL DEFAULT 1"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` // Note: Updated must below Created for AfterSet. + Updated time.Time `xorm:"-" json:"-"` // Note: Updated must below Created for AfterSet. UpdatedUnix int64 - HasRecentActivity bool `xorm:"-"` - HasUsed bool `xorm:"-"` + HasRecentActivity bool `xorm:"-" json:"-"` + HasUsed bool `xorm:"-" json:"-"` } func (k *PublicKey) BeforeInsert() { @@ -568,14 +568,14 @@ type DeployKey struct { RepoID int64 `xorm:"UNIQUE(s) INDEX"` Name string Fingerprint string - Content string `xorm:"-"` + Content string `xorm:"-" json:"-"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` // Note: Updated must below Created for AfterSet. + Updated time.Time `xorm:"-" json:"-"` // Note: Updated must below Created for AfterSet. UpdatedUnix int64 - HasRecentActivity bool `xorm:"-"` - HasUsed bool `xorm:"-"` + HasRecentActivity bool `xorm:"-" json:"-"` + HasUsed bool `xorm:"-" json:"-"` } func (k *DeployKey) BeforeInsert() { diff --git a/models/ssh_key_test.go b/models/ssh_key_test.go index 537f16b0..02086ac4 100644 --- a/models/ssh_key_test.go +++ b/models/ssh_key_test.go @@ -11,7 +11,7 @@ import ( . "github.com/smartystreets/goconvey/convey" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/setting" ) func init() { diff --git a/models/token.go b/models/token.go index 940d76f4..7e1dc166 100644 --- a/models/token.go +++ b/models/token.go @@ -10,7 +10,7 @@ import ( "github.com/go-xorm/xorm" gouuid "github.com/satori/go.uuid" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/pkg/tool" ) // AccessToken represents a personal access token. @@ -20,12 +20,12 @@ type AccessToken struct { Name string Sha1 string `xorm:"UNIQUE VARCHAR(40)"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` // Note: Updated must below Created for AfterSet. + Updated time.Time `xorm:"-" json:"-"` // Note: Updated must below Created for AfterSet. UpdatedUnix int64 - HasRecentActivity bool `xorm:"-"` - HasUsed bool `xorm:"-"` + HasRecentActivity bool `xorm:"-" json:"-"` + HasUsed bool `xorm:"-" json:"-"` } func (t *AccessToken) BeforeInsert() { diff --git a/models/two_factor.go b/models/two_factor.go index 60d26147..b9e3581d 100644 --- a/models/two_factor.go +++ b/models/two_factor.go @@ -15,9 +15,9 @@ import ( "github.com/pquerna/otp/totp" log "gopkg.in/clog.v1" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/tool" ) // TwoFactor represents a two-factor authentication token. @@ -25,7 +25,7 @@ type TwoFactor struct { ID int64 UserID int64 `xorm:"UNIQUE"` Secret string - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 } diff --git a/models/update.go b/models/update.go index 95122fcc..db01392c 100644 --- a/models/update.go +++ b/models/update.go @@ -10,7 +10,7 @@ import ( "os/exec" "strings" - git "github.com/gogits/git-module" + git "github.com/gogs/git-module" ) // CommitToPushCommit transforms a git.Commit to PushCommit type. diff --git a/models/user.go b/models/user.go index 831ba3f4..ab836b49 100644 --- a/models/user.go +++ b/models/user.go @@ -26,13 +26,13 @@ import ( "golang.org/x/crypto/pbkdf2" log "gopkg.in/clog.v1" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/avatar" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/tool" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/avatar" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/tool" ) type UserType int @@ -55,17 +55,17 @@ type User struct { LoginSource int64 `xorm:"NOT NULL DEFAULT 0"` LoginName string Type UserType - OwnedOrgs []*User `xorm:"-"` - Orgs []*User `xorm:"-"` - Repos []*Repository `xorm:"-"` + OwnedOrgs []*User `xorm:"-" json:"-"` + Orgs []*User `xorm:"-" json:"-"` + Repos []*Repository `xorm:"-" json:"-"` Location string Website string Rands string `xorm:"VARCHAR(10)"` Salt string `xorm:"VARCHAR(10)"` - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` + Updated time.Time `xorm:"-" json:"-"` UpdatedUnix int64 // Remember visibility choice for convenience, true for private @@ -95,8 +95,8 @@ type User struct { Description string NumTeams int NumMembers int - Teams []*Team `xorm:"-"` - Members []*User `xorm:"-"` + Teams []*Team `xorm:"-" json:"-"` + Members []*User `xorm:"-" json:"-"` } func (u *User) BeforeInsert() { @@ -120,6 +120,11 @@ func (u *User) AfterSet(colName string, _ xorm.Cell) { } } +// IDStr returns string representation of user's ID. +func (u *User) IDStr() string { + return com.ToStr(u.ID) +} + func (u *User) APIFormat() *api.User { return &api.User{ ID: u.ID, diff --git a/models/user_cache.go b/models/user_cache.go new file mode 100644 index 00000000..45c3721b --- /dev/null +++ b/models/user_cache.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +// MailResendCacheKey returns key used for cache mail resend. +func (u *User) MailResendCacheKey() string { + return "MailResend_" + u.IDStr() +} + +// TwoFactorCacheKey returns key used for cache two factor passcode. +// e.g. TwoFactor_1_012664 +func (u *User) TwoFactorCacheKey(passcode string) string { + return "TwoFactor_" + u.IDStr() + "_" + passcode +} diff --git a/models/user_mail.go b/models/user_mail.go index a5911aa7..46197490 100644 --- a/models/user_mail.go +++ b/models/user_mail.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - "github.com/gogits/gogs/models/errors" + "github.com/gogs/gogs/models/errors" ) // EmailAdresses is the list of all email addresses of a user. Can contain the @@ -18,7 +18,7 @@ type EmailAddress struct { UID int64 `xorm:"INDEX NOT NULL"` Email string `xorm:"UNIQUE NOT NULL"` IsActivated bool - IsPrimary bool `xorm:"-"` + IsPrimary bool `xorm:"-" json:"-"` } // GetEmailAddresses returns all email addresses belongs to given user. diff --git a/models/webhook.go b/models/webhook.go index c37a1a47..59a1962a 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -9,22 +9,22 @@ import ( "crypto/sha256" "crypto/tls" "encoding/hex" - "encoding/json" "fmt" "io/ioutil" "strings" "time" "github.com/go-xorm/xorm" + "github.com/json-iterator/go" gouuid "github.com/satori/go.uuid" log "gopkg.in/clog.v1" - api "github.com/gogits/go-gogs-client" + api "github.com/gogs/go-gogs-client" - "github.com/gogits/gogs/models/errors" - "github.com/gogits/gogs/pkg/httplib" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/sync" + "github.com/gogs/gogs/models/errors" + "github.com/gogs/gogs/pkg/httplib" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/sync" ) var HookQueue = sync.NewUniqueQueue(setting.Webhook.QueueLength) @@ -68,8 +68,8 @@ type HookEvents struct { Fork bool `json:"fork"` Push bool `json:"push"` Issues bool `json:"issues"` - IssueComment bool `json:"issue_comment"` PullRequest bool `json:"pull_request"` + IssueComment bool `json:"issue_comment"` Release bool `json:"release"` } @@ -97,18 +97,18 @@ type Webhook struct { OrgID int64 URL string `xorm:"url TEXT"` ContentType HookContentType - Secret string `xorm:"TEXT"` - Events string `xorm:"TEXT"` - *HookEvent `xorm:"-"` - IsSSL bool `xorm:"is_ssl"` + Secret string `xorm:"TEXT"` + Events string `xorm:"TEXT"` + *HookEvent `xorm:"-"` // LEGACY [1.0]: Cannot ignore JSON here, it breaks old backup archive + IsSSL bool `xorm:"is_ssl"` IsActive bool HookTaskType HookTaskType Meta string `xorm:"TEXT"` // store hook-specific attributes LastStatus HookStatus // Last delivery status - Created time.Time `xorm:"-"` + Created time.Time `xorm:"-" json:"-"` CreatedUnix int64 - Updated time.Time `xorm:"-"` + Updated time.Time `xorm:"-" json:"-"` UpdatedUnix int64 } @@ -126,7 +126,7 @@ func (w *Webhook) AfterSet(colName string, _ xorm.Cell) { switch colName { case "events": w.HookEvent = &HookEvent{} - if err = json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil { + if err = jsoniter.Unmarshal([]byte(w.Events), w.HookEvent); err != nil { log.Error(3, "Unmarshal [%d]: %v", w.ID, err) } case "created_unix": @@ -138,7 +138,7 @@ func (w *Webhook) AfterSet(colName string, _ xorm.Cell) { func (w *Webhook) GetSlackHook() *SlackMeta { s := &SlackMeta{} - if err := json.Unmarshal([]byte(w.Meta), s); err != nil { + if err := jsoniter.Unmarshal([]byte(w.Meta), s); err != nil { log.Error(2, "GetSlackHook [%d]: %v", w.ID, err) } return s @@ -151,7 +151,7 @@ func (w *Webhook) History(page int) ([]*HookTask, error) { // UpdateEvent handles conversion from HookEvent to Events. func (w *Webhook) UpdateEvent() error { - data, err := json.Marshal(w.HookEvent) + data, err := jsoniter.Marshal(w.HookEvent) w.Events = string(data) return err } @@ -186,18 +186,18 @@ func (w *Webhook) HasIssuesEvent() bool { (w.ChooseEvents && w.HookEvents.Issues) } -// HasIssueCommentEvent returns true if hook enabled issue comment event. -func (w *Webhook) HasIssueCommentEvent() bool { - return w.SendEverything || - (w.ChooseEvents && w.HookEvents.IssueComment) -} - // HasPullRequestEvent returns true if hook enabled pull request event. func (w *Webhook) HasPullRequestEvent() bool { return w.SendEverything || (w.ChooseEvents && w.HookEvents.PullRequest) } +// HasIssueCommentEvent returns true if hook enabled issue comment event. +func (w *Webhook) HasIssueCommentEvent() bool { + return w.SendEverything || + (w.ChooseEvents && w.HookEvents.IssueComment) +} + // HasReleaseEvent returns true if hook enabled release event. func (w *Webhook) HasReleaseEvent() bool { return w.SendEverything || @@ -210,15 +210,15 @@ type eventChecker struct { } func (w *Webhook) EventsArray() []string { - events := make([]string, 0, 7) + events := make([]string, 0, 8) eventCheckers := []eventChecker{ {w.HasCreateEvent, HOOK_EVENT_CREATE}, {w.HasDeleteEvent, HOOK_EVENT_DELETE}, {w.HasForkEvent, HOOK_EVENT_FORK}, {w.HasPushEvent, HOOK_EVENT_PUSH}, {w.HasIssuesEvent, HOOK_EVENT_ISSUES}, - {w.HasIssueCommentEvent, HOOK_EVENT_ISSUE_COMMENT}, {w.HasPullRequestEvent, HOOK_EVENT_PULL_REQUEST}, + {w.HasIssueCommentEvent, HOOK_EVENT_ISSUE_COMMENT}, {w.HasReleaseEvent, HOOK_EVENT_RELEASE}, } for _, c := range eventCheckers { @@ -392,8 +392,8 @@ const ( HOOK_EVENT_FORK HookEventType = "fork" HOOK_EVENT_PUSH HookEventType = "push" HOOK_EVENT_ISSUES HookEventType = "issues" - HOOK_EVENT_ISSUE_COMMENT HookEventType = "issue_comment" HOOK_EVENT_PULL_REQUEST HookEventType = "pull_request" + HOOK_EVENT_ISSUE_COMMENT HookEventType = "issue_comment" HOOK_EVENT_RELEASE HookEventType = "release" ) @@ -418,21 +418,21 @@ type HookTask struct { Type HookTaskType URL string `xorm:"TEXT"` Signature string `xorm:"TEXT"` - api.Payloader `xorm:"-"` + api.Payloader `xorm:"-" json:"-"` PayloadContent string `xorm:"TEXT"` ContentType HookContentType EventType HookEventType IsSSL bool IsDelivered bool Delivered int64 - DeliveredString string `xorm:"-"` + DeliveredString string `xorm:"-" json:"-"` // History info. IsSucceed bool RequestContent string `xorm:"TEXT"` - RequestInfo *HookRequest `xorm:"-"` + RequestInfo *HookRequest `xorm:"-" json:"-"` ResponseContent string `xorm:"TEXT"` - ResponseInfo *HookResponse `xorm:"-"` + ResponseInfo *HookResponse `xorm:"-" json:"-"` } func (t *HookTask) BeforeUpdate() { @@ -456,7 +456,7 @@ func (t *HookTask) AfterSet(colName string, _ xorm.Cell) { } t.RequestInfo = &HookRequest{} - if err = json.Unmarshal([]byte(t.RequestContent), t.RequestInfo); err != nil { + if err = jsoniter.Unmarshal([]byte(t.RequestContent), t.RequestInfo); err != nil { log.Error(3, "Unmarshal[%d]: %v", t.ID, err) } @@ -466,14 +466,14 @@ func (t *HookTask) AfterSet(colName string, _ xorm.Cell) { } t.ResponseInfo = &HookResponse{} - if err = json.Unmarshal([]byte(t.ResponseContent), t.ResponseInfo); err != nil { + if err = jsoniter.Unmarshal([]byte(t.ResponseContent), t.ResponseInfo); err != nil { log.Error(3, "Unmarshal [%d]: %v", t.ID, err) } } } func (t *HookTask) MarshalJSON(v interface{}) string { - p, err := json.Marshal(v) + p, err := jsoniter.Marshal(v) if err != nil { log.Error(3, "Marshal [%d]: %v", t.ID, err) } @@ -549,14 +549,14 @@ func prepareHookTasks(e Engine, repo *Repository, event HookEventType, p api.Pay if !w.HasIssuesEvent() { continue } - case HOOK_EVENT_ISSUE_COMMENT: - if !w.HasIssueCommentEvent() { - continue - } case HOOK_EVENT_PULL_REQUEST: if !w.HasPullRequestEvent() { continue } + case HOOK_EVENT_ISSUE_COMMENT: + if !w.HasIssueCommentEvent() { + continue + } case HOOK_EVENT_RELEASE: if !w.HasReleaseEvent() { continue diff --git a/models/webhook_dingtalk.go b/models/webhook_dingtalk.go index 735cddde..99b623cc 100644 --- a/models/webhook_dingtalk.go +++ b/models/webhook_dingtalk.go @@ -5,12 +5,13 @@ package models import ( - "encoding/json" "fmt" "strings" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/json-iterator/go" + + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" ) const ( @@ -41,7 +42,7 @@ type DingtalkPayload struct { } func (p *DingtalkPayload) JSONPayload() ([]byte, error) { - data, err := json.MarshalIndent(p, "", " ") + data, err := jsoniter.MarshalIndent(p, "", " ") if err != nil { return []byte{}, err } diff --git a/models/webhook_discord.go b/models/webhook_discord.go index 9cacc837..5bd61a7a 100644 --- a/models/webhook_discord.go +++ b/models/webhook_discord.go @@ -5,15 +5,16 @@ package models import ( - "encoding/json" "fmt" "strconv" "strings" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/json-iterator/go" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" + + "github.com/gogs/gogs/pkg/setting" ) type DiscordEmbedFooterObject struct { @@ -49,7 +50,7 @@ type DiscordPayload struct { } func (p *DiscordPayload) JSONPayload() ([]byte, error) { - data, err := json.MarshalIndent(p, "", " ") + data, err := jsoniter.MarshalIndent(p, "", " ") if err != nil { return []byte{}, err } @@ -371,8 +372,8 @@ func getDiscordReleasePayload(p *api.ReleasePayload) (*DiscordPayload, error) { func GetDiscordPayload(p api.Payloader, event HookEventType, meta string) (payload *DiscordPayload, err error) { slack := &SlackMeta{} - if err := json.Unmarshal([]byte(meta), &slack); err != nil { - return nil, fmt.Errorf("json.Unmarshal: %v", err) + if err := jsoniter.Unmarshal([]byte(meta), &slack); err != nil { + return nil, fmt.Errorf("jsoniter.Unmarshal: %v", err) } switch event { diff --git a/models/webhook_slack.go b/models/webhook_slack.go index 76954795..09e58d3d 100644 --- a/models/webhook_slack.go +++ b/models/webhook_slack.go @@ -5,14 +5,15 @@ package models import ( - "encoding/json" "fmt" "strings" - "github.com/gogits/git-module" - api "github.com/gogits/go-gogs-client" + "github.com/json-iterator/go" - "github.com/gogits/gogs/pkg/setting" + "github.com/gogs/git-module" + api "github.com/gogs/go-gogs-client" + + "github.com/gogs/gogs/pkg/setting" ) type SlackMeta struct { @@ -40,7 +41,7 @@ type SlackPayload struct { } func (p *SlackPayload) JSONPayload() ([]byte, error) { - data, err := json.MarshalIndent(p, "", " ") + data, err := jsoniter.MarshalIndent(p, "", " ") if err != nil { return []byte{}, err } @@ -288,8 +289,8 @@ func getSlackReleasePayload(p *api.ReleasePayload) (*SlackPayload, error) { func GetSlackPayload(p api.Payloader, event HookEventType, meta string) (payload *SlackPayload, err error) { slack := &SlackMeta{} - if err := json.Unmarshal([]byte(meta), &slack); err != nil { - return nil, fmt.Errorf("json.Unmarshal: %v", err) + if err := jsoniter.Unmarshal([]byte(meta), &slack); err != nil { + return nil, fmt.Errorf("Unmarshal: %v", err) } switch event { diff --git a/models/wiki.go b/models/wiki.go index 7e085d0a..882ccdf0 100644 --- a/models/wiki.go +++ b/models/wiki.go @@ -15,10 +15,10 @@ import ( "github.com/Unknwon/com" - "github.com/gogits/git-module" + "github.com/gogs/git-module" - "github.com/gogits/gogs/pkg/setting" - "github.com/gogits/gogs/pkg/sync" + "github.com/gogs/gogs/pkg/setting" + "github.com/gogs/gogs/pkg/sync" ) var wikiWorkingPool = sync.NewExclusivePool() |