diff options
Diffstat (limited to 'internal/db/migrations')
-rw-r--r-- | internal/db/migrations/migrations.go | 390 | ||||
-rw-r--r-- | internal/db/migrations/v13.go | 52 | ||||
-rw-r--r-- | internal/db/migrations/v14.go | 24 | ||||
-rw-r--r-- | internal/db/migrations/v15.go | 104 | ||||
-rw-r--r-- | internal/db/migrations/v16.go | 77 | ||||
-rw-r--r-- | internal/db/migrations/v17.go | 22 | ||||
-rw-r--r-- | internal/db/migrations/v18.go | 34 | ||||
-rw-r--r-- | internal/db/migrations/v19.go | 18 |
8 files changed, 721 insertions, 0 deletions
diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go new file mode 100644 index 00000000..79534484 --- /dev/null +++ b/internal/db/migrations/migrations.go @@ -0,0 +1,390 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "strings" + "time" + + "github.com/unknwon/com" + log "gopkg.in/clog.v1" + "xorm.io/xorm" + + "gogs.io/gogs/internal/tool" +) + +const _MIN_DB_VER = 10 + +type Migration interface { + Description() string + Migrate(*xorm.Engine) error +} + +type migration struct { + description string + migrate func(*xorm.Engine) error +} + +func NewMigration(desc string, fn func(*xorm.Engine) error) Migration { + return &migration{desc, fn} +} + +func (m *migration) Description() string { + return m.description +} + +func (m *migration) Migrate(x *xorm.Engine) error { + return m.migrate(x) +} + +// The version table. Should have only one row with id==1 +type Version struct { + ID int64 + Version int64 +} + +// This is a sequence of migrations. Add new migrations to the bottom of the list. +// If you want to "retire" a migration, remove it from the top of the list and +// update _MIN_VER_DB accordingly +var migrations = []Migration{ + // v0 -> v4 : before 0.6.0 -> last support 0.7.33 + // v4 -> v10: before 0.7.0 -> last support 0.9.141 + NewMigration("generate rands and salt for organizations", generateOrgRandsAndSalt), // V10 -> V11:v0.8.5 + NewMigration("convert date to unix timestamp", convertDateToUnix), // V11 -> V12:v0.9.2 + NewMigration("convert LDAP UseSSL option to SecurityProtocol", ldapUseSSLToSecurityProtocol), // V12 -> V13:v0.9.37 + + // v13 -> v14:v0.9.87 + NewMigration("set comment updated with created", setCommentUpdatedWithCreated), + // v14 -> v15:v0.9.147 + NewMigration("generate and migrate Git hooks", generateAndMigrateGitHooks), + // v15 -> v16:v0.10.16 + NewMigration("update repository sizes", updateRepositorySizes), + // v16 -> v17:v0.10.31 + NewMigration("remove invalid protect branch whitelist", removeInvalidProtectBranchWhitelist), + // v17 -> v18:v0.11.48 + NewMigration("store long text in repository description field", updateRepositoryDescriptionField), + // v18 -> v19:v0.11.55 + NewMigration("clean unlinked webhook and hook_tasks", cleanUnlinkedWebhookAndHookTasks), +} + +// Migrate database to current version +func Migrate(x *xorm.Engine) error { + if err := x.Sync(new(Version)); err != nil { + return fmt.Errorf("sync: %v", err) + } + + currentVersion := &Version{ID: 1} + has, err := x.Get(currentVersion) + if err != nil { + return fmt.Errorf("get: %v", err) + } else if !has { + // If the version record does not exist we think + // it is a fresh installation and we can skip all migrations. + currentVersion.ID = 0 + currentVersion.Version = int64(_MIN_DB_VER + len(migrations)) + + if _, err = x.InsertOne(currentVersion); err != nil { + return fmt.Errorf("insert: %v", err) + } + } + + v := currentVersion.Version + if _MIN_DB_VER > v { + log.Fatal(0, ` +Hi there, thank you for using Gogs for so long! +However, Gogs has stopped supporting auto-migration from your previously installed version. +But the good news is, it's very easy to fix this problem! +You can migrate your older database using a previous release, then you can upgrade to the newest version. + +Please save following instructions to somewhere and start working: + +- If you were using below 0.6.0 (e.g. 0.5.x), download last supported archive from following link: + https://gogs.io/gogs/releases/tag/v0.7.33 +- If you were using below 0.7.0 (e.g. 0.6.x), download last supported archive from following link: + https://gogs.io/gogs/releases/tag/v0.9.141 + +Once finished downloading, + +1. Extract the archive and to upgrade steps as usual. +2. Run it once. To verify, you should see some migration traces. +3. Once it starts web server successfully, stop it. +4. Now it's time to put back the release archive you originally intent to upgrade. +5. Enjoy! + +In case you're stilling getting this notice, go through instructions again until it disappears.`) + return nil + } + + if int(v-_MIN_DB_VER) > len(migrations) { + // User downgraded Gogs. + currentVersion.Version = int64(len(migrations) + _MIN_DB_VER) + _, err = x.Id(1).Update(currentVersion) + return err + } + for i, m := range migrations[v-_MIN_DB_VER:] { + log.Info("Migration: %s", m.Description()) + if err = m.Migrate(x); err != nil { + return fmt.Errorf("do migrate: %v", err) + } + currentVersion.Version = v + int64(i) + 1 + if _, err = x.Id(1).Update(currentVersion); err != nil { + return err + } + } + return nil +} + +func generateOrgRandsAndSalt(x *xorm.Engine) (err error) { + type User struct { + ID int64 `xorm:"pk autoincr"` + Rands string `xorm:"VARCHAR(10)"` + Salt string `xorm:"VARCHAR(10)"` + } + + orgs := make([]*User, 0, 10) + if err = x.Where("type=1").And("rands=''").Find(&orgs); err != nil { + return fmt.Errorf("select all organizations: %v", err) + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + for _, org := range orgs { + if org.Rands, err = tool.RandomString(10); err != nil { + return err + } + if org.Salt, err = tool.RandomString(10); err != nil { + return err + } + if _, err = sess.ID(org.ID).Update(org); err != nil { + return err + } + } + + return sess.Commit() +} + +type TAction struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 +} + +func (t *TAction) TableName() string { return "action" } + +type TNotice struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 +} + +func (t *TNotice) TableName() string { return "notice" } + +type TComment struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 +} + +func (t *TComment) TableName() string { return "comment" } + +type TIssue struct { + ID int64 `xorm:"pk autoincr"` + DeadlineUnix int64 + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TIssue) TableName() string { return "issue" } + +type TMilestone struct { + ID int64 `xorm:"pk autoincr"` + DeadlineUnix int64 + ClosedDateUnix int64 +} + +func (t *TMilestone) TableName() string { return "milestone" } + +type TAttachment struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 +} + +func (t *TAttachment) TableName() string { return "attachment" } + +type TLoginSource struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TLoginSource) TableName() string { return "login_source" } + +type TPull struct { + ID int64 `xorm:"pk autoincr"` + MergedUnix int64 +} + +func (t *TPull) TableName() string { return "pull_request" } + +type TRelease struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 +} + +func (t *TRelease) TableName() string { return "release" } + +type TRepo struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TRepo) TableName() string { return "repository" } + +type TMirror struct { + ID int64 `xorm:"pk autoincr"` + UpdatedUnix int64 + NextUpdateUnix int64 +} + +func (t *TMirror) TableName() string { return "mirror" } + +type TPublicKey struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TPublicKey) TableName() string { return "public_key" } + +type TDeployKey struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TDeployKey) TableName() string { return "deploy_key" } + +type TAccessToken struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TAccessToken) TableName() string { return "access_token" } + +type TUser struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TUser) TableName() string { return "user" } + +type TWebhook struct { + ID int64 `xorm:"pk autoincr"` + CreatedUnix int64 + UpdatedUnix int64 +} + +func (t *TWebhook) TableName() string { return "webhook" } + +func convertDateToUnix(x *xorm.Engine) (err error) { + log.Info("This migration could take up to minutes, please be patient.") + type Bean struct { + ID int64 `xorm:"pk autoincr"` + Created time.Time + Updated time.Time + Merged time.Time + Deadline time.Time + ClosedDate time.Time + NextUpdate time.Time + } + + var tables = []struct { + name string + cols []string + bean interface{} + }{ + {"action", []string{"created"}, new(TAction)}, + {"notice", []string{"created"}, new(TNotice)}, + {"comment", []string{"created"}, new(TComment)}, + {"issue", []string{"deadline", "created", "updated"}, new(TIssue)}, + {"milestone", []string{"deadline", "closed_date"}, new(TMilestone)}, + {"attachment", []string{"created"}, new(TAttachment)}, + {"login_source", []string{"created", "updated"}, new(TLoginSource)}, + {"pull_request", []string{"merged"}, new(TPull)}, + {"release", []string{"created"}, new(TRelease)}, + {"repository", []string{"created", "updated"}, new(TRepo)}, + {"mirror", []string{"updated", "next_update"}, new(TMirror)}, + {"public_key", []string{"created", "updated"}, new(TPublicKey)}, + {"deploy_key", []string{"created", "updated"}, new(TDeployKey)}, + {"access_token", []string{"created", "updated"}, new(TAccessToken)}, + {"user", []string{"created", "updated"}, new(TUser)}, + {"webhook", []string{"created", "updated"}, new(TWebhook)}, + } + + for _, table := range tables { + log.Info("Converting table: %s", table.name) + if err = x.Sync2(table.bean); err != nil { + return fmt.Errorf("Sync [table: %s]: %v", table.name, err) + } + + offset := 0 + for { + beans := make([]*Bean, 0, 100) + if err = x.Sql(fmt.Sprintf("SELECT * FROM `%s` ORDER BY id ASC LIMIT 100 OFFSET %d", + table.name, offset)).Find(&beans); err != nil { + return fmt.Errorf("select beans [table: %s, offset: %d]: %v", table.name, offset, err) + } + log.Trace("Table [%s]: offset: %d, beans: %d", table.name, offset, len(beans)) + if len(beans) == 0 { + break + } + offset += 100 + + baseSQL := "UPDATE `" + table.name + "` SET " + for _, bean := range beans { + valSQLs := make([]string, 0, len(table.cols)) + for _, col := range table.cols { + fieldSQL := "" + fieldSQL += col + "_unix = " + + switch col { + case "deadline": + if bean.Deadline.IsZero() { + continue + } + fieldSQL += com.ToStr(bean.Deadline.Unix()) + case "created": + fieldSQL += com.ToStr(bean.Created.Unix()) + case "updated": + fieldSQL += com.ToStr(bean.Updated.Unix()) + case "closed_date": + fieldSQL += com.ToStr(bean.ClosedDate.Unix()) + case "merged": + fieldSQL += com.ToStr(bean.Merged.Unix()) + case "next_update": + fieldSQL += com.ToStr(bean.NextUpdate.Unix()) + } + + valSQLs = append(valSQLs, fieldSQL) + } + + if len(valSQLs) == 0 { + continue + } + + if _, err = x.Exec(baseSQL + strings.Join(valSQLs, ",") + " WHERE id = " + com.ToStr(bean.ID)); err != nil { + return fmt.Errorf("update bean [table: %s, id: %d]: %v", table.name, bean.ID, err) + } + } + } + } + + return nil +} diff --git a/internal/db/migrations/v13.go b/internal/db/migrations/v13.go new file mode 100644 index 00000000..1097956e --- /dev/null +++ b/internal/db/migrations/v13.go @@ -0,0 +1,52 @@ +// Copyright 2016 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "strings" + + "github.com/unknwon/com" + "xorm.io/xorm" + "github.com/json-iterator/go" +) + +func ldapUseSSLToSecurityProtocol(x *xorm.Engine) error { + results, err := x.Query("SELECT `id`,`cfg` FROM `login_source` WHERE `type` = 2 OR `type` = 5") + if err != nil { + if strings.Contains(err.Error(), "no such column") { + return nil + } + return fmt.Errorf("select LDAP login sources: %v", err) + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + for _, result := range results { + cfg := map[string]interface{}{} + if err = jsoniter.Unmarshal(result["cfg"], &cfg); err != nil { + return fmt.Errorf("unmarshal JSON config: %v", err) + } + if com.ToStr(cfg["UseSSL"]) == "true" { + cfg["SecurityProtocol"] = 1 // LDAPS + } + delete(cfg, "UseSSL") + + data, err := jsoniter.Marshal(&cfg) + if err != nil { + return fmt.Errorf("marshal JSON config: %v", err) + } + + if _, err = sess.Exec("UPDATE `login_source` SET `cfg`=? WHERE `id`=?", + string(data), com.StrTo(result["id"]).MustInt64()); err != nil { + return fmt.Errorf("update config column: %v", err) + } + } + return sess.Commit() +} diff --git a/internal/db/migrations/v14.go b/internal/db/migrations/v14.go new file mode 100644 index 00000000..de8babed --- /dev/null +++ b/internal/db/migrations/v14.go @@ -0,0 +1,24 @@ +// Copyright 2016 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "xorm.io/xorm" +) + +func setCommentUpdatedWithCreated(x *xorm.Engine) (err error) { + type Comment struct { + UpdatedUnix int64 + } + + if err = x.Sync2(new(Comment)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } else if _, err = x.Exec("UPDATE comment SET updated_unix = created_unix"); err != nil { + return fmt.Errorf("set update_unix: %v", err) + } + return nil +} diff --git a/internal/db/migrations/v15.go b/internal/db/migrations/v15.go new file mode 100644 index 00000000..7f3b9504 --- /dev/null +++ b/internal/db/migrations/v15.go @@ -0,0 +1,104 @@ +// Copyright 2017 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/unknwon/com" + "xorm.io/xorm" + log "gopkg.in/clog.v1" + + "gogs.io/gogs/internal/setting" +) + +func generateAndMigrateGitHooks(x *xorm.Engine) (err error) { + type Repository struct { + ID int64 + OwnerID int64 + Name string + } + type User struct { + ID int64 + Name string + } + var ( + hookNames = []string{"pre-receive", "update", "post-receive"} + hookTpls = []string{ + fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' pre-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf), + fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' update $1 $2 $3\n", setting.ScriptType, setting.AppPath, setting.CustomConf), + fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' post-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf), + } + ) + + // Cleanup old update.log and http.log files. + filepath.Walk(setting.LogRootPath, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && + (strings.HasPrefix(filepath.Base(path), "update.log") || + strings.HasPrefix(filepath.Base(path), "http.log")) { + os.Remove(path) + } + return nil + }) + + return x.Where("id > 0").Iterate(new(Repository), + func(idx int, bean interface{}) error { + repo := bean.(*Repository) + if repo.Name == "." || repo.Name == ".." { + return nil + } + + user := new(User) + has, err := x.Where("id = ?", repo.OwnerID).Get(user) + if err != nil { + return fmt.Errorf("query owner of repository [repo_id: %d, owner_id: %d]: %v", repo.ID, repo.OwnerID, err) + } else if !has { + return nil + } + + repoBase := filepath.Join(setting.RepoRootPath, strings.ToLower(user.Name), strings.ToLower(repo.Name)) + repoPath := repoBase + ".git" + wikiPath := repoBase + ".wiki.git" + log.Trace("[%04d]: %s", idx, repoPath) + + // Note: we should not create hookDir here because update hook file should already exists inside this direcotry, + // if this directory does not exist, the current setup is not correct anyway. + hookDir := filepath.Join(repoPath, "hooks") + customHookDir := filepath.Join(repoPath, "custom_hooks") + wikiHookDir := filepath.Join(wikiPath, "hooks") + + for i, hookName := range hookNames { + oldHookPath := filepath.Join(hookDir, hookName) + newHookPath := filepath.Join(customHookDir, hookName) + + // Gogs didn't allow user to set custom update hook thus no migration for it. + // In case user runs this migration multiple times, and custom hook exists, + // we assume it's been migrated already. + if hookName != "update" && com.IsFile(oldHookPath) && !com.IsExist(customHookDir) { + os.MkdirAll(customHookDir, os.ModePerm) + if err = os.Rename(oldHookPath, newHookPath); err != nil { + return fmt.Errorf("move hook file to custom directory '%s' -> '%s': %v", oldHookPath, newHookPath, err) + } + } + + if err = ioutil.WriteFile(oldHookPath, []byte(hookTpls[i]), os.ModePerm); err != nil { + return fmt.Errorf("write hook file '%s': %v", oldHookPath, err) + } + + if com.IsDir(wikiPath) { + os.MkdirAll(wikiHookDir, os.ModePerm) + wikiHookPath := filepath.Join(wikiHookDir, hookName) + if err = ioutil.WriteFile(wikiHookPath, []byte(hookTpls[i]), os.ModePerm); err != nil { + return fmt.Errorf("write wiki hook file '%s': %v", wikiHookPath, err) + } + } + } + return nil + }) +} diff --git a/internal/db/migrations/v16.go b/internal/db/migrations/v16.go new file mode 100644 index 00000000..b374be39 --- /dev/null +++ b/internal/db/migrations/v16.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "path/filepath" + "strings" + + "xorm.io/xorm" + log "gopkg.in/clog.v1" + + "github.com/gogs/git-module" + + "gogs.io/gogs/internal/setting" +) + +func updateRepositorySizes(x *xorm.Engine) (err error) { + log.Info("This migration could take up to minutes, please be patient.") + type Repository struct { + ID int64 + OwnerID int64 + Name string + Size int64 + } + type User struct { + ID int64 + Name string + } + if err = x.Sync2(new(Repository)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + + // For the sake of SQLite3, we can't use x.Iterate here. + offset := 0 + for { + repos := make([]*Repository, 0, 10) + if err = x.Sql(fmt.Sprintf("SELECT * FROM `repository` ORDER BY id ASC LIMIT 10 OFFSET %d", offset)). + Find(&repos); err != nil { + return fmt.Errorf("select repos [offset: %d]: %v", offset, err) + } + log.Trace("Select [offset: %d, repos: %d]", offset, len(repos)) + if len(repos) == 0 { + break + } + offset += 10 + + for _, repo := range repos { + if repo.Name == "." || repo.Name == ".." { + continue + } + + user := new(User) + has, err := x.Where("id = ?", repo.OwnerID).Get(user) + if err != nil { + return fmt.Errorf("query owner of repository [repo_id: %d, owner_id: %d]: %v", repo.ID, repo.OwnerID, err) + } else if !has { + continue + } + + repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(user.Name), strings.ToLower(repo.Name)) + ".git" + countObject, err := git.GetRepoSize(repoPath) + if err != nil { + log.Warn("GetRepoSize: %v", err) + continue + } + + repo.Size = countObject.Size + countObject.SizePack + if _, err = x.Id(repo.ID).Cols("size").Update(repo); err != nil { + return fmt.Errorf("update size: %v", err) + } + } + } + return nil +} diff --git a/internal/db/migrations/v17.go b/internal/db/migrations/v17.go new file mode 100644 index 00000000..279ddf25 --- /dev/null +++ b/internal/db/migrations/v17.go @@ -0,0 +1,22 @@ +// Copyright 2017 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "xorm.io/xorm" +) + +func removeInvalidProtectBranchWhitelist(x *xorm.Engine) error { + exist, err := x.IsTableExist("protect_branch_whitelist") + if err != nil { + return fmt.Errorf("IsTableExist: %v", err) + } else if !exist { + return nil + } + _, err = x.Exec("DELETE FROM protect_branch_whitelist WHERE protect_branch_id = 0") + return err +} diff --git a/internal/db/migrations/v18.go b/internal/db/migrations/v18.go new file mode 100644 index 00000000..9ebb46ed --- /dev/null +++ b/internal/db/migrations/v18.go @@ -0,0 +1,34 @@ +// Copyright 2018 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "xorm.io/xorm" + + "gogs.io/gogs/internal/setting" +) + +func updateRepositoryDescriptionField(x *xorm.Engine) error { + exist, err := x.IsTableExist("repository") + if err != nil { + return fmt.Errorf("IsTableExist: %v", err) + } else if !exist { + return nil + } + switch { + case setting.UseMySQL: + _, err = x.Exec("ALTER TABLE `repository` MODIFY `description` VARCHAR(512);") + case setting.UseMSSQL: + _, err = x.Exec("ALTER TABLE `repository` ALTER COLUMN `description` VARCHAR(512);") + case setting.UsePostgreSQL: + _, err = x.Exec("ALTER TABLE `repository` ALTER COLUMN `description` TYPE VARCHAR(512);") + case setting.UseSQLite3: + // Sqlite3 uses TEXT type by default for any string type field. + // Keep this comment to mention that we don't missed any option. + } + return err +} diff --git a/internal/db/migrations/v19.go b/internal/db/migrations/v19.go new file mode 100644 index 00000000..bae2e355 --- /dev/null +++ b/internal/db/migrations/v19.go @@ -0,0 +1,18 @@ +// Copyright 2018 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "xorm.io/xorm" +) + +func cleanUnlinkedWebhookAndHookTasks(x *xorm.Engine) error { + _, err := x.Exec(`DELETE FROM webhook WHERE repo_id NOT IN (SELECT id FROM repository);`) + if err != nil { + return err + } + _, err = x.Exec(`DELETE FROM hook_task WHERE repo_id NOT IN (SELECT id FROM repository);`) + return err +} |