mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-08-29 05:46:05 +02:00
Add migration test (#5773)
* Add migration test This commit adds a simple migration test for v1.5.3, v1.6.4 and v1.7.0-rc3 Signed-off-by: Andrew Thornton <art27@cantab.net> * Automigrate based on available dbs * remove old ini file * Standardise the dialect names
This commit is contained in:
parent
bc748f205a
commit
8917d66571
12 changed files with 277 additions and 1 deletions
|
@ -127,6 +127,7 @@ pipeline:
|
||||||
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
||||||
- apt-get install -y git-lfs
|
- apt-get install -y git-lfs
|
||||||
- (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.sqlite.test)' | sh)) &
|
- (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.sqlite.test)' | sh)) &
|
||||||
|
- make test-sqlite-migration
|
||||||
- make test-sqlite
|
- make test-sqlite
|
||||||
when:
|
when:
|
||||||
event: [ push, tag, pull_request ]
|
event: [ push, tag, pull_request ]
|
||||||
|
@ -141,6 +142,7 @@ pipeline:
|
||||||
commands:
|
commands:
|
||||||
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
||||||
- apt-get install -y git-lfs
|
- apt-get install -y git-lfs
|
||||||
|
- make test-mysql-migration
|
||||||
- make integration-test-coverage
|
- make integration-test-coverage
|
||||||
when:
|
when:
|
||||||
event: [ push, pull_request ]
|
event: [ push, pull_request ]
|
||||||
|
@ -157,6 +159,7 @@ pipeline:
|
||||||
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
||||||
- apt-get install -y git-lfs
|
- apt-get install -y git-lfs
|
||||||
- (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.test)' | sh)) &
|
- (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.test)' | sh)) &
|
||||||
|
- make test-mysql-migration
|
||||||
- make test-mysql
|
- make test-mysql
|
||||||
when:
|
when:
|
||||||
event: [ tag ]
|
event: [ tag ]
|
||||||
|
@ -172,6 +175,7 @@ pipeline:
|
||||||
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
||||||
- apt-get install -y git-lfs
|
- apt-get install -y git-lfs
|
||||||
- (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.test)' | sh)) &
|
- (sleep 1200 && (echo 'kill -ABRT $(pidof gitea) $(pidof integrations.test)' | sh)) &
|
||||||
|
- make test-pgsql-migration
|
||||||
- make test-pgsql
|
- make test-pgsql
|
||||||
when:
|
when:
|
||||||
event: [ push, tag, pull_request ]
|
event: [ push, tag, pull_request ]
|
||||||
|
@ -186,6 +190,7 @@ pipeline:
|
||||||
commands:
|
commands:
|
||||||
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
- curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
|
||||||
- apt-get install -y git-lfs
|
- apt-get install -y git-lfs
|
||||||
|
- make test-mssql-migration
|
||||||
- make test-mssql
|
- make test-mssql
|
||||||
when:
|
when:
|
||||||
event: [ push, tag, pull_request ]
|
event: [ push, tag, pull_request ]
|
||||||
|
|
28
Makefile
28
Makefile
|
@ -35,7 +35,7 @@ endif
|
||||||
|
|
||||||
LDFLAGS := -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)"
|
LDFLAGS := -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)"
|
||||||
|
|
||||||
PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list ./... | grep -v /vendor/))
|
PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list ./... | grep -v /vendor/)))
|
||||||
SOURCES ?= $(shell find . -name "*.go" -type f)
|
SOURCES ?= $(shell find . -name "*.go" -type f)
|
||||||
|
|
||||||
TAGS ?=
|
TAGS ?=
|
||||||
|
@ -197,6 +197,10 @@ test-vendor: vendor
|
||||||
test-sqlite: integrations.sqlite.test
|
test-sqlite: integrations.sqlite.test
|
||||||
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test
|
||||||
|
|
||||||
|
.PHONY: test-sqlite-migration
|
||||||
|
test-sqlite-migration: migrations.sqlite.test
|
||||||
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./migrations.sqlite.test
|
||||||
|
|
||||||
generate-ini:
|
generate-ini:
|
||||||
sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \
|
sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \
|
||||||
-e 's|{{TEST_MYSQL_DBNAME}}|${TEST_MYSQL_DBNAME}|g' \
|
-e 's|{{TEST_MYSQL_DBNAME}}|${TEST_MYSQL_DBNAME}|g' \
|
||||||
|
@ -218,14 +222,28 @@ generate-ini:
|
||||||
test-mysql: integrations.test generate-ini
|
test-mysql: integrations.test generate-ini
|
||||||
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mysql.ini ./integrations.test
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mysql.ini ./integrations.test
|
||||||
|
|
||||||
|
.PHONY: test-mysql-migration
|
||||||
|
test-mysql-migration: migrations.test generate-ini
|
||||||
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mysql.ini ./migrations.test
|
||||||
|
|
||||||
.PHONY: test-pgsql
|
.PHONY: test-pgsql
|
||||||
test-pgsql: integrations.test generate-ini
|
test-pgsql: integrations.test generate-ini
|
||||||
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/pgsql.ini ./integrations.test
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/pgsql.ini ./integrations.test
|
||||||
|
|
||||||
|
.PHONY: test-pgsql-migration
|
||||||
|
test-pgsql-migration: migrations.test generate-ini
|
||||||
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/pgsql.ini ./migrations.test
|
||||||
|
|
||||||
|
|
||||||
.PHONY: test-mssql
|
.PHONY: test-mssql
|
||||||
test-mssql: integrations.test generate-ini
|
test-mssql: integrations.test generate-ini
|
||||||
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mssql.ini ./integrations.test
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mssql.ini ./integrations.test
|
||||||
|
|
||||||
|
.PHONY: test-mssql-migration
|
||||||
|
test-mssql-migration: migrations.test generate-ini
|
||||||
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mssql.ini ./migrations.test
|
||||||
|
|
||||||
|
|
||||||
.PHONY: bench-sqlite
|
.PHONY: bench-sqlite
|
||||||
bench-sqlite: integrations.sqlite.test
|
bench-sqlite: integrations.sqlite.test
|
||||||
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench .
|
GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test -test.cpuprofile=cpu.out -test.run DontRunTests -test.bench .
|
||||||
|
@ -252,6 +270,14 @@ integrations.sqlite.test: $(SOURCES)
|
||||||
integrations.cover.test: $(SOURCES)
|
integrations.cover.test: $(SOURCES)
|
||||||
$(GO) test -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test
|
$(GO) test -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test
|
||||||
|
|
||||||
|
.PHONY: migrations.test
|
||||||
|
migrations.test: $(SOURCES)
|
||||||
|
$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.test
|
||||||
|
|
||||||
|
.PHONY: migrations.sqlite.test
|
||||||
|
migrations.sqlite.test: $(SOURCES)
|
||||||
|
$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
|
||||||
|
|
||||||
.PHONY: check
|
.PHONY: check
|
||||||
check: test
|
check: test
|
||||||
|
|
||||||
|
|
BIN
integrations/migration-test/gitea-v1.5.3.mysql.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.5.3.mysql.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.5.3.postgres.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.5.3.postgres.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.5.3.sqlite3.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.5.3.sqlite3.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.6.4.mysql.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.6.4.mysql.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.6.4.postgres.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.6.4.postgres.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.6.4.sqlite3.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.6.4.sqlite3.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.7.0.mysql.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.7.0.mysql.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.7.0.postgres.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.7.0.postgres.sql.gz
Normal file
Binary file not shown.
BIN
integrations/migration-test/gitea-v1.7.0.sqlite3.sql.gz
Normal file
BIN
integrations/migration-test/gitea-v1.7.0.sqlite3.sql.gz
Normal file
Binary file not shown.
245
integrations/migration-test/migration_test.go
Normal file
245
integrations/migration-test/migration_test.go
Normal file
|
@ -0,0 +1,245 @@
|
||||||
|
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package migrations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"compress/gzip"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models"
|
||||||
|
"code.gitea.io/gitea/models/migrations"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
|
||||||
|
"github.com/go-xorm/xorm"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
var currentEngine *xorm.Engine
|
||||||
|
|
||||||
|
func initMigrationTest() {
|
||||||
|
giteaRoot := os.Getenv("GITEA_ROOT")
|
||||||
|
if giteaRoot == "" {
|
||||||
|
fmt.Println("Environment variable $GITEA_ROOT not set")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
setting.AppPath = path.Join(giteaRoot, "gitea")
|
||||||
|
if _, err := os.Stat(setting.AppPath); err != nil {
|
||||||
|
fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
giteaConf := os.Getenv("GITEA_CONF")
|
||||||
|
if giteaConf == "" {
|
||||||
|
fmt.Println("Environment variable $GITEA_CONF not set")
|
||||||
|
os.Exit(1)
|
||||||
|
} else if !path.IsAbs(giteaConf) {
|
||||||
|
setting.CustomConf = path.Join(giteaRoot, giteaConf)
|
||||||
|
} else {
|
||||||
|
setting.CustomConf = giteaConf
|
||||||
|
}
|
||||||
|
|
||||||
|
setting.NewContext()
|
||||||
|
setting.CheckLFSVersion()
|
||||||
|
models.LoadConfigs()
|
||||||
|
}
|
||||||
|
|
||||||
|
func getDialect() string {
|
||||||
|
dialect := "sqlite"
|
||||||
|
switch {
|
||||||
|
case setting.UseSQLite3:
|
||||||
|
dialect = "sqlite"
|
||||||
|
case setting.UseMySQL:
|
||||||
|
dialect = "mysql"
|
||||||
|
case setting.UsePostgreSQL:
|
||||||
|
dialect = "pgsql"
|
||||||
|
case setting.UseMSSQL:
|
||||||
|
dialect = "mssql"
|
||||||
|
}
|
||||||
|
return dialect
|
||||||
|
}
|
||||||
|
|
||||||
|
func availableVersions() ([]string, error) {
|
||||||
|
migrationsDir, err := os.Open("integrations/migration-test")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer migrationsDir.Close()
|
||||||
|
versionRE, err := regexp.Compile("gitea-v(?P<version>.+)\\." + regexp.QuoteMeta(models.DbCfg.Type) + "\\.sql.gz")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
filenames, err := migrationsDir.Readdirnames(-1)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
versions := []string{}
|
||||||
|
for _, filename := range filenames {
|
||||||
|
if versionRE.MatchString(filename) {
|
||||||
|
substrings := versionRE.FindStringSubmatch(filename)
|
||||||
|
versions = append(versions, substrings[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(versions)
|
||||||
|
return versions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readSQLFromFile(version string) (string, error) {
|
||||||
|
filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, models.DbCfg.Type)
|
||||||
|
|
||||||
|
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(filename)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
gr, err := gzip.NewReader(file)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer gr.Close()
|
||||||
|
|
||||||
|
bytes, err := ioutil.ReadAll(gr)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(bytes), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreOldDB(t *testing.T, version string) bool {
|
||||||
|
data, err := readSQLFromFile(version)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
if len(data) == 0 {
|
||||||
|
log.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case setting.UseSQLite3:
|
||||||
|
os.Remove(models.DbCfg.Path)
|
||||||
|
err := os.MkdirAll(path.Dir(models.DbCfg.Path), os.ModePerm)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", models.DbCfg.Path, models.DbCfg.Timeout))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_, err = db.Exec(data)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
db.Close()
|
||||||
|
|
||||||
|
case setting.UseMySQL:
|
||||||
|
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
|
||||||
|
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", models.DbCfg.Name))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true",
|
||||||
|
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_, err = db.Exec(data)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
db.Close()
|
||||||
|
|
||||||
|
case setting.UsePostgreSQL:
|
||||||
|
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
|
||||||
|
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", models.DbCfg.Name))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
db.Close()
|
||||||
|
|
||||||
|
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
|
||||||
|
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name, models.DbCfg.SSLMode))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_, err = db.Exec(data)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
db.Close()
|
||||||
|
|
||||||
|
case setting.UseMSSQL:
|
||||||
|
host, port := models.ParseMSSQLHostPort(models.DbCfg.Host)
|
||||||
|
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
|
||||||
|
host, port, "master", models.DbCfg.User, models.DbCfg.Passwd))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_, err = db.Exec("DROP DATABASE IF EXISTS gitea")
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = db.Exec("CREATE DATABASE gitea")
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = db.Exec(data)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
db.Close()
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func wrappedMigrate(x *xorm.Engine) error {
|
||||||
|
currentEngine = x
|
||||||
|
return migrations.Migrate(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
func doMigrationTest(t *testing.T, version string) {
|
||||||
|
log.Printf("Performing migration test for %s version: %s", models.DbCfg.Type, version)
|
||||||
|
if !restoreOldDB(t, version) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
setting.NewXORMLogService(false)
|
||||||
|
err := models.SetEngine()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
err = models.NewEngine(wrappedMigrate)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
currentEngine.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMigrations(t *testing.T) {
|
||||||
|
initMigrationTest()
|
||||||
|
|
||||||
|
dialect := models.DbCfg.Type
|
||||||
|
versions, err := availableVersions()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
if len(versions) == 0 {
|
||||||
|
log.Printf("No old database versions available to migration test for %s\n", dialect)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Preparing to test %d migrations for %s\n", len(versions), dialect)
|
||||||
|
for _, version := range versions {
|
||||||
|
doMigrationTest(t, version)
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue