Add command to recreate tables (#12407)

Provides new command: `gitea doctor recreate-table` which will recreate
db tables and copy the old data in to the new table.

This function can be used to remove the old warning of struct defaults being
out of date.

Fix #8868
Fix #3265
Fix #8894

Signed-off-by: Andrew Thornton <art27@cantab.net>

cmd/doctor.go

@@ -26,6 +26,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 	"xorm.io/builder"
+	"xorm.io/xorm"
 
 	"github.com/urfave/cli"
 )
@@ -62,6 +63,27 @@ var CmdDoctor = cli.Command{
 			Usage: `Name of the log file (default: "doctor.log"). Set to "-" to output to stdout, set to "" to disable`,
 		},
 	},
+	Subcommands: []cli.Command{
+		cmdRecreateTable,
+	},
+}
+
+var cmdRecreateTable = cli.Command{
+	Name:      "recreate-table",
+	Usage:     "Recreate tables from XORM definitions and copy the data.",
+	ArgsUsage: "[TABLE]... : (TABLEs to recreate - leave blank for all)",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "debug",
+			Usage: "Print SQL commands sent",
+		},
+	},
+	Description: `The database definitions Gitea uses change across versions, sometimes changing default values and leaving old unused columns.
+
+This command will cause Xorm to recreate tables, copying over the data and deleting the old table.
+
+You should back-up your database before doing this and ensure that your database is up-to-date first.`,
+	Action: runRecreateTable,
 }
 
 type check struct {
@@ -136,6 +158,47 @@ var checklist = []check{
 	// more checks please append here
 }
 
+func runRecreateTable(ctx *cli.Context) error {
+	// Redirect the default golog to here
+	golog.SetFlags(0)
+	golog.SetPrefix("")
+	golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
+
+	setting.NewContext()
+	setting.InitDBConfig()
+
+	setting.EnableXORMLog = ctx.Bool("debug")
+	setting.Database.LogSQL = ctx.Bool("debug")
+	setting.Cfg.Section("log").Key("XORM").SetValue(",")
+
+	setting.NewXORMLogService(!ctx.Bool("debug"))
+	if err := models.SetEngine(); err != nil {
+		fmt.Println(err)
+		fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.")
+		return nil
+	}
+
+	args := ctx.Args()
+	names := make([]string, 0, ctx.NArg())
+	for i := 0; i < ctx.NArg(); i++ {
+		names = append(names, args.Get(i))
+	}
+
+	beans, err := models.NamesToBean(names...)
+	if err != nil {
+		return err
+	}
+	recreateTables := migrations.RecreateTables(beans...)
+
+	return models.NewEngine(context.Background(), func(x *xorm.Engine) error {
+		if err := migrations.EnsureUpToDate(x); err != nil {
+			return err
+		}
+		return recreateTables(x)
+	})
+
+}
+
 func runDoctor(ctx *cli.Context) error {
 
 	// Silence the default loggers

docs/content/doc/usage/command-line.en-us.md

@@ -319,6 +319,36 @@ var checklist = []check{
 
 This function will receive a command line context and return a list of details about the problems or error.
 
+##### doctor recreate-table
+
+Sometimes when there are migrations the old columns and default values may be left
+unchanged in the database schema. This may lead to warning such as:
+
+```
+2020/08/02 11:32:29 ...rm/session_schema.go:360:Sync2() [W] Table user Column keep_activity_private db default is , struct default is 0
+```
+
+You can cause Gitea to recreate these tables and copy the old data into the new table
+with the defaults set appropriately by using:
+
+```
+gitea doctor recreate-table user
+```
+
+You can ask gitea to recreate multiple tables using:
+
+```
+gitea doctor recreate-table table1 table2 ...
+```
+
+And if you would like Gitea to recreate all tables simply call:
+
+```
+gitea doctor recreate-table
+```
+
+It is highly recommended to back-up your database before running these commands.
+
 #### manager
 
 Manage running server operations:

integrations/migration-test/migration_test.go

@@ -258,6 +258,30 @@ func doMigrationTest(t *testing.T, version string) {
 	err = models.NewEngine(context.Background(), wrappedMigrate)
 	assert.NoError(t, err)
 	currentEngine.Close()
+
+	err = models.SetEngine()
+	assert.NoError(t, err)
+
+	beans, _ := models.NamesToBean()
+
+	err = models.NewEngine(context.Background(), func(x *xorm.Engine) error {
+		currentEngine = x
+		return migrations.RecreateTables(beans...)(x)
+	})
+	assert.NoError(t, err)
+	currentEngine.Close()
+
+	// We do this a second time to ensure that there is not a problem with retained indices
+	err = models.SetEngine()
+	assert.NoError(t, err)
+
+	err = models.NewEngine(context.Background(), func(x *xorm.Engine) error {
+		currentEngine = x
+		return migrations.RecreateTables(beans...)(x)
+	})
+	assert.NoError(t, err)
+
+	currentEngine.Close()
 }
 
 func TestMigrations(t *testing.T) {

models/migrations/migrations.go

@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"fmt"
+	"reflect"
 	"regexp"
 	"strings"
 
@@ -327,6 +328,221 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t
 	return nil
 }
 
+// RecreateTables will recreate the tables for the provided beans using the newly provided bean definition and move all data to that new table
+// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
+func RecreateTables(beans ...interface{}) func(*xorm.Engine) error {
+	return func(x *xorm.Engine) error {
+		sess := x.NewSession()
+		defer sess.Close()
+		if err := sess.Begin(); err != nil {
+			return err
+		}
+		sess = sess.StoreEngine("InnoDB")
+		for _, bean := range beans {
+			log.Info("Recreating Table: %s for Bean: %s", x.TableName(bean), reflect.Indirect(reflect.ValueOf(bean)).Type().Name())
+			if err := recreateTable(sess, bean); err != nil {
+				return err
+			}
+		}
+		return sess.Commit()
+	}
+}
+
+// recreateTable will recreate the table using the newly provided bean definition and move all data to that new table
+// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
+// WARNING: YOU MUST COMMIT THE SESSION AT THE END
+func recreateTable(sess *xorm.Session, bean interface{}) error {
+	// TODO: This will not work if there are foreign keys
+
+	tableName := sess.Engine().TableName(bean)
+	tempTableName := fmt.Sprintf("tmp_recreate__%s", tableName)
+
+	// We need to move the old table away and create a new one with the correct columns
+	// We will need to do this in stages to prevent data loss
+	//
+	// First create the temporary table
+	if err := sess.Table(tempTableName).CreateTable(bean); err != nil {
+		log.Error("Unable to create table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	if err := sess.Table(tempTableName).CreateUniques(bean); err != nil {
+		log.Error("Unable to create uniques for table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	if err := sess.Table(tempTableName).CreateIndexes(bean); err != nil {
+		log.Error("Unable to create indexes for table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	// Work out the column names from the bean - these are the columns to select from the old table and install into the new table
+	table, err := sess.Engine().TableInfo(bean)
+	if err != nil {
+		log.Error("Unable to get table info. Error: %v", err)
+
+		return err
+	}
+	newTableColumns := table.Columns()
+	if len(newTableColumns) == 0 {
+		return fmt.Errorf("no columns in new table")
+	}
+	hasID := false
+	for _, column := range newTableColumns {
+		hasID = hasID || (column.IsPrimaryKey && column.IsAutoIncrement)
+	}
+
+	if hasID && setting.Database.UseMSSQL {
+		if _, err := sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT `%s` ON", tempTableName)); err != nil {
+			log.Error("Unable to set identity insert for table %s. Error: %v", tempTableName, err)
+			return err
+		}
+	}
+
+	sqlStringBuilder := &strings.Builder{}
+	_, _ = sqlStringBuilder.WriteString("INSERT INTO `")
+	_, _ = sqlStringBuilder.WriteString(tempTableName)
+	_, _ = sqlStringBuilder.WriteString("` (`")
+	_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+	_, _ = sqlStringBuilder.WriteString("`")
+	for _, column := range newTableColumns[1:] {
+		_, _ = sqlStringBuilder.WriteString(", `")
+		_, _ = sqlStringBuilder.WriteString(column.Name)
+		_, _ = sqlStringBuilder.WriteString("`")
+	}
+	_, _ = sqlStringBuilder.WriteString(")")
+	_, _ = sqlStringBuilder.WriteString(" SELECT ")
+	if newTableColumns[0].Default != "" {
+		_, _ = sqlStringBuilder.WriteString("COALESCE(`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`, ")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Default)
+		_, _ = sqlStringBuilder.WriteString(")")
+	} else {
+		_, _ = sqlStringBuilder.WriteString("`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`")
+	}
+
+	for _, column := range newTableColumns[1:] {
+		if column.Default != "" {
+			_, _ = sqlStringBuilder.WriteString(", COALESCE(`")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`, ")
+			_, _ = sqlStringBuilder.WriteString(column.Default)
+			_, _ = sqlStringBuilder.WriteString(")")
+		} else {
+			_, _ = sqlStringBuilder.WriteString(", `")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`")
+		}
+	}
+	_, _ = sqlStringBuilder.WriteString(" FROM `")
+	_, _ = sqlStringBuilder.WriteString(tableName)
+	_, _ = sqlStringBuilder.WriteString("`")
+
+	if _, err := sess.Exec(sqlStringBuilder.String()); err != nil {
+		log.Error("Unable to set copy data in to temp table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	if hasID && setting.Database.UseMSSQL {
+		if _, err := sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT `%s` OFF", tempTableName)); err != nil {
+			log.Error("Unable to switch off identity insert for table %s. Error: %v", tempTableName, err)
+			return err
+		}
+	}
+
+	switch {
+	case setting.Database.UseSQLite3:
+		// SQLite will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		if err := sess.Table(tempTableName).DropIndexes(bean); err != nil {
+			log.Error("Unable to drop indexes on temporary table %s. Error: %v", tempTableName, err)
+			return err
+		}
+
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+
+		if err := sess.Table(tableName).CreateIndexes(bean); err != nil {
+			log.Error("Unable to recreate indexes on table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		if err := sess.Table(tableName).CreateUniques(bean); err != nil {
+			log.Error("Unable to recreate uniques on table %s. Error: %v", tableName, err)
+			return err
+		}
+
+	case setting.Database.UseMySQL:
+		// MySQL will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		// SQLite and MySQL will move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+	case setting.Database.UsePostgreSQL:
+		// CASCADE causes postgres to drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		// CASCADE causes postgres to move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+
+		var indices []string
+		schema := sess.Engine().Dialect().URI().Schema
+		sess.Engine().SetSchema("")
+		if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+		sess.Engine().SetSchema(schema)
+
+		for _, index := range indices {
+			newIndexName := strings.Replace(index, "tmp_recreate__", "", 1)
+			if _, err := sess.Exec(fmt.Sprintf("ALTER INDEX `%s` RENAME TO `%s`", index, newIndexName)); err != nil {
+				log.Error("Unable to rename %s to %s. Error: %v", index, newIndexName, err)
+				return err
+			}
+		}
+
+	case setting.Database.UseMSSQL:
+		// MSSQL will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		// MSSQL sp_rename will move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("sp_rename `%s`,`%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+
+	default:
+		log.Fatal("Unrecognized DB")
+	}
+	return nil
+}
+
+// WARNING: YOU MUST COMMIT THE SESSION AT THE END
 func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...string) (err error) {
 	if tableName == "" || len(columnNames) == 0 {
 		return nil

models/models.go

@@ -10,6 +10,8 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"reflect"
+	"strings"
 
 	"code.gitea.io/gitea/modules/setting"
 
@@ -214,6 +216,36 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e
 	return nil
 }
 
+// NamesToBean return a list of beans or an error
+func NamesToBean(names ...string) ([]interface{}, error) {
+	beans := []interface{}{}
+	if len(names) == 0 {
+		beans = append(beans, tables...)
+		return beans, nil
+	}
+	// Need to map provided names to beans...
+	beanMap := make(map[string]interface{})
+	for _, bean := range tables {
+
+		beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean
+		beanMap[strings.ToLower(x.TableName(bean))] = bean
+		beanMap[strings.ToLower(x.TableName(bean, true))] = bean
+	}
+
+	gotBean := make(map[interface{}]bool)
+	for _, name := range names {
+		bean, ok := beanMap[strings.ToLower(strings.TrimSpace(name))]
+		if !ok {
+			return nil, fmt.Errorf("No table found that matches: %s", name)
+		}
+		if !gotBean[bean] {
+			beans = append(beans, bean)
+			gotBean[bean] = true
+		}
+	}
+	return beans, nil
+}
+
 // Statistic contains the database statistics
 type Statistic struct {
 	Counter struct {