Improve install code to avoid low-level mistakes.

cmd/cmd.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -56,16 +57,22 @@ func confirm() (bool, error) {
 	}
 }
 
-func initDB(ctx context.Context) error {
-	return initDBDisableConsole(ctx, false)
+func ensureInstallLock() {
+	if !setting.InstallLock {
+		log.Fatal("invalid installation settings in app.ini (%s), please use the correctly installed config file", setting.CustomConf)
+	}
 }
 
-func initDBDisableConsole(ctx context.Context, disableConsole bool) error {
+func initDB(ctx context.Context) error {
 	setting.NewContext()
 	setting.InitDBConfig()
-	setting.NewXORMLogService(disableConsole)
+	setting.NewXORMLogService(false)
+
+	if setting.Database.Type == "" {
+		return fmt.Errorf("invalid database settings in app.ini (%s), please use the correctly installed config file", setting.CustomConf)
+	}
 	if err := db.InitEngine(ctx); err != nil {
-		return fmt.Errorf("models.SetEngine: %v", err)
+		return fmt.Errorf("cmd.initDB: %v", err)
 	}
 	return nil
 }

cmd/convert.go

@@ -35,7 +35,6 @@ func runConvert(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	if !setting.Database.UseMySQL {
 		fmt.Println("This command can only be used with a MySQL database")

cmd/doctor.go

@@ -82,6 +82,8 @@ You should back-up your database before doing this and ensure that your database
 }
 
 func runRecreateTable(ctx *cli.Context) error {
+	ensureInstallLock()
+
 	// Redirect the default golog to here
 	golog.SetFlags(0)
 	golog.SetPrefix("")

cmd/dump_repo.go

@@ -88,7 +88,6 @@ func runDumpRepository(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	var (
 		serviceType structs.GitServiceType

cmd/migrate.go

@@ -36,7 +36,6 @@ func runMigrate(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
 		log.Fatal("Failed to initialize ORM engine: %v", err)

cmd/migrate_storage.go

@@ -120,7 +120,6 @@ func runMigrateStorage(ctx *cli.Context) error {
 	log.Info("Custom path: %s", setting.CustomPath)
 	log.Info("Log path: %s", setting.LogRootPath)
 	log.Info("Configuration file: %s", setting.CustomConf)
-	setting.InitDBConfig()
 
 	if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
 		log.Fatal("Failed to initialize ORM engine: %v", err)

models/db/engine.go

@@ -8,7 +8,6 @@ package db
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"io"
 	"reflect"
@@ -92,8 +91,8 @@ func init() {
 	}
 }
 
-// NewEngine returns a new xorm engine from the configuration
-func NewEngine() (*xorm.Engine, error) {
+// newXORMEngine returns a new XORM engine from the configuration
+func newXORMEngine() (*xorm.Engine, error) {
 	connStr, err := setting.DBConnStr()
 	if err != nil {
 		return nil, err
@@ -126,40 +125,49 @@ func SyncAllTables() error {
 	return x.StoreEngine("InnoDB").Sync2(tables...)
 }
 
-// InitEngine sets the xorm.Engine
-func InitEngine(ctx context.Context) (err error) {
-	x, err = NewEngine()
+// InitEngine initializes the xorm.Engine and set it as db.DefaultContext
+func InitEngine(ctx context.Context) error {
+	xormEngine, err := newXORMEngine()
 	if err != nil {
-		return fmt.Errorf("Failed to connect to database: %v", err)
+		return fmt.Errorf("failed to connect to database: %v", err)
 	}
 
-	x.SetMapper(names.GonicMapper{})
+	xormEngine.SetMapper(names.GonicMapper{})
 	// WARNING: for serv command, MUST remove the output to os.stdout,
 	// so use log file to instead print to stdout.
-	x.SetLogger(NewXORMLogger(setting.Database.LogSQL))
-	x.ShowSQL(setting.Database.LogSQL)
-	x.SetMaxOpenConns(setting.Database.MaxOpenConns)
-	x.SetMaxIdleConns(setting.Database.MaxIdleConns)
-	x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+	xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL))
+	xormEngine.ShowSQL(setting.Database.LogSQL)
+	xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns)
+	xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns)
+	xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+	xormEngine.SetDefaultContext(ctx)
+
+	SetDefaultEngine(ctx, xormEngine)
+	return nil
+}
 
+// SetDefaultEngine sets the default engine for db
+func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) {
+	x = eng
 	DefaultContext = &Context{
 		Context: ctx,
 		e:       x,
 	}
-	x.SetDefaultContext(ctx)
-	return nil
 }
 
-// SetEngine is used by unit test code
-func SetEngine(eng *xorm.Engine) {
-	x = eng
-	DefaultContext = &Context{
-		Context: context.Background(),
-		e:       x,
+// UnsetDefaultEngine closes and unsets the default engine
+// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now,
+// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `x` and DefaultContext without close
+// Global database engine related functions are all racy and there is no graceful close right now.
+func UnsetDefaultEngine() {
+	if x != nil {
+		_ = x.Close()
+		x = nil
 	}
+	DefaultContext = nil
 }
 
-// InitEngineWithMigration initializes a new xorm.Engine
+// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext
 // This function must never call .Sync2() if the provided migration function fails.
 // When called from the "doctor" command, the migration function is a version check
 // that prevents the doctor from fixing anything in the database if the migration level
@@ -226,14 +234,6 @@ func NamesToBean(names ...string) ([]interface{}, error) {
 	return beans, nil
 }
 
-// Ping tests if database is alive
-func Ping() error {
-	if x != nil {
-		return x.Ping()
-	}
-	return errors.New("database not configured")
-}
-
 // DumpDatabase dumps all data from database according the special database SQL syntax to file system.
 func DumpDatabase(filePath, dbType string) error {
 	var tbs []*schemas.Table
@@ -291,11 +291,3 @@ func GetMaxID(beanOrTableName interface{}) (maxID int64, err error) {
 	_, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID)
 	return
 }
-
-// FindByMaxID filled results as the condition from database
-func FindByMaxID(maxID int64, limit int, results interface{}) error {
-	return x.Where("id <= ?", maxID).
-		OrderBy("id DESC").
-		Limit(limit).
-		Find(results)
-}

models/db/install/db.go

@@ -0,0 +1,65 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package install
+
+import (
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func getXORMEngine() *xorm.Engine {
+	return db.DefaultContext.(*db.Context).Engine().(*xorm.Engine)
+}
+
+// CheckDatabaseConnection checks the database connection
+func CheckDatabaseConnection() error {
+	e := db.GetEngine(db.DefaultContext)
+	_, err := e.Exec("SELECT 1")
+	return err
+}
+
+// GetMigrationVersion gets the database migration version
+func GetMigrationVersion() (int64, error) {
+	var installedDbVersion int64
+	x := getXORMEngine()
+	exist, err := x.IsTableExist("version")
+	if err != nil {
+		return 0, err
+	}
+	if !exist {
+		return 0, nil
+	}
+	_, err = x.Table("version").Cols("`version`").Get(&installedDbVersion)
+	if err != nil {
+		return 0, err
+	}
+	return installedDbVersion, nil
+}
+
+// HasPostInstallationUsers checks whether there are users after installation
+func HasPostInstallationUsers() (bool, error) {
+	x := getXORMEngine()
+	exist, err := x.IsTableExist("user")
+	if err != nil {
+		return false, err
+	}
+	if !exist {
+		return false, nil
+	}
+
+	// if there are 2 or more users in database, we consider there are users created after installation
+	threshold := 2
+	if !setting.IsProd {
+		// to debug easily, with non-prod RUN_MODE, we only check the count to 1
+		threshold = 1
+	}
+	res, err := x.Table("user").Cols("id").Limit(threshold).Query()
+	if err != nil {
+		return false, err
+	}
+	return len(res) >= threshold, nil
+}

models/migrations/migrations_test.go

@@ -5,6 +5,7 @@
 package migrations
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"os"
@@ -85,21 +86,11 @@ func removeAllWithRetry(dir string) error {
 	return err
 }
 
-// newEngine sets the xorm.Engine
-func newEngine() (*xorm.Engine, error) {
-	x, err := db.NewEngine()
-	if err != nil {
-		return x, fmt.Errorf("Failed to connect to database: %v", err)
+func newXORMEngine() (*xorm.Engine, error) {
+	if err := db.InitEngine(context.Background()); err != nil {
+		return nil, err
 	}
-
-	x.SetMapper(names.GonicMapper{})
-	// WARNING: for serv command, MUST remove the output to os.stdout,
-	// so use log file to instead print to stdout.
-	x.SetLogger(db.NewXORMLogger(setting.Database.LogSQL))
-	x.ShowSQL(setting.Database.LogSQL)
-	x.SetMaxOpenConns(setting.Database.MaxOpenConns)
-	x.SetMaxIdleConns(setting.Database.MaxIdleConns)
-	x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+	x := unittest.GetXORMEngine()
 	return x, nil
 }
 
@@ -213,7 +204,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 		return nil, deferFn
 	}
 
-	x, err := newEngine()
+	x, err := newXORMEngine()
 	assert.NoError(t, err)
 	if x != nil {
 		oldDefer := deferFn

models/unittest/fixtures.go

@@ -18,7 +18,8 @@ import (
 
 var fixtures *testfixtures.Loader
 
-func getXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
+// GetXORMEngine gets the XORM engine
+func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
 	if len(engine) == 1 {
 		return engine[0]
 	}
@@ -27,7 +28,7 @@ func getXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
 
 // InitFixtures initialize test fixtures for a test database
 func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
-	e := getXORMEngine(engine...)
+	e := GetXORMEngine(engine...)
 	var testfiles func(*testfixtures.Loader) error
 	if opts.Dir != "" {
 		testfiles = testfixtures.Directory(opts.Dir)
@@ -69,7 +70,7 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 
 // LoadFixtures load fixtures for a test database
 func LoadFixtures(engine ...*xorm.Engine) error {
-	e := getXORMEngine(engine...)
+	e := GetXORMEngine(engine...)
 	var err error
 	// Database transaction conflicts could occur and result in ROLLBACK
 	// As a simple workaround, we just retry 20 times.

models/unittest/testdb.go

@@ -5,6 +5,7 @@
 package unittest
 
 import (
+	"context"
 	"fmt"
 	"net/url"
 	"os"
@@ -124,7 +125,7 @@ func CreateTestEngine(opts FixturesOptions) error {
 		return err
 	}
 	x.SetMapper(names.GonicMapper{})
-	db.SetEngine(x)
+	db.SetDefaultEngine(context.Background(), x)
 
 	if err = db.SyncAllTables(); err != nil {
 		return err

modules/private/internal.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"log"
 	"net"
 	"net/http"
 
@@ -17,6 +18,9 @@ import (
 )
 
 func newRequest(ctx context.Context, url, method string) *httplib.Request {
+	if setting.InternalToken == "" {
+		log.Fatal("no internal token, can not send internal request to server. please use correct app.ini")
+	}
 	return httplib.NewRequest(url, method).
 		SetContext(ctx).
 		Header("Authorization",
@@ -44,9 +48,6 @@ func newInternalRequest(ctx context.Context, url, method string) *httplib.Reques
 	})
 	if setting.Protocol == setting.UnixSocket {
 		req.SetTransport(&http.Transport{
-			Dial: func(_, _ string) (net.Conn, error) {
-				return net.Dial("unix", setting.HTTPAddr)
-			},
 			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
 				var d net.Dialer
 				return d.DialContext(ctx, "unix", setting.HTTPAddr)