codeintel: RFC 235: Migrate SQLite data to Postgres (#13932)

enterprise/cmd/precise-code-intel-bundle-manager/internal/readers/migrate.go

@@ -2,8 +2,11 @@ package readers
 
 import (
 	"context"
+	"database/sql"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"sync"
@@ -12,18 +15,32 @@ import (
 	"github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-bundle-manager/internal/paths"
 	"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/bundles/persistence"
 	"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/bundles/persistence/cache"
+	"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/bundles/persistence/postgres"
 	"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/bundles/persistence/sqlite/migrate"
+	"github.com/sourcegraph/sourcegraph/internal/db/basestore"
 )
 
 // NumMigrateRoutines is the number of goroutines launched to migrate bundle files.
-var NumMigrateRoutines = 1 // runtime.NumCPU() * 2
+var NumMigrateRoutines = 1
 
-// Migrate runs through each SQLite database on disk and opens a store instance which will perform
+func Migrate(bundleDir string, storeCache cache.StoreCache, db *sql.DB) error {
+	if err := migrateVersions(bundleDir, storeCache); err != nil {
+		return fmt.Errorf("failed to migrate SQLite bundle versions: %s", err.Error())
+	}
+
+	if err := migrateToPostgres(bundleDir, storeCache, db); err != nil {
+		return fmt.Errorf("failed to migrate SQLite bundles to Postgres: %s", err.Error())
+	}
+
+	return nil
+}
+
+// migrateVersions runs through each SQLite database on disk and opens a store instance which will perform
 // any necessary migrations to transform it to the newest schema. Because this may have a non-negligible
 // cost cost some intersection of migrations and database size, we try to pay this cost up-front instead
 // of being paid on-demand when the database is opened within the query path. This method does not block
 // the startup of the bundle manager as it does not change the correctness of the service.
-func Migrate(bundleDir string, storeCache cache.StoreCache) error {
+func migrateVersions(bundleDir string, storeCache cache.StoreCache) error {
 	version := migrate.CurrentSchemaVersion
 	migrationMarkerFilename := paths.MigrationMarkerFilename(bundleDir, version)
 
@@ -136,3 +153,59 @@ func touchFile(filename string) {
 func noopHandler(store persistence.Store) error {
 	return nil
 }
+
+func migrateToPostgres(bundleDir string, storeCache cache.StoreCache, db *sql.DB) error {
+	paths, err := sqlitePaths(bundleDir)
+	if err != nil {
+		return err
+	}
+	if len(paths) == 0 {
+		return nil
+	}
+	bundleFilenames := paths
+
+	ctx := context.Background()
+
+	bundleIDs, err := basestore.ScanInts(db.QueryContext(ctx, `SELECT dump_id FROM lsif_data_metadata`))
+	if err != nil {
+		return err
+	}
+
+	bundleIDMap := map[int]struct{}{}
+	for _, bundleID := range bundleIDs {
+		bundleIDMap[bundleID] = struct{}{}
+	}
+
+	updateIDs := map[string]int{}
+	for _, filename := range bundleFilenames {
+		bundleID, err := strconv.Atoi(filepath.Base(filepath.Dir(filename)))
+		if err != nil {
+			log15.Error("Failed to extract bundle id from filename", "err", err, "filename", filename)
+			continue
+		}
+
+		if _, ok := bundleIDMap[bundleID]; ok {
+			continue
+		}
+
+		updateIDs[filename] = bundleID
+	}
+
+	if len(updateIDs) == 0 {
+		return nil
+	}
+
+	log15.Info(
+		"Migrating bundle data to Postgres in background",
+		"numBundles", len(updateIDs),
+	)
+
+	for filename, bundleID := range updateIDs {
+		if err := postgres.MigrateBundleToPostgres(ctx, bundleID, filename, db); err != nil {
+			log15.Error("Failed to migrate bundle", "err", err, "filename", filename)
+		}
+	}
+
+	log15.Info("Finished migration to Postgres")
+	return nil
+}

enterprise/cmd/precise-code-intel-bundle-manager/main.go

@@ -62,7 +62,7 @@ func main() {
 		log.Fatalf("failed to migrate paths: %s", err)
 	}
 
-	if err := readers.Migrate(bundleDir, storeCache); err != nil {
+	if err := readers.Migrate(bundleDir, storeCache, codeIntelDB); err != nil {
 		log.Fatalf("failed to migrate readers: %s", err)
 	}