Change migration to use new version tracking

Dockerfile

@@ -9,9 +9,8 @@ RUN apk update && apk add --no-cache git \
     && cd timescale-prometheus \
     && go mod download \
     && GIT_COMMIT=$(git rev-list -1 HEAD) \
-    && VERSION=$(git describe $GIT_COMMIT) \
     && CGO_ENABLED=0 go build -a \
-    --ldflags '-w' --ldflags "-X main.CommitHash=$GIT_COMMIT" --ldflags "-X main.Version=$VERSION" \
+    --ldflags '-w' --ldflags "-X main.CommitHash=$GIT_COMMIT" \
     -o /go/timescale-prometheus ./cmd/timescale-prometheus
 
 # Final image

cmd/timescale-prometheus/version_info.go

@@ -1,6 +1,20 @@
 package main
 
 var (
-	Version    = "0.1.0-beta.1"
+	// Rules for Versioning:
+	// A release version cannot contains `dev` in it's pre-release tag.
+	// The next development cycle MUST meet two requirements:
+	// 1) The development cycle version must be higher in semver than the release.
+	//    Thus you either need to increase the patch or pre-release tag. You
+	//    SHOULD always increase the version by the smallest amount possible
+	//    (e.g. prefer 0.1.2 to 0.2.0)
+	// 2) It must include `dev` as last part of prelease.
+	//
+	// Example if releasing 0.1.1, the next development cycle is 0.1.2-dev
+	// Example if releasing 0.1.3-beta the next development cycle is 0.1.3-beta.1.dev
+	//
+	// When introducing a new SQL migration script always increment the numeral after
+	// the `dev` tag. Add the SQL file corresponding to the /new/ version.
+	Version    = "0.1.0-beta.2-dev.0"
 	CommitHash = ""
 )

pkg/pgmodel/end_to_end_tests/migrate_test.go

@@ -85,16 +85,6 @@ func TestMigrationLib(t *testing.T) {
 				"2-toc-run_first.sql",
 				"1-toc-run_second.sql",
 			},
-			"versions/0.1.0": {
-				"1-migration.sql",
-			},
-			"versions/0.2.0": {
-				"1-migration.sql",
-			},
-			"versions/0.10.0": {
-				"2-toc_migration.sql",
-				"1-toc_migration.sql",
-			},
 		}
 
 		expected := []string{

pkg/pgmodel/migrate.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
@@ -34,7 +35,7 @@ const (
 	truncateMigrationsTable     = "TRUNCATE prom_schema_migrations"
 
 	preinstallScripts = "preinstall"
-	versionScripts    = "versions"
+	versionScripts    = "versions/dev"
 	idempotentScripts = "idempotent"
 )
 
@@ -47,7 +48,8 @@ var (
 			"matcher-functions.sql",
 		},
 	}
-	migrateMutex = &sync.Mutex{}
+	migrateMutex            = &sync.Mutex{}
+	migrationFileNameRegexp = regexp.MustCompile(`([[:digit:]]*)-[[:word:]]*.sql`)
 )
 
 type VersionInfo struct {
@@ -210,6 +212,22 @@ func getDBVersion(db *pgxpool.Pool) (semver.Version, error) {
 	return version, nil
 }
 
+func (t *Migrator) execMigrationFile(tx pgx.Tx, fileName string) error {
+	f, err := t.sqlFiles.Open(fileName)
+	if err != nil {
+		return fmt.Errorf("unable to get migration script: name %s, err %w", fileName, err)
+	}
+	contents, err := replaceSchemaNames(f)
+	if err != nil {
+		return fmt.Errorf("unable to read migration script: name %s, err %w", fileName, err)
+	}
+	_, err = tx.Exec(context.Background(), string(contents))
+	if err != nil {
+		return fmt.Errorf("error executing migration script: name %s, err %w", fileName, err)
+	}
+	return nil
+}
+
 // execMigrationFiles finds all the migration files in a directory, orders them
 // (either by ToC or by their numerical prefix) and executes them in a transaction.
 func (t *Migrator) execMigrationFiles(tx pgx.Tx, dirName string) error {
@@ -258,17 +276,9 @@ func (t *Migrator) execMigrationFiles(tx pgx.Tx, dirName string) error {
 
 	for _, e := range entries {
 		fileName := filepath.Join(dirName, e)
-		f, err := t.sqlFiles.Open(fileName)
+		err := t.execMigrationFile(tx, fileName)
 		if err != nil {
-			return fmt.Errorf("unable to get migration script: name %s, err %w", fileName, err)
-		}
-		contents, err := replaceSchemaNames(f)
-		if err != nil {
-			return fmt.Errorf("unable to read migration script: name %s, err %w", fileName, err)
-		}
-		_, err = tx.Exec(context.Background(), string(contents))
-		if err != nil {
-			return fmt.Errorf("error executing migration script: name %s, err %w", fileName, err)
+			return err
 		}
 	}
 
@@ -326,42 +336,88 @@ func replaceSchemaNames(r io.ReadCloser) (string, error) {
 	return s, err
 }
 
+//A migration file is inside a directory that is a semver version number. The filename itself has the format
+//<migration file number)-<description>.sql. That file correspond to the semver of <dirname>-dev.<migration file number>
+//All app versions >= to that semver will include the migration file
+func (t *Migrator) getMigrationFileVersion(dirName string, fileName string) (*semver.Version, error) {
+	var migrationFileNumber int
+	matches := migrationFileNameRegexp.FindStringSubmatch(fileName)
+	if len(matches) < 2 {
+		return nil, fmt.Errorf("unable to parse the migration file name %v", fileName)
+	}
+	n, err := fmt.Sscanf(matches[1], "%d", &migrationFileNumber)
+	if n != 1 || err != nil {
+		return nil, fmt.Errorf("unable to parse the migration file name %v: %w", fileName, err)
+	}
+
+	migrationFileVersion, err := semver.Make(dirName)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse version from directory %v: %w", dirName, err)
+	}
+	devPrVersion, err := semver.NewPRVersion("dev")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create dev PR version: %w", err)
+	}
+	migrationNumberPrVersion, err := semver.NewPRVersion(fmt.Sprintf("%d", migrationFileNumber))
+	if err != nil {
+		return nil, fmt.Errorf("unable to create dev PR version: %w", err)
+	}
+
+	migrationFileVersion.Pre = append(migrationFileVersion.Pre, devPrVersion, migrationNumberPrVersion)
+	return &migrationFileVersion, nil
+}
+
 // upgradeVersion finds all the versions between `from` and `to`, sorts them
 // using semantic version ordering and applies them sequentially in the supplied transaction.
 func (t *Migrator) upgradeVersion(tx pgx.Tx, from, to semver.Version) error {
-	f, err := t.sqlFiles.Open(versionScripts)
+	devDirFile, err := t.sqlFiles.Open(versionScripts)
 	if err != nil {
 		return fmt.Errorf("unable to open migration scripts: %w", err)
 	}
 
-	entries, err := f.Readdir(-1)
+	versionDirInfoEntries, err := devDirFile.Readdir(-1)
 	if err != nil {
 		return fmt.Errorf("unable to get migration scripts: %w", err)
 	}
 
-	versions := make(semver.Versions, 0, len(entries))
+	versions := make(semver.Versions, 0)
+	versionMap := make(map[string]string)
 
-	for _, e := range entries {
-		version, err := semver.Make(e.Name())
+	for _, versionDirInfo := range versionDirInfoEntries {
+		if !versionDirInfo.IsDir() {
+			return fmt.Errorf("Not a directory inside %v: %v", versionScripts, versionDirInfo.Name())
+		}
 
-		// Ignoring malformated entries.
+		versionDirFile, err := t.sqlFiles.Open(versionScripts + "/" + versionDirInfo.Name())
 		if err != nil {
-			log.Warn("msg", "Ignoring malformed dir name in migration version directories (not semver)", "dirname", e.Name(), "err", err.Error())
-			continue
+			return fmt.Errorf("unable to open migration scripts: %w", err)
 		}
 
-		versions = append(versions, version)
+		migrationFileInfoEntries, err := versionDirFile.Readdir(-1)
+		for _, migrationFileInfo := range migrationFileInfoEntries {
+			migrationFileVersion, err := t.getMigrationFileVersion(versionDirInfo.Name(), migrationFileInfo.Name())
+			if err != nil {
+				return err
+			}
+			path := versionScripts + "/" + versionDirInfo.Name() + "/" + migrationFileInfo.Name()
+
+			_, existing := versionMap[migrationFileVersion.String()]
+			if existing {
+				return fmt.Errorf("Found two migration files with the same version: %v", migrationFileVersion.String())
+			}
+			versionMap[migrationFileVersion.String()] = path
+			versions = append(versions, *migrationFileVersion)
+		}
 	}
 
 	sort.Sort(versions)
 
 	for _, v := range versions {
-		dirname := fmt.Sprintf("%s", v.String())
-
-		//When comparing to the latest version use the post-<current> as well since at least
-		//for now we don't bump versions post-release (pbbl should)
+		//When comparing to the latest version use >= (INCLUSIVE). A migration file
+		//that's marked as version X is part of that version.
 		if from.Compare(v) < 0 && to.Compare(v) >= 0 {
-			if err = t.execMigrationFiles(tx, filepath.Join("versions", dirname)); err != nil {
+			filename := versionMap[v.String()]
+			if err = t.execMigrationFile(tx, filename); err != nil {
 				return err
 			}
 		}

pkg/pgmodel/migrations/Readme.md

@@ -4,19 +4,23 @@ This directory contains sql scripts for schema setup and upgrading.
 
 All scripts are contained in the `sql` directory and are separated as follows:
 
-1. `preinstall` - This directory contains all scripts that will be executed on 
+1. `preinstall` - This directory contains all scripts that will be executed on
     a new database install.
-2. `idempotent` - This directory contains all scripts that contain idempotent 
+2. `idempotent` - This directory contains all scripts that contain idempotent
     content which is executed after a fresh install or a version upgrade.
-3. `versions` - This directory contains subdirectories that are named after 
-    the app version they were introduced in. Their names follow the semantic versioning syntax.
+3. `versions/dev` - This directory contains subdirectories that are named after
+    the development version they were introduced in. A version's migrations cannot
+    be modified. So, to introduce a new version, you have to bump the app version.
+    For example, if the current app version is 0.1.1-dev, to introduce a new migration
+    script, you must add a sql file name `versions/dev/0.1.1/1-blah.sql` and bump
+    the app version to 0.1.1-dev.1.
 
 All script files are executed in a explicit order. Ordering can happen in two ways:
 
-- Using a table of contents which needs to be present in the `pkg/pgmodel/migrate.go` 
+- Using a table of contents which needs to be present in the `pkg/pgmodel/migrate.go`
     file. Files not present in the ToC will be ignored.
-- Using a numbered prefix delimited by a `-` e.g. `1-base.sql`, `2-secondary.sql`. 
-    In this case, fils are ordered by the numeric prefix, low to high. 
+- Using a numbered prefix delimited by a `-` e.g. `1-base.sql`, `2-secondary.sql`.
+    In this case, fils are ordered by the numeric prefix, low to high.
     Files that do not follow this format will be ignored.
 
 **NOTE** If any changes are made to the `sql` directory, you must rerun

pkg/pgmodel/test_migrations/sql/versions/dev/0.10.0/1-migr_98_at.sql

@@ -0,0 +1 @@
+INSERT INTO log VALUES('migration 0.10.0=1');

pkg/pgmodel/test_migrations/sql/versions/dev/0.10.0/2-1_mig.sql

@@ -0,0 +1 @@
+INSERT INTO log VALUES('migration 0.10.0=2');