chore(migration): migration generic store sample implementation [WIP DO_NOT_MERGE]

Makefile

@@ -775,4 +775,4 @@ mitre:
 
 .PHONY: bootstrap_migration
 bootstrap_migration:
-	$(SILENT)if [[ "x${DESCRIPTION}" == "x" ]]; then echo "Please set a description for your migration in the DESCRIPTION environment variable"; else go run tools/generate-helpers/bootstrap-migration/main.go --root . --description "${DESCRIPTION}" ;fi
+	$(SILENT)if [[ "x${DESCRIPTION}" == "x" ]]; then echo "Please set a description for your migration in the DESCRIPTION environment variable"; else go run tools/generate-helpers/bootstrap-migration/main.go --root . --description "${DESCRIPTION}" --storeObject "${STORE_OBJECT}" ;fi

migrator/README.md

@@ -44,22 +44,27 @@ A migration can read from any of the databases, make changes to the data or to t
 Script should correspond to single change. Script should be part of the same release as this change.
 Here are the steps to write migration script:
 
-1. Run `DESCRIPTION="xxx" make bootstrap_migration` with a proper description of what the migration will do
+1. Run `DESCRIPTION="xxx" STORE_OBJECT="storage.Object0,storage.Object1" make bootstrap_migration` with a proper description of what the migration will do
    in the `DESCRIPTION` environment variable.
+and OPTIONALLY with the objects being migrated to create `gen.go` files for each object being migrated
+    in the optional `STORE_OBJECT` environment variable.
 
-2. Determine if this change breaks a previous releases database.  If so increment the `MinimumSupportedDBVersionSeqNum` 
-   to the `CurrentDBVersionSeqNum` of the release immediately following the release that cannot tolerate the change. 
-   For example, in 4.2 a column `column_v2` is added to replace the `column_v1` column in 4.1.  All the code from 4.2
+2. If using the optional `STORE_OBJECT` parameter, generate the migration stores with the following command:
+    `gogen -run pg-table-bindings`
+
+3. Determine if this change breaks a previous releases database.  If so increment the `MinimumSupportedDBVersionSeqNum` to
+   the `CurrentDBVersionSeqNum` of the release immediately following the release that cannot tolerate the change. For
+   example, in 4.2 a column `column_v2` is added to replace the `column_v1` column in 4.1.  All the code from 4.2
    onward will not reference `column_v1`.  At some point in the future a rollback to 4.1 will not longer be supported
    and we want to remove `column_v1`.  To do so, we will upgrade the schema to remove the column
    and update the `MinimumSupportedDBVersionSeqNum` to be the value of `CurrentDBVersionSeqNum` in 4.2
    as 4.1 will no longer be supported.  The migration process will inform the user of an error when trying to migrate
    to a software version that can no longer be supported by the database.
 
-3. Write the migration code and associated tests in the generated `migration_impl.go` and `migration_test.go` files.
+4. Write the migration code and associated tests in the generated `migration_impl.go` and `migration_test.go` files.
    The files contain a number of TODOs to help with the tasks to complete when writing the migration code itself.
 
-4. To better understand how to write the `migration.go` and `migration_test.go` files, look at existing examples
+5. To better understand how to write the `migration.go` and `migration_test.go` files, look at existing examples
 in `migrations` directory, or at the examples listed below.
 
     - [#1](https://github.com/stackrox/rox/pull/8609)
@@ -194,11 +199,12 @@ In migrator, there are a multiple ways to access data.
    }
    ```
 
-3. Duplicate the Postgres Store
-   This method is used in version 73 and 74 to migrate all tables from RocksDB to Postgres. In addition to frozen schema,
-   the store to access the data are also frozen for migration. The migrations with this method are closely associated
-   with current release eg. search/delete with schema and the prototypes of the objects. This method is NOT recommended for
-   4.0 and beyond.
+3. Stores
+
+    Generate a version of the Postgres Store based off the migrator's Generic Store via the bootstrap process.
+    The migrations with this method are closely associated with current release eg. search/delete with schema
+    and the prototypes of the objects. The search schema and the prototypes of the objects must be backwards
+    compatible.  The advantage of this approach is it would be more familiar for the engineers.
 
 #### Conversion tool
 

migrator/migrations/m_187_to_m_188_add_nftables_to_policy/migration.go

@@ -0,0 +1,32 @@
+package m187tom188
+
+import (
+	"github.com/pkg/errors"
+	"github.com/stackrox/rox/generated/storage"
+	"github.com/stackrox/rox/migrator/migrations"
+	"github.com/stackrox/rox/migrator/types"
+	"github.com/stackrox/rox/pkg/postgres"
+)
+
+var (
+	migration = types.Migration{
+		StartingSeqNum: 187,
+		VersionAfter:   &storage.Version{SeqNum: 188},
+		Run: func(databases *types.Databases) error {
+			err := updatePolicies(databases.PostgresDB)
+			if err != nil {
+				return errors.Wrap(err, "updating policies")
+			}
+			return nil
+		},
+	}
+)
+
+func updatePolicies(_ postgres.DB) error {
+	// OBE for testing
+	return nil
+}
+
+func init() {
+	migrations.MustRegisterMigration(migration)
+}

migrator/migrations/m_188_to_m_189_test_generic_example/migration_impl.go

@@ -0,0 +1,98 @@
+package m188tom189
+
+import (
+	"embed"
+
+	policypostgresstore "github.com/stackrox/rox/migrator/migrations/m_188_to_m_189_test_generic_example/policy/store"
+	"github.com/stackrox/rox/migrator/migrations/policymigrationhelper"
+	"github.com/stackrox/rox/migrator/types"
+)
+
+var (
+	//go:embed policies_before_and_after
+	policyDiffFS embed.FS
+
+	// Add policy exclusions only if the existing name, description and policy sections haven't changed.
+	fieldsToCompareForExclusions = []policymigrationhelper.FieldComparator{
+		policymigrationhelper.NameComparator,
+		policymigrationhelper.DescriptionComparator,
+		policymigrationhelper.PolicySectionComparator,
+	}
+
+	// Update description only if the existing name, description and policy sections haven't changed.
+	fieldsToCompareForDescription = []policymigrationhelper.FieldComparator{
+		policymigrationhelper.NameComparator,
+		policymigrationhelper.PolicySectionComparator,
+	}
+
+	policyDiffs = []policymigrationhelper.PolicyDiff{
+		{
+			FieldsToCompare: fieldsToCompareForDescription,
+			PolicyFileName:  "add_instruction.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "apk.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "automount_service_account_token.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "dnf.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "exec-dnf.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "exec-remote-copy.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "host_network.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "no_resources_specified.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "pod_exec.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "privileged.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "restricted_host_ports.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "root_user.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "secret_env.json",
+		},
+		{
+			FieldsToCompare: fieldsToCompareForExclusions,
+			PolicyFileName:  "apt.json",
+		},
+	}
+)
+
+func migrate(databases *types.Databases) error {
+	policyStore := policypostgresstore.New(databases.PostgresDB)
+
+	return policymigrationhelper.MigratePoliciesWithDiffsAndStore(
+		policyDiffFS,
+		policyDiffs,
+		policyStore.Exists,
+		policyStore.Get,
+		policyStore.Upsert,
+	)
+}

migrator/migrations/m_188_to_m_189_test_generic_example/migration_test.go

@@ -0,0 +1,151 @@
+//go:build sql_integration
+
+package m188tom189
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stackrox/rox/generated/storage"
+	frozenSchema "github.com/stackrox/rox/migrator/migrations/frozenschema/v73"
+	policyPostgresStore "github.com/stackrox/rox/migrator/migrations/m_188_to_m_189_test_generic_example/policy/store"
+	pghelper "github.com/stackrox/rox/migrator/migrations/postgreshelper"
+	"github.com/stackrox/rox/migrator/types"
+	"github.com/stackrox/rox/pkg/fixtures"
+	"github.com/stackrox/rox/pkg/postgres/pgutils"
+	"github.com/stackrox/rox/pkg/sac"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+type policyMigrationTestSuite struct {
+	suite.Suite
+
+	db          *pghelper.TestPostgres
+	policyStore policyPostgresStore.Store
+
+	ctx context.Context
+}
+
+func TestMigration(t *testing.T) {
+	suite.Run(t, new(policyMigrationTestSuite))
+}
+
+func simplePolicy(policyID string) *storage.Policy {
+	return &storage.Policy{
+		Id:   policyID,
+		Name: fmt.Sprintf("Policy with id %s", policyID),
+	}
+}
+
+func (s *policyMigrationTestSuite) SetupTest() {
+	s.db = pghelper.ForT(s.T(), false)
+	s.policyStore = policyPostgresStore.New(s.db.DB)
+	pgutils.CreateTableFromModel(context.Background(), s.db.GetGormDB(), frozenSchema.CreateTablePoliciesStmt)
+
+	s.ctx = sac.WithAllAccess(context.Background())
+
+	// insert other un policies that won't be migrated in the db for migration to run successfully
+	policies := []*storage.Policy{
+		simplePolicy("880fd131-46f0-43d2-82c9-547f5aa7e043"),
+		simplePolicy("47cb9e0a-879a-417b-9a8f-de644d7c8a77"),
+		simplePolicy("6226d4ad-7619-4a0b-a160-46373cfcee66"),
+		simplePolicy("436811e7-892f-4da6-a0f5-8cc459f1b954"),
+		simplePolicy("742e0361-bddd-4a2d-8758-f2af6197f61d"),
+		simplePolicy("16c95922-08c4-41b6-a721-dc4b2a806632"),
+		simplePolicy("a9b9ecf7-9707-4e32-8b62-d03018ed454f"),
+		simplePolicy("32d770b9-c6ba-4398-b48a-0c3e807644ed"),
+	}
+
+	s.NoError(s.policyStore.UpsertMany(s.ctx, policies))
+}
+
+func (s *policyMigrationTestSuite) TearDownTest() {
+	s.db.Teardown(s.T())
+}
+
+// TestPolicyDescriptionMigration tests that at least one of the policies that needs to have its description
+// updated does indeed get successfully get migrated
+func (s *policyMigrationTestSuite) TestPolicyDescriptionMigration() {
+
+	testPolicy := fixtures.GetPolicy()
+	testPolicy.Id = "80267b36-2182-4fb3-8b53-e80c031f4ad8"
+	testPolicy.Name = "ADD Command used instead of COPY"
+	testPolicy.Description = "Alert on deployments using a ADD command"
+	testPolicy.PolicySections = []*storage.PolicySection{
+		{
+			PolicyGroups: []*storage.PolicyGroup{
+				{
+					FieldName: "Dockerfile Line",
+					Values: []*storage.PolicyValue{
+						{
+							Value: "ADD=.*",
+						},
+					},
+				},
+			},
+		},
+	}
+	require.NoError(s.T(), s.policyStore.Upsert(s.ctx, testPolicy))
+
+	s.NoError(migration.Run(&types.Databases{
+		PostgresDB: s.db.DB,
+		GormDB:     s.db.GetGormDB(),
+	}))
+
+	expectedDescription := "Alert on deployments using an ADD command"
+	policy, exists, err := s.policyStore.Get(s.ctx, testPolicy.GetId())
+	s.NoError(err)
+	s.True(exists)
+	s.Equal(expectedDescription, policy.GetDescription(), "description doesn't match after migration")
+}
+
+// TestPolicyExclusionMigration tests that at least one of the policies that needs to have an exclusion added
+// does indeed get successfully get migrated
+func (s *policyMigrationTestSuite) TestPolicyExclusionMigration() {
+	testPolicy := fixtures.GetPolicy()
+	testPolicy.Id = "6abcaa13-9ed6-4109-a1a7-be2e8280e49e"
+	testPolicy.Name = "Docker CIS 5.7: Ensure privileged ports are not mapped within containers"
+	testPolicy.Description = "The TCP/IP port numbers below 1024 are considered privileged ports. Normal users and processes are not allowed to use them for various security reasons. Containers are, however, allowed to map their ports to privileged ports."
+	testPolicy.PolicySections = []*storage.PolicySection{
+		{
+			PolicyGroups: []*storage.PolicyGroup{
+				{
+					FieldName:       "Exposed Node Port",
+					BooleanOperator: storage.BooleanOperator_AND,
+					Values: []*storage.PolicyValue{
+						{
+							Value: "<= 1024",
+						},
+						{
+							Value: "> 0",
+						},
+					},
+				},
+			},
+		},
+	}
+	require.NoError(s.T(), s.policyStore.Upsert(s.ctx, testPolicy))
+
+	s.NoError(migration.Run(&types.Databases{
+		PostgresDB: s.db.DB,
+		GormDB:     s.db.GetGormDB(),
+	}))
+
+	policy, exists, err := s.policyStore.Get(s.ctx, testPolicy.GetId())
+	s.True(exists)
+	s.NoError(err)
+	expectedExclusions := []*storage.Exclusion{
+		{
+			Name: "Don't alert on the router-default deployment in namespace openshift-ingress",
+			Deployment: &storage.Exclusion_Deployment{
+				Name: "router-default",
+				Scope: &storage.Scope{
+					Namespace: "openshift-ingress",
+				},
+			},
+		},
+	}
+	s.ElementsMatch(policy.Exclusions, expectedExclusions, "exclusion do not match after migration")
+}