Merge pull request #181 from josephschorr/joseph.schorr/PROJQUAY-185/…

…remove-enc-token

Remove the active migration for encrypted tokens now that it is complete

Dockerfile.cirun

@@ -1,7 +1,6 @@
 FROM quay-ci-base
 RUN mkdir -p conf/stack
 RUN rm -rf test/data/test.db
-ENV ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE remove-old-fields
 ADD cirun.config.yaml conf/stack/config.yaml
 RUN /usr/bin/scl enable python27 rh-nginx112 "LOGGING_LEVEL=INFO python initdb.py"
 ENTRYPOINT ["/quay-registry/quay-entrypoint.sh"]

Makefile

@@ -43,25 +43,25 @@ conf/stack/license: $(QUAY_CONFIG)/local/license
 	ln -s $(QUAY_CONFIG)/local/license conf/stack/license
 
 unit-test:
-	ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields TEST=true PYTHONPATH="." py.test \
+	TEST=true PYTHONPATH="." py.test \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose -x \
 	./
 
 registry-test:
-	TEST=true ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields PYTHONPATH="." py.test  \
+	TEST=true PYTHONPATH="." py.test  \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose --show-count -x \
 	test/registry/registry_tests.py
 
 registry-test-old:
-	TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test  \
+	TEST=true PYTHONPATH="." py.test  \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose --show-count -x \
 	./test/registry_tests.py
 
 buildman-test:
-	TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
+	TEST=true PYTHONPATH="." py.test \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose --show-count -x \
 	./buildman/
@@ -71,8 +71,8 @@ certs-test:
 
 full-db-test: ensure-test-db
 	TEST=true PYTHONPATH=. QUAY_OVERRIDE_CONFIG='{"DATABASE_SECRET_KEY": "anothercrazykey!"}' \
-	ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields alembic upgrade head
-	TEST=true PYTHONPATH=. ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields \
+	alembic upgrade head
+	TEST=true PYTHONPATH=. \
 	SKIP_DB_SCHEMA=true py.test --timeout=7200 \
 	--verbose --show-count -x --ignore=endpoints/appr/test/ \
 	./

Makefile.ci

@@ -8,21 +8,21 @@ export PATH := ./venv/bin:$(PATH)
 all: test
 
 unit-test:
-	TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test \
+	TEST=true PYTHONPATH="." py.test \
 	-m $(PYTEST_MARK) \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose -x \
 	./
 
 registry-test:
-	TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test  \
+	TEST=true PYTHONPATH="." py.test  \
 	-m $(PYTEST_MARK) \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose --show-count -x \
 	test/registry/registry_tests.py
 
 registry-test-old:
-	TEST=true PYTHONPATH="." ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields py.test  \
+	TEST=true PYTHONPATH="." py.test  \
 	--cov="." --cov-report=html --cov-report=term-missing \
 	--timeout=3600 --verbose --show-count -x \
 	./test/registry_tests.py
@@ -35,8 +35,8 @@ gunicorn-tests:
 
 full-db-test: ensure-test-db
 	TEST=true PYTHONPATH=. QUAY_OVERRIDE_CONFIG='{"DATABASE_SECRET_KEY": "anothercrazykey!"}' \
-	ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields alembic upgrade head
-	TEST=true PYTHONPATH=. ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE=remove-old-fields \
+	alembic upgrade head
+	TEST=true PYTHONPATH=. \
 	SKIP_DB_SCHEMA=true py.test --timeout=7200 \
 	-m $(PYTEST_MARK) \
 	--verbose --show-count -x --ignore=endpoints/appr/test/ \

active_migration.py

@@ -1,31 +1,4 @@
 from enum import Enum, unique
 from data.migrationutil import DefinedDataMigration, MigrationPhase
 
-
-@unique
-class ERTMigrationFlags(Enum):
-    """ Flags for the encrypted robot token migration. """
-
-    READ_OLD_FIELDS = "read-old"
-    WRITE_OLD_FIELDS = "write-old"
-
-
-ActiveDataMigration = DefinedDataMigration(
-    "encrypted_robot_tokens",
-    "ENCRYPTED_ROBOT_TOKEN_MIGRATION_PHASE",
-    [
-        MigrationPhase(
-            "add-new-fields",
-            "c13c8052f7a6",
-            [ERTMigrationFlags.READ_OLD_FIELDS, ERTMigrationFlags.WRITE_OLD_FIELDS],
-        ),
-        MigrationPhase(
-            "backfill-then-read-only-new", "703298a825c2", [ERTMigrationFlags.WRITE_OLD_FIELDS]
-        ),
-        MigrationPhase(
-            "change-column-types", "49e1138ed12d", [ERTMigrationFlags.WRITE_OLD_FIELDS],
-        ),
-        MigrationPhase("stop-writing-both", "49e1138ed12d", []),
-        MigrationPhase("remove-old-fields", "c059b952ed76", []),
-    ],
-)
+ActiveDataMigration = None

auth/auth_context_type.py

@@ -260,7 +260,7 @@ def to_signed_dict(self):
         # TODO: Remove this all once the new code is fully deployed.
         if self.token:
             dict_data.update(
-                {"kind": "token", "token": self.token.code,}
+                {"kind": "token", "token": self.token.get_code(),}
             )
 
         if self.oauthtoken:

buildman/component/buildcomponent.py

@@ -8,7 +8,6 @@
 from autobahn.wamp.exception import ApplicationError
 from trollius import From, Return
 
-from active_migration import ActiveDataMigration, ERTMigrationFlags
 from buildman.server import BuildJobResult
 from buildman.component.basecomponent import BaseComponent
 from buildman.component.buildparse import extract_current_step
@@ -162,21 +161,13 @@ def start_build(self, build_job):
         #  sha: the sha1 identifier of the commit to check out
         #  private_key: the key used to get read access to the git repository
 
-        # TODO(remove-unenc): Remove legacy field.
         private_key = None
         if (
             build_job.repo_build.trigger is not None
             and build_job.repo_build.trigger.secure_private_key is not None
         ):
             private_key = build_job.repo_build.trigger.secure_private_key.decrypt()
 
-        if (
-            ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS)
-            and private_key is None
-            and build_job.repo_build.trigger is not None
-        ):
-            private_key = build_job.repo_build.trigger.private_key
-
         if private_key is not None:
             build_arguments["git"] = {
                 "url": build_config["trigger_metadata"].get("git_url", ""),

buildtrigger/basehandler.py

@@ -3,7 +3,6 @@
 from jsonschema import validate
 from six import add_metaclass
 
-from active_migration import ActiveDataMigration, ERTMigrationFlags
 from endpoints.building import PreparedBuild
 from data import model
 from buildtrigger.triggerutil import get_trigger_config, InvalidServiceException
@@ -147,16 +146,12 @@ def __init__(self, trigger, override_config=None):
     def auth_token(self):
         """ Returns the auth token for the trigger. """
         # NOTE: This check is for testing.
-        if isinstance(self.trigger.auth_token, str):
+        if hasattr(self.trigger, "auth_token"):
             return self.trigger.auth_token
 
-        # TODO(remove-unenc): Remove legacy field.
         if self.trigger.secure_auth_token is not None:
             return self.trigger.secure_auth_token.decrypt()
 
-        if ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS):
-            return self.trigger.auth_token
-
         return None
 
     @abstractmethod

data/database.py

@@ -25,7 +25,6 @@
 import resumablehashlib
 from cachetools.func import lru_cache
 
-from active_migration import ERTMigrationFlags, ActiveDataMigration
 from data.fields import (
     ResumableSHA256Field,
     ResumableSHA1Field,
@@ -883,12 +882,6 @@ class AccessTokenKind(BaseModel):
 class AccessToken(BaseModel):
     friendly_name = CharField(null=True)
 
-    # TODO(remove-unenc): This field is deprecated and should be removed soon.
-    code = deprecated_field(
-        CharField(default=random_string_generator(length=64), unique=True, index=True, null=True),
-        ERTMigrationFlags.WRITE_OLD_FIELDS,
-    )
-
     token_name = CharField(default=random_string_generator(length=32), unique=True, index=True)
     token_code = EncryptedCharField(default_token_length=32)
 
@@ -899,10 +892,7 @@ class AccessToken(BaseModel):
     kind = ForeignKeyField(AccessTokenKind, null=True)
 
     def get_code(self):
-        if ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS):
-            return self.code
-        else:
-            return self.token_name + self.token_code.decrypt()
+        return self.token_name + self.token_code.decrypt()
 
 
 class BuildTriggerService(BaseModel):
@@ -919,10 +909,6 @@ class RepositoryBuildTrigger(BaseModel):
     repository = ForeignKeyField(Repository)
     connected_user = QuayUserField()
 
-    # TODO(remove-unenc): These fields are deprecated and should be removed soon.
-    auth_token = deprecated_field(CharField(null=True), ERTMigrationFlags.WRITE_OLD_FIELDS)
-    private_key = deprecated_field(TextField(null=True), ERTMigrationFlags.WRITE_OLD_FIELDS)
-
     secure_auth_token = EncryptedCharField(null=True)
     secure_private_key = EncryptedTextField(null=True)
     fully_migrated = BooleanField(default=False)
@@ -1256,12 +1242,6 @@ class OAuthApplication(BaseModel):
     secure_client_secret = EncryptedCharField(default_token_length=40, null=True)
     fully_migrated = BooleanField(default=False)
 
-    # TODO(remove-unenc): This field is deprecated and should be removed soon.
-    client_secret = deprecated_field(
-        CharField(default=random_string_generator(length=40), null=True),
-        ERTMigrationFlags.WRITE_OLD_FIELDS,
-    )
-
     redirect_uri = CharField()
     application_uri = CharField()
     organization = QuayUserField()
@@ -1274,11 +1254,6 @@ class OAuthApplication(BaseModel):
 class OAuthAuthorizationCode(BaseModel):
     application = ForeignKeyField(OAuthApplication)
 
-    # TODO(remove-unenc): This field is deprecated and should be removed soon.
-    code = deprecated_field(
-        CharField(index=True, unique=True, null=True), ERTMigrationFlags.WRITE_OLD_FIELDS
-    )
-
     code_name = CharField(index=True, unique=True)
     code_credential = CredentialField()
 
@@ -1294,11 +1269,6 @@ class OAuthAccessToken(BaseModel):
     token_name = CharField(index=True, unique=True)
     token_code = CredentialField()
 
-    # TODO(remove-unenc): This field is deprecated and should be removed soon.
-    access_token = deprecated_field(
-        CharField(index=True, null=True), ERTMigrationFlags.WRITE_OLD_FIELDS
-    )
-
     token_type = CharField(default="Bearer")
     expires_at = DateTimeField()
     data = TextField()  # This is context for which this token was generated, such as the user
@@ -1600,12 +1570,6 @@ class AppSpecificAuthToken(BaseModel):
     token_name = CharField(index=True, unique=True, default=random_string_generator(60))
     token_secret = EncryptedCharField(default_token_length=60)
 
-    # TODO(remove-unenc): This field is deprecated and should be removed soon.
-    token_code = deprecated_field(
-        CharField(default=random_string_generator(length=120), unique=True, index=True, null=True),
-        ERTMigrationFlags.WRITE_OLD_FIELDS,
-    )
-
     created = DateTimeField(default=datetime.now)
     expiration = DateTimeField(null=True)
     last_accessed = DateTimeField(null=True)

data/model/appspecifictoken.py

@@ -2,7 +2,6 @@
 
 from datetime import datetime
 
-from active_migration import ActiveDataMigration, ERTMigrationFlags
 from data.database import AppSpecificAuthToken, User, random_string_generator
 from data.model import config
 from data.model._basequery import update_last_accessed
@@ -40,17 +39,12 @@ def create_token(user, title, expiration=_default_expiration_duration_opt):
     assert token_name
     assert token_secret
 
-    # TODO(remove-unenc): Remove legacy handling.
-    old_token_code = (
-        token_code if ActiveDataMigration.has_flag(ERTMigrationFlags.WRITE_OLD_FIELDS) else None
-    )
     return AppSpecificAuthToken.create(
         user=user,
         title=title,
         expiration=expiration,
         token_name=token_name,
         token_secret=DecryptedValue(token_secret),
-        token_code=old_token_code,
     )
 
 
@@ -154,35 +148,9 @@ def access_valid_token(token_code):
     except AppSpecificAuthToken.DoesNotExist:
         pass
 
-    # TODO(remove-unenc): Remove legacy handling.
-    if ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS):
-        try:
-            token = (
-                AppSpecificAuthToken.select(AppSpecificAuthToken, User)
-                .join(User)
-                .where(
-                    AppSpecificAuthToken.token_code == token_code,
-                    (
-                        (AppSpecificAuthToken.expiration > datetime.now())
-                        | (AppSpecificAuthToken.expiration >> None)
-                    ),
-                )
-                .get()
-            )
-
-            update_last_accessed(token)
-            return token
-        except AppSpecificAuthToken.DoesNotExist:
-            return None
-
     return None
 
 
 def get_full_token_string(token):
-    # TODO(remove-unenc): Remove legacy handling.
-    if ActiveDataMigration.has_flag(ERTMigrationFlags.READ_OLD_FIELDS):
-        if not token.token_name:
-            return token.token_code
-
     assert token.token_name
     return "%s%s" % (token.token_name, token.token_secret.decrypt())

data/model/build.py

@@ -4,7 +4,6 @@
 
 from peewee import JOIN
 
-from active_migration import ActiveDataMigration, ERTMigrationFlags
 from data.database import (
     BuildTriggerService,
     RepositoryBuildTrigger,
@@ -41,11 +40,7 @@
 def update_build_trigger(trigger, config, auth_token=None, write_token=None):
     trigger.config = json.dumps(config or {})
 
-    # TODO(remove-unenc): Remove legacy field.
     if auth_token is not None:
-        if ActiveDataMigration.has_flag(ERTMigrationFlags.WRITE_OLD_FIELDS):
-            trigger.auth_token = auth_token
-
         trigger.secure_auth_token = DecryptedValue(auth_token)
 
     if write_token is not None:
@@ -56,17 +51,10 @@ def update_build_trigger(trigger, config, auth_token=None, write_token=None):
 
 def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None, config=None):
     service = BuildTriggerService.get(name=service_name)
-
-    # TODO(remove-unenc): Remove legacy field.
-    old_auth_token = None
-    if ActiveDataMigration.has_flag(ERTMigrationFlags.WRITE_OLD_FIELDS):
-        old_auth_token = auth_token
-
     secure_auth_token = DecryptedValue(auth_token) if auth_token else None
     trigger = RepositoryBuildTrigger.create(
         repository=repo,
         service=service,
-        auth_token=old_auth_token,
         secure_auth_token=secure_auth_token,
         connected_user=user,
         pull_robot=pull_robot,