From 2c5e6e56570734c617f5381282143f89269e49f4 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 11 Jul 2025 11:00:14 +0900 Subject: [PATCH 1/9] Add DynamoDB permission test --- .github/workflows/permission-check.yaml | 29 ++++ core/build.gradle | 33 ++++ .../DynamoAdminPermissionIntegrationTest.java | 62 ++++++++ .../scalar/db/storage/dynamo/DynamoEnv.java | 5 +- .../DynamoPermissionIntegrationTest.java | 36 +++++ .../dynamo/DynamoPermissionTestUtils.java | 146 ++++++++++++++++++ 6 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java create mode 100644 core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java create mode 100644 core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index 5740a084d4..3c0941792e 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -86,3 +86,32 @@ jobs: with: name: cassandra_3.11_permission_integration_test_reports path: core/build/reports/tests/integrationTestCassandraPermission + + integration-test-permission-dynamo: + name: DynamoDB Permission Integration Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }}) + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.JAVA_VENDOR }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Execute Gradle 'integrationTestDynamoPermission' task + run: ./gradlew integrationTestDynamoPermission + env: + DYNAMO_ACCESS_KEY_ID: ${{ secrets.DYNAMO_ACCESS_KEY }} + DYNAMO_SECRET_ACCESS_KEY: ${{ secrets.DYNAMO_SECRET_ACCESS_KEY }} + + - name: Upload Gradle test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: dynamo_permission_integration_test_reports + path: core/build/reports/tests/integrationTestDynamoPermission \ No newline at end of file diff --git a/core/build.gradle b/core/build.gradle index 5017ebaa16..8882e4351f 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -45,6 +45,9 @@ sourceSets { srcDir file('src/integration-test/java') include '**/com/scalar/db/common/*.java' include '**/com/scalar/db/storage/dynamo/*.java' + exclude '**/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java' + exclude '**/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java' + exclude '**/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java' } resources.srcDir file('src/integration-test/resources') } @@ -84,6 +87,20 @@ sourceSets { } resources.srcDir file('src/integration-test/resources') } + integrationTestDynamoPermission { + java { + compileClasspath += main.output + test.output + runtimeClasspath += main.output + test.output + srcDir file('src/integration-test/java') + include '**/com/scalar/db/common/*.java' + include '**/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java' + include '**/com/scalar/db/storage/dynamo/DynamoAdminTestUtils.java' + include '**/com/scalar/db/storage/dynamo/DynamoEnv.java' + include '**/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java' + include '**/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java' + } + resources.srcDir file('src/integration-test/resources') + } } configurations { @@ -108,6 +125,9 @@ configurations { integrationTestCassandraPermissionImplementation.extendsFrom testImplementation integrationTestCassandraPermissionRuntimeOnly.extendsFrom testRuntimeOnly integrationTestCassandraPermissionCompileOnly.extendsFrom testCompileOnly + integrationTestDynamoPermissionImplementation.extendsFrom testImplementation + integrationTestDynamoPermissionRuntimeOnly.extendsFrom testRuntimeOnly + integrationTestDynamoPermissionCompileOnly.extendsFrom testCompileOnly } dependencies { @@ -120,6 +140,8 @@ dependencies { implementation platform("software.amazon.awssdk:bom:${awssdkVersion}") implementation 'software.amazon.awssdk:applicationautoscaling' implementation 'software.amazon.awssdk:dynamodb' + implementation 'software.amazon.awssdk:iam' + implementation 'software.amazon.awssdk:iam-policy-builder' implementation "org.apache.commons:commons-dbcp2:${commonsDbcp2Version}" implementation "com.mysql:mysql-connector-j:${mysqlDriverVersion}" implementation "org.postgresql:postgresql:${postgresqlDriverVersion}" @@ -231,6 +253,17 @@ task integrationTestCassandraPermission(type: Test) { } } +task integrationTestDynamoPermission(type: Test) { + description = 'Runs the integration tests for DynamoDB permissions.' + group = 'verification' + testClassesDirs = sourceSets.integrationTestDynamoPermission.output.classesDirs + classpath = sourceSets.integrationTestDynamoPermission.runtimeClasspath + outputs.upToDateWhen { false } // ensures integration tests are run every time when called + options { + systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") }) + } +} + spotless { java { target 'src/*/java/**/*.java' diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java new file mode 100644 index 0000000000..b2fcd6aa5e --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java @@ -0,0 +1,62 @@ +package com.scalar.db.storage.dynamo; + +import static com.scalar.db.storage.dynamo.DynamoPermissionTestUtils.SLEEP_BETWEEN_TESTS_SECONDS; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.DistributedStorageAdminPermissionIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import com.scalar.db.util.PermissionTestUtils; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class DynamoAdminPermissionIntegrationTest + extends DistributedStorageAdminPermissionIntegrationTestBase { + @Override + protected Properties getProperties(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Properties getPropertiesForNormalUser(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Map getCreationOptions() { + return ImmutableMap.of(DynamoAdmin.NO_SCALING, "false", DynamoAdmin.NO_BACKUP, "false"); + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new DynamoAdminTestUtils(getProperties(testName)); + } + + @Override + protected PermissionTestUtils getPermissionTestUtils(String testName) { + return new DynamoPermissionTestUtils(getProperties(testName)); + } + + @Override + protected void sleepBetweenTests() { + Uninterruptibles.sleepUninterruptibly(SLEEP_BETWEEN_TESTS_SECONDS, TimeUnit.SECONDS); + } + + @Test + @Override + @Disabled("Import-related functionality is not supported in DynamoDB") + public void getImportTableMetadata_WithSufficientPermission_ShouldSucceed() {} + + @Test + @Override + @Disabled("Import-related functionality is not supported in DynamoDB") + public void addRawColumnToTable_WithSufficientPermission_ShouldSucceed() {} + + @Test + @Override + @Disabled("Import-related functionality is not supported in DynamoDB") + public void importTable_WithSufficientPermission_ShouldSucceed() {} +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java index cf7e719dfb..a978a2e3d0 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java @@ -10,12 +10,14 @@ public final class DynamoEnv { private static final String PROP_DYNAMO_REGION = "scalardb.dynamo.region"; private static final String PROP_DYNAMO_ACCESS_KEY_ID = "scalardb.dynamo.access_key_id"; private static final String PROP_DYNAMO_SECRET_ACCESS_KEY = "scalardb.dynamo.secret_access_key"; + private static final String PROP_DYNAMO_EMULATOR = "scalardb.dynamo.emulator"; private static final String PROP_DYNAMO_CREATE_OPTIONS = "scalardb.dynamo.create_options"; private static final String DEFAULT_DYNAMO_ENDPOINT_OVERRIDE = "http://localhost:8000"; private static final String DEFAULT_DYNAMO_REGION = "us-west-2"; private static final String DEFAULT_DYNAMO_ACCESS_KEY_ID = "fakeMyKeyId"; private static final String DEFAULT_DYNAMO_SECRET_ACCESS_KEY = "fakeSecretAccessKey"; + private static final String DEFAULT_DYNAMO_EMULATOR = "true"; private static final ImmutableMap DEFAULT_DYNAMO_CREATE_OPTIONS = ImmutableMap.of(DynamoAdmin.NO_SCALING, "true", DynamoAdmin.NO_BACKUP, "true"); @@ -30,9 +32,10 @@ public static Properties getProperties(String testName) { System.getProperty(PROP_DYNAMO_ACCESS_KEY_ID, DEFAULT_DYNAMO_ACCESS_KEY_ID); String secretAccessKey = System.getProperty(PROP_DYNAMO_SECRET_ACCESS_KEY, DEFAULT_DYNAMO_SECRET_ACCESS_KEY); + String isEmulator = System.getProperty(PROP_DYNAMO_EMULATOR, DEFAULT_DYNAMO_EMULATOR); Properties properties = new Properties(); - if (endpointOverride != null) { + if (Boolean.parseBoolean(isEmulator) && endpointOverride != null) { properties.setProperty(DynamoConfig.ENDPOINT_OVERRIDE, endpointOverride); } properties.setProperty(DatabaseConfig.CONTACT_POINTS, region); diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java new file mode 100644 index 0000000000..beb5816f5b --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java @@ -0,0 +1,36 @@ +package com.scalar.db.storage.dynamo; + +import com.google.common.collect.ImmutableMap; +import com.scalar.db.api.DistributedStoragePermissionIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import com.scalar.db.util.PermissionTestUtils; +import java.util.Map; +import java.util.Properties; + +public class DynamoPermissionIntegrationTest + extends DistributedStoragePermissionIntegrationTestBase { + @Override + protected Properties getProperties(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Properties getPropertiesForNormalUser(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Map getCreationOptions() { + return ImmutableMap.of(DynamoAdmin.NO_SCALING, "false", DynamoAdmin.NO_BACKUP, "false"); + } + + @Override + protected PermissionTestUtils getPermissionTestUtils(String testName) { + return new DynamoPermissionTestUtils(getProperties(testName)); + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new DynamoAdminTestUtils(getProperties(testName)); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java new file mode 100644 index 0000000000..c4fefe54da --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java @@ -0,0 +1,146 @@ +package com.scalar.db.storage.dynamo; + +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.util.PermissionTestUtils; +import java.util.Optional; +import java.util.Properties; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.iam.IamClient; +import software.amazon.awssdk.services.iam.model.AttachUserPolicyRequest; +import software.amazon.awssdk.services.iam.model.AttachedPolicy; +import software.amazon.awssdk.services.iam.model.CreatePolicyRequest; +import software.amazon.awssdk.services.iam.model.CreatePolicyVersionRequest; +import software.amazon.awssdk.services.iam.model.DeletePolicyVersionRequest; +import software.amazon.awssdk.services.iam.model.ListAttachedUserPoliciesRequest; +import software.amazon.awssdk.services.iam.model.ListPolicyVersionsRequest; +import software.amazon.awssdk.services.iam.model.User; + +public class DynamoPermissionTestUtils implements PermissionTestUtils { + public static final int SLEEP_BETWEEN_TESTS_SECONDS = 10; + private static final String IAM_POLICY_NAME = "test-dynamodb-permissions"; + private static final IamPolicy POLICY = + IamPolicy.builder() + .addStatement( + s -> + s.effect(IamEffect.ALLOW) + .addAction("dynamodb:ConditionCheckItem") + .addAction("dynamodb:PutItem") + .addAction("dynamodb:ListTables") + .addAction("dynamodb:DeleteItem") + .addAction("dynamodb:Scan") + .addAction("dynamodb:Query") + .addAction("dynamodb:UpdateItem") + .addAction("dynamodb:DeleteTable") + .addAction("dynamodb:UpdateContinuousBackups") + .addAction("dynamodb:CreateTable") + .addAction("dynamodb:DescribeTable") + .addAction("dynamodb:GetItem") + .addAction("dynamodb:DescribeContinuousBackups") + .addAction("dynamodb:UpdateTable") + .addAction("application-autoscaling:RegisterScalableTarget") + .addAction("application-autoscaling:DeleteScalingPolicy") + .addAction("application-autoscaling:PutScalingPolicy") + .addAction("application-autoscaling:DeregisterScalableTarget") + .addAction("application-autoscaling:TagResource") + .addResource(IamResource.ALL)) + .build(); + private final IamClient client; + + public DynamoPermissionTestUtils(Properties properties) { + DynamoConfig config = new DynamoConfig(new DatabaseConfig(properties)); + this.client = + IamClient.builder() + .region(Region.of(config.getRegion())) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + config.getAccessKeyId(), config.getSecretAccessKey()))) + .build(); + } + + @Override + public void createNormalUser(String userName, String password) { + // Do nothing for DynamoDB. + } + + @Override + public void dropNormalUser(String userName) { + // Do nothing for DynamoDB. + } + + @Override + public void grantRequiredPermission(String userName) { + try { + User user = client.getUser().user(); + Optional attachedPolicyArn = getAttachedPolicyArn(user.userName()); + if (attachedPolicyArn.isPresent()) { + deleteStalePolicyVersions(attachedPolicyArn.get()); + createNewPolicyVersion(attachedPolicyArn.get()); + } else { + String policyArn = createNewPolicy(); + client.attachUserPolicy( + AttachUserPolicyRequest.builder() + .userName(user.userName()) + .policyArn(policyArn) + .build()); + } + } catch (Exception e) { + throw new RuntimeException("Failed to grant required permissions", e); + } + } + + @Override + public void close() { + client.close(); + } + + private Optional getAttachedPolicyArn(String userName) { + AttachedPolicy attachedPolicy = + client + .listAttachedUserPolicies( + ListAttachedUserPoliciesRequest.builder().userName(userName).build()) + .attachedPolicies().stream() + .filter(policy -> policy.policyName().equals(DynamoPermissionTestUtils.IAM_POLICY_NAME)) + .findFirst() + .orElse(null); + return Optional.ofNullable(attachedPolicy).map(AttachedPolicy::policyArn); + } + + private String createNewPolicy() { + return client + .createPolicy( + CreatePolicyRequest.builder() + .policyName(IAM_POLICY_NAME) + .policyDocument(POLICY.toJson()) + .build()) + .policy() + .arn(); + } + + private void deleteStalePolicyVersions(String policyArn) { + client.listPolicyVersions(ListPolicyVersionsRequest.builder().policyArn(policyArn).build()) + .versions().stream() + .filter(version -> !version.isDefaultVersion()) + .forEach( + version -> + client.deletePolicyVersion( + DeletePolicyVersionRequest.builder() + .policyArn(policyArn) + .versionId(version.versionId()) + .build())); + } + + private void createNewPolicyVersion(String policyArn) { + client.createPolicyVersion( + CreatePolicyVersionRequest.builder() + .policyArn(policyArn) + .policyDocument(POLICY.toJson()) + .setAsDefault(true) + .build()); + } +} From ae6b8a36c1bf4c500d70860f4c2cbbf153a00076 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 11 Jul 2025 14:58:25 +0900 Subject: [PATCH 2/9] Add pull_request trigger to test workflow --- .github/workflows/permission-check.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index 3c0941792e..7dc28ad96e 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -2,6 +2,7 @@ name: Test Permissions on: workflow_dispatch: + pull_request: env: TERM: dumb From 06cc84ee332467e0618d9ab2eabd06be6d416fb5 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 11 Jul 2025 15:49:56 +0900 Subject: [PATCH 3/9] Apply suggestions --- .github/workflows/permission-check.yaml | 2 +- core/build.gradle | 4 +-- .../dynamo/DynamoPermissionTestUtils.java | 35 ++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index 7dc28ad96e..b1afa72192 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -115,4 +115,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: dynamo_permission_integration_test_reports - path: core/build/reports/tests/integrationTestDynamoPermission \ No newline at end of file + path: core/build/reports/tests/integrationTestDynamoPermission diff --git a/core/build.gradle b/core/build.gradle index 8882e4351f..b446b0ff01 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -140,8 +140,8 @@ dependencies { implementation platform("software.amazon.awssdk:bom:${awssdkVersion}") implementation 'software.amazon.awssdk:applicationautoscaling' implementation 'software.amazon.awssdk:dynamodb' - implementation 'software.amazon.awssdk:iam' - implementation 'software.amazon.awssdk:iam-policy-builder' + testImplementation 'software.amazon.awssdk:iam' + testImplementation 'software.amazon.awssdk:iam-policy-builder' implementation "org.apache.commons:commons-dbcp2:${commonsDbcp2Version}" implementation "com.mysql:mysql-connector-j:${mysqlDriverVersion}" implementation "org.postgresql:postgresql:${postgresqlDriverVersion}" diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java index c4fefe54da..13bdea85c1 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java @@ -6,6 +6,7 @@ import java.util.Properties; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.policybuilder.iam.IamEffect; import software.amazon.awssdk.policybuilder.iam.IamPolicy; import software.amazon.awssdk.policybuilder.iam.IamResource; @@ -79,18 +80,34 @@ public void grantRequiredPermission(String userName) { User user = client.getUser().user(); Optional attachedPolicyArn = getAttachedPolicyArn(user.userName()); if (attachedPolicyArn.isPresent()) { - deleteStalePolicyVersions(attachedPolicyArn.get()); - createNewPolicyVersion(attachedPolicyArn.get()); + String policyArn = attachedPolicyArn.get(); + try { + deleteStalePolicyVersions(policyArn); + createNewPolicyVersion(policyArn); + } catch (SdkException e) { + throw new RuntimeException( + String.format( + "Failed to update policy for user: %s, policyArn: %s", userName, policyArn), + e); + } } else { String policyArn = createNewPolicy(); - client.attachUserPolicy( - AttachUserPolicyRequest.builder() - .userName(user.userName()) - .policyArn(policyArn) - .build()); + try { + client.attachUserPolicy( + AttachUserPolicyRequest.builder() + .userName(user.userName()) + .policyArn(policyArn) + .build()); + } catch (SdkException e) { + throw new RuntimeException( + String.format( + "Failed to attach new policy for user: %s, policyArn: %s", userName, policyArn), + e); + } } - } catch (Exception e) { - throw new RuntimeException("Failed to grant required permissions", e); + } catch (SdkException e) { + throw new RuntimeException( + String.format("Failed to grant required permissions for user: %s", userName), e); } } From 04e6dcf5b7a4497690162aeb529d2113666d4614 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 11 Jul 2025 16:40:51 +0900 Subject: [PATCH 4/9] Fix workflow to specify arguments --- .github/workflows/permission-check.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index b1afa72192..8ad2b92c77 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -105,10 +105,7 @@ jobs: uses: gradle/actions/setup-gradle@v4 - name: Execute Gradle 'integrationTestDynamoPermission' task - run: ./gradlew integrationTestDynamoPermission - env: - DYNAMO_ACCESS_KEY_ID: ${{ secrets.DYNAMO_ACCESS_KEY }} - DYNAMO_SECRET_ACCESS_KEY: ${{ secrets.DYNAMO_SECRET_ACCESS_KEY }} + run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.accessKeyId=${{ secrets.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secretAccessKey=${{ secrets.DYNAMO_SECRET_ACCESS_KEY }} - name: Upload Gradle test reports if: always() From 4a542bfeed52a5d18a3380e7f8b11f74c928bdc6 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Mon, 14 Jul 2025 11:12:09 +0900 Subject: [PATCH 5/9] [skip ci] Fix workflow file --- .github/workflows/permission-check.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index 8ad2b92c77..af7acf32b9 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -2,12 +2,13 @@ name: Test Permissions on: workflow_dispatch: - pull_request: env: TERM: dumb JAVA_VERSION: '8' JAVA_VENDOR: 'temurin' + DYNAMO_ACCESS_KEY_ID: ${{ secrets.DYNAMO_ACCESS_KEY }} + DYNAMO_SECRET_ACCESS_KEY: ${{ secrets.DYNAMO_SECRET_ACCESS_KEY }} jobs: integration-test-permission-cassandra-3-0: @@ -105,7 +106,7 @@ jobs: uses: gradle/actions/setup-gradle@v4 - name: Execute Gradle 'integrationTestDynamoPermission' task - run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.accessKeyId=${{ secrets.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secretAccessKey=${{ secrets.DYNAMO_SECRET_ACCESS_KEY }} + run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.accessKeyId=${{ env.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secretAccessKey=${{ env.DYNAMO_SECRET_ACCESS_KEY }} - name: Upload Gradle test reports if: always() From c884e44e27cf7dcadc2ad3cf908e2d40d2414a95 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Mon, 14 Jul 2025 11:30:14 +0900 Subject: [PATCH 6/9] [skip ci] Update error log message --- .../scalar/db/storage/dynamo/DynamoPermissionTestUtils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java index 13bdea85c1..05f2549475 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java @@ -107,7 +107,10 @@ public void grantRequiredPermission(String userName) { } } catch (SdkException e) { throw new RuntimeException( - String.format("Failed to grant required permissions for user: %s", userName), e); + String.format( + "Failed to grant required permissions for user: %s, error: %s", + userName, e.getMessage()), + e); } } From 5cd1285b3fee7fe422beba3ee6556a91fe2b4482 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Mon, 14 Jul 2025 11:40:04 +0900 Subject: [PATCH 7/9] [skip ci] Fix error handling --- ...ageAdminPermissionIntegrationTestBase.java | 31 ++++++++------- ...dStoragePermissionIntegrationTestBase.java | 39 +++++++++++-------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java index 8e74667b3f..cb5b36058b 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java @@ -55,19 +55,24 @@ public void beforeAll() throws Exception { Properties propertiesForRootUser = getProperties(TEST_NAME); Properties propertiesForNormalUser = getPropertiesForNormalUser(TEST_NAME); - // Initialize the admin for root user - StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); - adminForRootUser = factoryForRootUser.getStorageAdmin(); - - // Create normal user and give permissions - DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); - normalUserName = getUserNameFromConfig(config); - normalUserPassword = getPasswordFromConfig(config); - setUpNormalUser(); - - // Initialize the admin for normal user - StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); - adminForNormalUser = factoryForNormalUser.getStorageAdmin(); + try { + // Initialize the admin for root user + StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); + adminForRootUser = factoryForRootUser.getStorageAdmin(); + + // Create normal user and give permissions + DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); + normalUserName = getUserNameFromConfig(config); + normalUserPassword = getPasswordFromConfig(config); + setUpNormalUser(); + + // Initialize the admin for normal user + StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); + adminForNormalUser = factoryForNormalUser.getStorageAdmin(); + } catch (Exception e) { + logger.error("Failed to set up the test environment", e); + throw e; + } } @AfterAll diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java index f28e41a60c..f4be283209 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java @@ -51,23 +51,28 @@ public void beforeAll() throws Exception { Properties propertiesForRootUser = getProperties(TEST_NAME); Properties propertiesForNormalUser = getPropertiesForNormalUser(TEST_NAME); - // Create admin for root user - StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); - adminForRootUser = factoryForRootUser.getStorageAdmin(); - - // Create normal user and give permissions - DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); - normalUserName = getUserNameFromConfig(config); - normalUserPassword = getPasswordFromConfig(config); - setUpNormalUser(); - - // Create storage for normal user - StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); - storageForNormalUser = factoryForNormalUser.getStorage(); - - namespace = getNamespace(); - createTable(); - waitForTableCreation(); + try { + // Create admin for root user + StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); + adminForRootUser = factoryForRootUser.getStorageAdmin(); + + // Create normal user and give permissions + DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); + normalUserName = getUserNameFromConfig(config); + normalUserPassword = getPasswordFromConfig(config); + setUpNormalUser(); + + // Create storage for normal user + StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); + storageForNormalUser = factoryForNormalUser.getStorage(); + + namespace = getNamespace(); + createTable(); + waitForTableCreation(); + } catch (Exception e) { + logger.error("Failed to set up the test environment", e); + throw e; + } } @BeforeEach From 28cc91903da3870564137b1ee3ddf72741c81eba Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Mon, 14 Jul 2025 11:56:57 +0900 Subject: [PATCH 8/9] [skip ci] Fix workflow file --- .github/workflows/permission-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index af7acf32b9..4ec74cdbba 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -106,7 +106,7 @@ jobs: uses: gradle/actions/setup-gradle@v4 - name: Execute Gradle 'integrationTestDynamoPermission' task - run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.accessKeyId=${{ env.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secretAccessKey=${{ env.DYNAMO_SECRET_ACCESS_KEY }} + run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.access_key_id=${{ env.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secret_access_key=${{ env.DYNAMO_SECRET_ACCESS_KEY }} - name: Upload Gradle test reports if: always() From 0f740726ae29752f8eed34cbde5378b70a2624a3 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Tue, 15 Jul 2025 11:13:46 +0900 Subject: [PATCH 9/9] Apply suggestions --- .github/workflows/permission-check.yaml | 2 +- .../java/com/scalar/db/storage/dynamo/DynamoEnv.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index 4ec74cdbba..bd56dbeaa3 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -106,7 +106,7 @@ jobs: uses: gradle/actions/setup-gradle@v4 - name: Execute Gradle 'integrationTestDynamoPermission' task - run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.access_key_id=${{ env.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secret_access_key=${{ env.DYNAMO_SECRET_ACCESS_KEY }} + run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator_used=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.access_key_id=${{ env.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secret_access_key=${{ env.DYNAMO_SECRET_ACCESS_KEY }} - name: Upload Gradle test reports if: always() diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java index a978a2e3d0..1b1979e71e 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java @@ -10,14 +10,14 @@ public final class DynamoEnv { private static final String PROP_DYNAMO_REGION = "scalardb.dynamo.region"; private static final String PROP_DYNAMO_ACCESS_KEY_ID = "scalardb.dynamo.access_key_id"; private static final String PROP_DYNAMO_SECRET_ACCESS_KEY = "scalardb.dynamo.secret_access_key"; - private static final String PROP_DYNAMO_EMULATOR = "scalardb.dynamo.emulator"; + private static final String PROP_DYNAMO_EMULATOR_USED = "scalardb.dynamo.emulator_used"; private static final String PROP_DYNAMO_CREATE_OPTIONS = "scalardb.dynamo.create_options"; private static final String DEFAULT_DYNAMO_ENDPOINT_OVERRIDE = "http://localhost:8000"; private static final String DEFAULT_DYNAMO_REGION = "us-west-2"; private static final String DEFAULT_DYNAMO_ACCESS_KEY_ID = "fakeMyKeyId"; private static final String DEFAULT_DYNAMO_SECRET_ACCESS_KEY = "fakeSecretAccessKey"; - private static final String DEFAULT_DYNAMO_EMULATOR = "true"; + private static final String DEFAULT_DYNAMO_EMULATOR_USED = "true"; private static final ImmutableMap DEFAULT_DYNAMO_CREATE_OPTIONS = ImmutableMap.of(DynamoAdmin.NO_SCALING, "true", DynamoAdmin.NO_BACKUP, "true"); @@ -32,10 +32,11 @@ public static Properties getProperties(String testName) { System.getProperty(PROP_DYNAMO_ACCESS_KEY_ID, DEFAULT_DYNAMO_ACCESS_KEY_ID); String secretAccessKey = System.getProperty(PROP_DYNAMO_SECRET_ACCESS_KEY, DEFAULT_DYNAMO_SECRET_ACCESS_KEY); - String isEmulator = System.getProperty(PROP_DYNAMO_EMULATOR, DEFAULT_DYNAMO_EMULATOR); + String isEmulatorUsed = + System.getProperty(PROP_DYNAMO_EMULATOR_USED, DEFAULT_DYNAMO_EMULATOR_USED); Properties properties = new Properties(); - if (Boolean.parseBoolean(isEmulator) && endpointOverride != null) { + if (Boolean.parseBoolean(isEmulatorUsed) && endpointOverride != null) { properties.setProperty(DynamoConfig.ENDPOINT_OVERRIDE, endpointOverride); } properties.setProperty(DatabaseConfig.CONTACT_POINTS, region);