-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Performance Harness: Support for pseudo-parallel streams (#26219)
* Adds support for pseudo-parallel datasets * Ran ./gradlew :spotlessJavaApply * Automated Change * Fixes issue with parallel datasets credentials * Fixes filter for parallel credentials * Adds a new configurable property to build a pseudo-parallel catalog * Fixes Github Actions variable to be processed properly with the K8s harness yaml * Adds unit test for random streams and generating streams within the same configured catalog * Ran ./gradlew :spotlessJavaApply * Added additional description for GitHub Actions * Update connector-performance-command.yml Moved text up to connect with other argument discussion * Fixes spotBugs issue * Automated Commit - Formatting Changes * Update GitHub Action description --------- Co-authored-by: ryankfu <ryankfu@users.noreply.github.com> Co-authored-by: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com>
- Loading branch information
3 people
committed
May 22, 2023
1 parent
988ce24
commit 3689ff3
Showing
12 changed files
with
183 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
57 changes: 57 additions & 0 deletions
57
...ation-harness/src/test/java/io/airbyte/integrations/destination_performance/MainTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| /* | ||
| * Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination_performance; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.*; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class MainTest { | ||
|
|
||
| @Test | ||
| void testDuplicateStreams() throws JsonProcessingException { | ||
| final String simpleCatalog = """ | ||
| { | ||
| "streams": [ | ||
| { | ||
| "stream": { | ||
| "name": "users", | ||
| "namespace": "PERF_TEST_HARNESS", | ||
| "json_schema": { | ||
| "type": "object", | ||
| "properties": { | ||
| "id": { | ||
| "type": "number", | ||
| "airbyte_type": "integer" | ||
| }, | ||
| "academic_degree": { | ||
| "type": "string" | ||
| } | ||
| } | ||
| }, | ||
| "default_cursor_field": [], | ||
| "supported_sync_modes": ["full_refresh", "incremental"], | ||
| "source_defined_primary_key": [["id"]] | ||
| }, | ||
| "sync_mode": "full_refresh", | ||
| "primary_key": [["id"]], | ||
| "cursor_field": ["updated_at"], | ||
| "destination_sync_mode": "overwrite" | ||
| } | ||
| ] | ||
| } | ||
| """; | ||
| final ObjectMapper objectMapper = new ObjectMapper(); | ||
| final JsonNode root = objectMapper.readTree(simpleCatalog); | ||
| final int duplicateFactor = 10; | ||
| Main.duplicateStreams(root, duplicateFactor); | ||
| assertEquals(duplicateFactor, root.get("streams").size()); | ||
| assertEquals("users9", root.path("streams").get(9).path("stream").path("name").asText()); | ||
| } | ||
|
|
||
| } |
30 changes: 30 additions & 0 deletions
30
...src/test/java/io/airbyte/integrations/destination_performance/PerformanceHarnessTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /* | ||
| * Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination_performance; | ||
|
|
||
| import static org.junit.Assert.assertNotEquals; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Random; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class PerformanceHarnessTest { | ||
|
|
||
| @Test | ||
| public void testRandomStreamName() { | ||
| final List<String> streamNames = new ArrayList<>(); | ||
| final Random random = new Random(); | ||
| final int duplicateFactor = 1000; | ||
| // Keep this number high to avoid statistical collisions. Alternative was to consider chi-squared | ||
| for (int i = 1; i <= duplicateFactor; i++) { | ||
| streamNames.add("stream" + i); | ||
| } | ||
| final String streamName1 = PerformanceHarness.getStreamName(streamNames, random); | ||
| final String streamName2 = PerformanceHarness.getStreamName(streamNames, random); | ||
| assertNotEquals(streamName1, streamName2); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters