Skip to content

Commit

Permalink
Revert "Bmoric/add fetch source schema in oauth (#19392)" (#19512)
Browse files Browse the repository at this point in the history 
This reverts commit a968078.
  • Loading branch information
@benmoriceau @akashkulk
benmoriceau authored and akashkulk committed Nov 17, 2022
1 parent 5a74345 commit e68dd89
Show file tree
Hide file tree
Showing 6 changed files with 25 additions and 280 deletions.
4 changes: 0 additions & 4 deletions airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,10 +281,6 @@ public static void mergeMaps(final Map<String, Object> originalMap, final String
Entry::getValue)));
}

public static Map<String, String> deserializeToStringMap(JsonNode json) {
return OBJECT_MAPPER.convertValue(json, new TypeReference<>() {});
}

/**
* By the Jackson DefaultPrettyPrinter prints objects with an extra space as follows: {"name" :
* "airbyte"}. We prefer {"name": "airbyte"}.
Expand Down
2 changes: 1 addition & 1 deletion airbyte-server/src/main/java/io/airbyte/server/ServerApp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory,

final HealthCheckHandler healthCheckHandler = new HealthCheckHandler(configRepository);

final OAuthHandler oAuthHandler = new OAuthHandler(configRepository, httpClient, trackingClient, secretsRepositoryReader);
final OAuthHandler oAuthHandler = new OAuthHandler(configRepository, httpClient, trackingClient);

final SourceHandler sourceHandler = new SourceHandler(
configRepository,
Expand Down
136 changes: 23 additions & 113 deletions airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

package io.airbyte.server.handlers;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.annotations.VisibleForTesting;
import io.airbyte.analytics.TrackingClient;
import io.airbyte.api.model.generated.CompleteDestinationOAuthRequest;
import io.airbyte.api.model.generated.CompleteSourceOauthRequest;
Expand All @@ -14,32 +12,23 @@
import io.airbyte.api.model.generated.SetInstancewideDestinationOauthParamsRequestBody;
import io.airbyte.api.model.generated.SetInstancewideSourceOauthParamsRequestBody;
import io.airbyte.api.model.generated.SourceOauthConsentRequest;
import io.airbyte.commons.constants.AirbyteSecretConstants;
import io.airbyte.commons.json.JsonPaths;
import io.airbyte.commons.json.Jsons;
import io.airbyte.config.DestinationConnection;
import io.airbyte.config.DestinationOAuthParameter;
import io.airbyte.config.SourceConnection;
import io.airbyte.config.SourceOAuthParameter;
import io.airbyte.config.StandardDestinationDefinition;
import io.airbyte.config.StandardSourceDefinition;
import io.airbyte.config.persistence.ConfigNotFoundException;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.config.persistence.SecretsRepositoryReader;
import io.airbyte.oauth.OAuthFlowImplementation;
import io.airbyte.oauth.OAuthImplementationFactory;
import io.airbyte.persistence.job.factory.OAuthConfigSupplier;
import io.airbyte.persistence.job.tracker.TrackingMetadata;
import io.airbyte.protocol.models.ConnectorSpecification;
import io.airbyte.server.handlers.helpers.OAuthPathExtractor;
import io.airbyte.validation.json.JsonValidationException;
import java.io.IOException;
import java.net.http.HttpClient;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -51,97 +40,67 @@ public class OAuthHandler {
private final ConfigRepository configRepository;
private final OAuthImplementationFactory oAuthImplementationFactory;
private final TrackingClient trackingClient;
private final SecretsRepositoryReader secretsRepositoryReader;

public OAuthHandler(final ConfigRepository configRepository,
final HttpClient httpClient,
final TrackingClient trackingClient,
final SecretsRepositoryReader secretsRepositoryReader) {
final TrackingClient trackingClient) {
this.configRepository = configRepository;
this.oAuthImplementationFactory = new OAuthImplementationFactory(configRepository, httpClient);
this.trackingClient = trackingClient;
this.secretsRepositoryReader = secretsRepositoryReader;
}

public OAuthConsentRead getSourceOAuthConsent(final SourceOauthConsentRequest sourceOauthConsentRequest)
public OAuthConsentRead getSourceOAuthConsent(final SourceOauthConsentRequest sourceDefinitionIdRequestBody)
throws JsonValidationException, ConfigNotFoundException, IOException {
final StandardSourceDefinition sourceDefinition =
configRepository.getStandardSourceDefinition(sourceOauthConsentRequest.getSourceDefinitionId());
configRepository.getStandardSourceDefinition(sourceDefinitionIdRequestBody.getSourceDefinitionId());
final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(sourceDefinition);
final ConnectorSpecification spec = sourceDefinition.getSpec();
final Map<String, Object> metadata = generateSourceMetadata(sourceOauthConsentRequest.getSourceDefinitionId());
final Map<String, Object> metadata = generateSourceMetadata(sourceDefinitionIdRequestBody.getSourceDefinitionId());
final OAuthConsentRead result;
if (OAuthConfigSupplier.hasOAuthConfigSpecification(spec)) {
final JsonNode oAuthInputConfigurationForConsent;

if (sourceOauthConsentRequest.getSourceId() == null) {
oAuthInputConfigurationForConsent = sourceOauthConsentRequest.getoAuthInputConfiguration();
} else {
final SourceConnection hydratedSourceConnection =
secretsRepositoryReader.getSourceConnectionWithSecrets(sourceOauthConsentRequest.getSourceId());

oAuthInputConfigurationForConsent = getOAuthInputConfigurationForConsent(spec,
hydratedSourceConnection.getConfiguration(),
sourceOauthConsentRequest.getoAuthInputConfiguration());
}

result = new OAuthConsentRead().consentUrl(oAuthFlowImplementation.getSourceConsentUrl(
sourceOauthConsentRequest.getWorkspaceId(),
sourceOauthConsentRequest.getSourceDefinitionId(),
sourceOauthConsentRequest.getRedirectUrl(),
oAuthInputConfigurationForConsent,
sourceDefinitionIdRequestBody.getWorkspaceId(),
sourceDefinitionIdRequestBody.getSourceDefinitionId(),
sourceDefinitionIdRequestBody.getRedirectUrl(),
sourceDefinitionIdRequestBody.getoAuthInputConfiguration(),
spec.getAdvancedAuth().getOauthConfigSpecification()));
} else {
result = new OAuthConsentRead().consentUrl(oAuthFlowImplementation.getSourceConsentUrl(
sourceOauthConsentRequest.getWorkspaceId(),
sourceOauthConsentRequest.getSourceDefinitionId(),
sourceOauthConsentRequest.getRedirectUrl(), Jsons.emptyObject(), null));
sourceDefinitionIdRequestBody.getWorkspaceId(),
sourceDefinitionIdRequestBody.getSourceDefinitionId(),
sourceDefinitionIdRequestBody.getRedirectUrl(), Jsons.emptyObject(), null));
}
try {
trackingClient.track(sourceOauthConsentRequest.getWorkspaceId(), "Get Oauth Consent URL - Backend", metadata);
trackingClient.track(sourceDefinitionIdRequestBody.getWorkspaceId(), "Get Oauth Consent URL - Backend", metadata);
} catch (final Exception e) {
LOGGER.error(ERROR_MESSAGE, e);
}
return result;
}

public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsentRequest destinationOauthConsentRequest)
public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsentRequest destinationDefinitionIdRequestBody)
throws JsonValidationException, ConfigNotFoundException, IOException {
final StandardDestinationDefinition destinationDefinition =
configRepository.getStandardDestinationDefinition(destinationOauthConsentRequest.getDestinationDefinitionId());
configRepository.getStandardDestinationDefinition(destinationDefinitionIdRequestBody.getDestinationDefinitionId());
final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(destinationDefinition);
final ConnectorSpecification spec = destinationDefinition.getSpec();
final Map<String, Object> metadata = generateDestinationMetadata(destinationOauthConsentRequest.getDestinationDefinitionId());
final Map<String, Object> metadata = generateDestinationMetadata(destinationDefinitionIdRequestBody.getDestinationDefinitionId());
final OAuthConsentRead result;
if (OAuthConfigSupplier.hasOAuthConfigSpecification(spec)) {
final JsonNode oAuthInputConfigurationForConsent;

if (destinationOauthConsentRequest.getDestinationId() == null) {
oAuthInputConfigurationForConsent = destinationOauthConsentRequest.getoAuthInputConfiguration();
} else {
final DestinationConnection hydratedSourceConnection =
secretsRepositoryReader.getDestinationConnectionWithSecrets(destinationOauthConsentRequest.getDestinationId());

oAuthInputConfigurationForConsent = getOAuthInputConfigurationForConsent(spec,
hydratedSourceConnection.getConfiguration(),
destinationOauthConsentRequest.getoAuthInputConfiguration());

}

result = new OAuthConsentRead().consentUrl(oAuthFlowImplementation.getDestinationConsentUrl(
destinationOauthConsentRequest.getWorkspaceId(),
destinationOauthConsentRequest.getDestinationDefinitionId(),
destinationOauthConsentRequest.getRedirectUrl(),
oAuthInputConfigurationForConsent,
destinationDefinitionIdRequestBody.getWorkspaceId(),
destinationDefinitionIdRequestBody.getDestinationDefinitionId(),
destinationDefinitionIdRequestBody.getRedirectUrl(),
destinationDefinitionIdRequestBody.getoAuthInputConfiguration(),
spec.getAdvancedAuth().getOauthConfigSpecification()));
} else {
result = new OAuthConsentRead().consentUrl(oAuthFlowImplementation.getDestinationConsentUrl(
destinationOauthConsentRequest.getWorkspaceId(),
destinationOauthConsentRequest.getDestinationDefinitionId(),
destinationOauthConsentRequest.getRedirectUrl(), Jsons.emptyObject(), null));
destinationDefinitionIdRequestBody.getWorkspaceId(),
destinationDefinitionIdRequestBody.getDestinationDefinitionId(),
destinationDefinitionIdRequestBody.getRedirectUrl(), Jsons.emptyObject(), null));
}
try {
trackingClient.track(destinationOauthConsentRequest.getWorkspaceId(), "Get Oauth Consent URL - Backend", metadata);
trackingClient.track(destinationDefinitionIdRequestBody.getWorkspaceId(), "Get Oauth Consent URL - Backend", metadata);
} catch (final Exception e) {
LOGGER.error(ERROR_MESSAGE, e);
}
Expand Down Expand Up @@ -236,19 +195,6 @@ public void setDestinationInstancewideOauthParams(final SetInstancewideDestinati
configRepository.writeDestinationOAuthParam(param);
}

private JsonNode getOAuthInputConfigurationForConsent(final ConnectorSpecification spec,
final JsonNode hydratedSourceConnectionConfiguration,
final JsonNode oAuthInputConfiguration) {
final Map<String, String> fieldsToGet =
buildJsonPathFromOAuthFlowInitParameters(OAuthPathExtractor.extractOauthConfigurationPaths(
spec.getAdvancedAuth().getOauthConfigSpecification().getOauthUserInputFromConnectorConfigSpecification()));

final JsonNode oAuthInputConfigurationFromDB = getOAuthInputConfiguration(hydratedSourceConnectionConfiguration, fieldsToGet);

return getOauthFromDBIfNeeded(oAuthInputConfigurationFromDB,
oAuthInputConfiguration);
}

private Map<String, Object> generateSourceMetadata(final UUID sourceDefinitionId)
throws JsonValidationException, ConfigNotFoundException, IOException {
final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(sourceDefinitionId);
Expand All @@ -261,40 +207,4 @@ private Map<String, Object> generateDestinationMetadata(final UUID destinationDe
return TrackingMetadata.generateDestinationDefinitionMetadata(destinationDefinition);
}

@VisibleForTesting
Map<String, String> buildJsonPathFromOAuthFlowInitParameters(final Map<String, List<String>> oAuthFlowInitParameters) {
return oAuthFlowInitParameters.entrySet().stream()
.map(entry -> Map.entry(entry.getKey(), "$." + String.join(".", entry.getValue())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}

@VisibleForTesting
JsonNode getOauthFromDBIfNeeded(final JsonNode oAuthInputConfigurationFromDB, final JsonNode oAuthInputConfigurationFromInput) {
final Map<String, String> result = new HashMap<>();

Jsons.deserializeToStringMap(oAuthInputConfigurationFromInput)
.forEach((k, v) -> {
if (AirbyteSecretConstants.SECRETS_MASK.equals(v)) {
if (oAuthInputConfigurationFromDB.has(k)) {
result.put(k, oAuthInputConfigurationFromDB.get(k).textValue());
} else {
LOGGER.warn("Missing the key {} in the config store in DB", k);
}

} else {
result.put(k, v);
}
});

return Jsons.jsonNode(result);
}

@VisibleForTesting
JsonNode getOAuthInputConfiguration(final JsonNode hydratedSourceConnectionConfiguration, final Map<String, String> pathsToGet) {
return Jsons.jsonNode(pathsToGet.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,
entry -> JsonPaths.getSingleValue(hydratedSourceConnectionConfiguration, entry.getValue()).get())));
}

}
40 changes: 0 additions & 40 deletions airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/OAuthPathExtractor.java
View file

This file was deleted.

79 changes: 1 addition & 78 deletions airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,13 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.analytics.TrackingClient;
import io.airbyte.api.model.generated.SetInstancewideDestinationOauthParamsRequestBody;
import io.airbyte.api.model.generated.SetInstancewideSourceOauthParamsRequestBody;
import io.airbyte.commons.json.Jsons;
import io.airbyte.config.DestinationOAuthParameter;
import io.airbyte.config.SourceOAuthParameter;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.config.persistence.SecretsRepositoryReader;
import io.airbyte.validation.json.JsonValidationException;
import java.io.IOException;
import java.net.http.HttpClient;
Expand All @@ -36,7 +34,6 @@ class OAuthHandlerTest {
private OAuthHandler handler;
private TrackingClient trackingClient;
private HttpClient httpClient;
private SecretsRepositoryReader secretsRepositoryReader;
private static final String CLIENT_ID = "123";
private static final String CLIENT_ID_KEY = "client_id";
private static final String CLIENT_SECRET_KEY = "client_secret";
Expand All @@ -47,8 +44,7 @@ public void init() {
configRepository = Mockito.mock(ConfigRepository.class);
trackingClient = mock(TrackingClient.class);
httpClient = Mockito.mock(HttpClient.class);
secretsRepositoryReader = mock(SecretsRepositoryReader.class);
handler = new OAuthHandler(configRepository, httpClient, trackingClient, secretsRepositoryReader);
handler = new OAuthHandler(configRepository, httpClient, trackingClient);
}

@Test
Expand Down Expand Up @@ -155,77 +151,4 @@ void resetDestinationInstancewideOauthParams() throws JsonValidationException, I
assertEquals(oauthParameterId, capturedValues.get(1).getOauthParameterId());
}

@Test
void testBuildJsonPathFromOAuthFlowInitParameters() {
final Map<String, List<String>> input = Map.ofEntries(
Map.entry("field1", List.of("1")),
Map.entry("field2", List.of("2", "3")));

final Map<String, String> expected = Map.ofEntries(
Map.entry("field1", "$.1"),
Map.entry("field2", "$.2.3"));

assertEquals(expected, handler.buildJsonPathFromOAuthFlowInitParameters(input));
}

@Test
void testGetOAuthInputConfiguration() {
final JsonNode hydratedConfig = Jsons.deserialize(
"""
{
"field1": "1",
"field2": "2",
"field3": {
"field3_1": "3_1",
"field3_2": "3_2"
}
}
""");

final Map<String, String> pathsToGet = Map.ofEntries(
Map.entry("field1", "$.field1"),
Map.entry("field3_1", "$.field3.field3_1"),
Map.entry("field3_2", "$.field3.field3_2"));

final JsonNode expected = Jsons.deserialize(
"""
{
"field1": "1",
"field3_1": "3_1",
"field3_2": "3_2"
}
""");

assertEquals(expected, handler.getOAuthInputConfiguration(hydratedConfig, pathsToGet));
}

@Test
void testGetOauthFromDBIfNeeded() {
final JsonNode fromInput = Jsons.deserialize(
"""
{
"testMask": "**********",
"testNotMask": "this"
}
""");

final JsonNode fromDb = Jsons.deserialize(
"""
{
"testMask": "mask",
"testNotMask": "notThis"
}
""");

final JsonNode expected = Jsons.deserialize(
"""
{
"testMask": "mask",
"testNotMask": "this"
}
""");

assertEquals(expected, handler.getOauthFromDBIfNeeded(fromDb, fromInput));
}

}
Loading

0 comments on commit e68dd89

Please sign in to comment.