Skip to content

Commit

Permalink
Add secrets for pulling custom connector images (#20272)
Browse files Browse the repository at this point in the history 
* enable ui

* allow upload custom image

* remove fe change

* change pull secrets to accept a list instead

* comment fix

* remove unused var

* naming
  • Loading branch information
@xiaohansong
xiaohansong committed Dec 9, 2022
1 parent f5e15de commit d48d497
Show file tree
Hide file tree
Showing 6 changed files with 29 additions and 21 deletions.
16 changes: 8 additions & 8 deletions airbyte-commons-worker/src/main/java/io/airbyte/workers/WorkerConfigs.java
View file
Expand Up @@ -21,7 +21,7 @@ public class WorkerConfigs {
private final Map<String, String> workerKubeNodeSelectors;
private final Optional<Map<String, String>> workerIsolatedKubeNodeSelectors;
private final Map<String, String> workerKubeAnnotations;
private final String jobImagePullSecret;
private final List<String> jobImagePullSecrets;
private final String jobImagePullPolicy;
private final String sidecarImagePullPolicy;
private final String jobSocatImage;
Expand All @@ -45,7 +45,7 @@ public WorkerConfigs(final Configs configs) {
configs.getJobKubeNodeSelectors(),
configs.getUseCustomKubeNodeSelector() ? Optional.of(configs.getIsolatedJobKubeNodeSelectors()) : Optional.empty(),
configs.getJobKubeAnnotations(),
configs.getJobKubeMainContainerImagePullSecret(),
configs.getJobKubeMainContainerImagePullSecrets(),
configs.getJobKubeMainContainerImagePullPolicy(),
configs.getJobKubeSidecarContainerImagePullPolicy(),
configs.getJobKubeSocatImage(),
Expand Down Expand Up @@ -77,7 +77,7 @@ public static WorkerConfigs buildSpecWorkerConfigs(final Configs configs) {
nodeSelectors,
configs.getUseCustomKubeNodeSelector() ? Optional.of(configs.getIsolatedJobKubeNodeSelectors()) : Optional.empty(),
annotations,
configs.getJobKubeMainContainerImagePullSecret(),
configs.getJobKubeMainContainerImagePullSecrets(),
configs.getJobKubeMainContainerImagePullPolicy(),
configs.getJobKubeSidecarContainerImagePullPolicy(),
configs.getJobKubeSocatImage(),
Expand Down Expand Up @@ -109,7 +109,7 @@ public static WorkerConfigs buildCheckWorkerConfigs(final Configs configs) {
nodeSelectors,
configs.getUseCustomKubeNodeSelector() ? Optional.of(configs.getIsolatedJobKubeNodeSelectors()) : Optional.empty(),
annotations,
configs.getJobKubeMainContainerImagePullSecret(),
configs.getJobKubeMainContainerImagePullSecrets(),
configs.getJobKubeMainContainerImagePullPolicy(),
configs.getJobKubeSidecarContainerImagePullPolicy(),
configs.getJobKubeSocatImage(),
Expand Down Expand Up @@ -141,7 +141,7 @@ public static WorkerConfigs buildDiscoverWorkerConfigs(final Configs configs) {
nodeSelectors,
configs.getUseCustomKubeNodeSelector() ? Optional.of(configs.getIsolatedJobKubeNodeSelectors()) : Optional.empty(),
annotations,
configs.getJobKubeMainContainerImagePullSecret(),
configs.getJobKubeMainContainerImagePullSecrets(),
configs.getJobKubeMainContainerImagePullPolicy(),
configs.getJobKubeSidecarContainerImagePullPolicy(),
configs.getJobKubeSocatImage(),
Expand All @@ -162,7 +162,7 @@ public static WorkerConfigs buildReplicationWorkerConfigs(final Configs configs)
configs.getJobKubeNodeSelectors(),
configs.getUseCustomKubeNodeSelector() ? Optional.of(configs.getIsolatedJobKubeNodeSelectors()) : Optional.empty(),
configs.getJobKubeAnnotations(),
configs.getJobKubeMainContainerImagePullSecret(),
configs.getJobKubeMainContainerImagePullSecrets(),
configs.getJobKubeMainContainerImagePullPolicy(),
configs.getJobKubeSidecarContainerImagePullPolicy(),
configs.getJobKubeSocatImage(),
Expand Down Expand Up @@ -195,8 +195,8 @@ public Map<String, String> getWorkerKubeAnnotations() {
return workerKubeAnnotations;
}

public String getJobImagePullSecret() {
return jobImagePullSecret;
public List<String> getJobImagePullSecrets() {
return jobImagePullSecrets;
}

public String getJobImagePullPolicy() {
Expand Down
9 changes: 7 additions & 2 deletions airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubePodProcess.java
View file
Expand Up @@ -366,7 +366,7 @@ public KubePodProcess(final boolean isOrchestrator,
final Map<String, String> files,
final String entrypointOverride,
final ResourceRequirements resourceRequirements,
final String imagePullSecret,
final List<String> imagePullSecrets,
final List<TolerationPOJO> tolerations,
final Map<String, String> nodeSelectors,
final Map<String, String> labels,
Expand Down Expand Up @@ -515,8 +515,13 @@ public KubePodProcess(final boolean isOrchestrator,
podBuilder = podBuilder.withServiceAccount("airbyte-admin").withAutomountServiceAccountToken(true);
}

List<LocalObjectReference> pullSecrets = imagePullSecrets
.stream()
.map(imagePullSecret -> new LocalObjectReference(imagePullSecret))
.collect(Collectors.toList());

final Pod pod = podBuilder.withTolerations(buildPodTolerations(tolerations))
.withImagePullSecrets(new LocalObjectReference(imagePullSecret)) // An empty string turns this into a no-op setting.
.withImagePullSecrets(pullSecrets) // An empty list or an empty LocalObjectReference turns this into a no-op setting.
.withNodeSelector(nodeSelectors)
.withRestartPolicy("Never")
.withInitContainers(init)
Expand Down
2 changes: 1 addition & 1 deletion airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java
View file
Expand Up @@ -130,7 +130,7 @@ public Process create(
files,
entrypoint,
resourceRequirements,
workerConfigs.getJobImagePullSecret(),
workerConfigs.getJobImagePullSecrets(),
workerConfigs.getWorkerKubeTolerations(),
nodeSelectors,
allLabels,
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java
View file
Expand Up @@ -464,7 +464,7 @@ public interface Configs {
/**
* Define the Job pod connector image pull secret. Useful when hosting private images.
*/
String getJobKubeMainContainerImagePullSecret();
List<String> getJobKubeMainContainerImagePullSecrets();

/**
* Define the Memory request for the Sidecar
Expand Down
11 changes: 7 additions & 4 deletions airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java
View file
Expand Up @@ -738,12 +738,15 @@ public String getJobKubeSidecarContainerImagePullPolicy() {

/**
* Returns the name of the secret to be used when pulling down docker images for jobs. Automatically
* injected in the KubePodProcess class and used in the job pod templates. The empty string is a
* no-op value.
* injected in the KubePodProcess class and used in the job pod templates.
*
* Can provide multiple strings seperated by comma(,) to indicate pulling from different
* repositories. The empty string is a no-op value.
*/
@Override
public String getJobKubeMainContainerImagePullSecret() {
return getEnvOrDefault(JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET, "");
public List<String> getJobKubeMainContainerImagePullSecrets() {
String secrets = getEnvOrDefault(JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET, "");
return Arrays.stream(secrets.split(",")).collect(Collectors.toList());
}

@Override
Expand Down
10 changes: 5 additions & 5 deletions airbyte-workers/src/main/java/io/airbyte/workers/config/WorkerConfigurationBeanFactory.java
View file
Expand Up @@ -227,7 +227,7 @@ public WorkerConfigs checkWorkerConfigs(
@Named("isolatedNodeSelectors") final Map<String, String> isolatedNodeSelectors,
@Named("useIsolatedNodeSelector") final boolean useIsolatedNodeSelector,
@Named("checkJobKubeAnnotations") final Map<String, String> annotations,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final String mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final List<String> mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-policy}") final String mainContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.sidecar.container.image-pull-policy}") final String sidecarContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.images.socat}") final String socatImage,
Expand Down Expand Up @@ -261,7 +261,7 @@ public WorkerConfigs defaultWorkerConfigs(
@Named("isolatedNodeSelectors") final Map<String, String> isolatedNodeSelectors,
@Named("useIsolatedNodeSelector") final boolean useIsolatedNodeSelector,
@Named("defaultJobKubeAnnotations") final Map<String, String> annotations,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final String mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final List<String> mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-policy}") final String mainContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.sidecar.container.image-pull-policy}") final String sidecarContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.images.socat}") final String socatImage,
Expand Down Expand Up @@ -296,7 +296,7 @@ public WorkerConfigs discoverWorkerConfigs(
@Named("isolatedNodeSelectors") final Map<String, String> isolatedNodeSelectors,
@Named("useIsolatedNodeSelector") final boolean useIsolatedNodeSelector,
@Named("discoverJobKubeAnnotations") final Map<String, String> annotations,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final String mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final List<String> mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-policy}") final String mainContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.sidecar.container.image-pull-policy}") final String sidecarContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.images.socat}") final String socatImage,
Expand Down Expand Up @@ -330,7 +330,7 @@ public WorkerConfigs replicationWorkerConfigs(
@Named("isolatedNodeSelectors") final Map<String, String> isolatedNodeSelectors,
@Named("useIsolatedNodeSelector") final boolean useIsolatedNodeSelector,
@Named("defaultJobKubeAnnotations") final Map<String, String> annotations,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final String mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final List<String> mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-policy}") final String mainContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.sidecar.container.image-pull-policy}") final String sidecarContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.images.socat}") final String socatImage,
Expand Down Expand Up @@ -365,7 +365,7 @@ public WorkerConfigs specWorkerConfigs(
@Named("isolatedNodeSelectors") final Map<String, String> isolatedNodeSelectors,
@Named("useIsolatedNodeSelector") final boolean useIsolatedNodeSelector,
@Named("specJobKubeAnnotations") final Map<String, String> annotations,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final String mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-secret}") final List<String> mainContainerImagePullSecret,
@Value("${airbyte.worker.job.kube.main.container.image-pull-policy}") final String mainContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.sidecar.container.image-pull-policy}") final String sidecarContainerImagePullPolicy,
@Value("${airbyte.worker.job.kube.images.socat}") final String socatImage,
Expand Down

0 comments on commit d48d497

Please sign in to comment.