Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ZEPPELIN-3825] Allow custom service account for GCSNotebookRepo #3207

Closed
wants to merge 2 commits into from
Closed

[ZEPPELIN-3825] Allow custom service account for GCSNotebookRepo #3207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
84b8da4
ZEPPELIN-3825. Allow custom service account for GCSNotebookRepo
Oct 22, 2018
15c8aa0
Merge branch 'master' into ZEPPELIN-3825
Nov 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 9 additions & 0 deletions conf/zeppelin-site.xml.template
View file
Open in desktop
Expand Up @@ -84,6 +84,15 @@
</description>
</property>

<property>
<name>zeppelin.notebook.gcs.credentialsJsonFilePath</name>
<value>path/to/key.json</value>
<description>
Path to GCS credential key file for authentication with Google Storage.
</description>
</property>


<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
Expand Down
18 changes: 16 additions & 2 deletions docs/setup/storage/storage.md
View file
Open in desktop
Expand Up @@ -317,7 +317,7 @@ Or, if you want to simultaneously use your local git storage with GCS, use the f
### Google Cloud API Authentication

Note: On Google App Engine, Google Cloud Shell, and Google Compute Engine, these
steps are not necessary, as build-in credentials are used by default.
steps are not necessary if you are using the default built in service account.

For more information, see [Application Default Credentials](https://cloud.google.com/docs/authentication/production)

Expand Down Expand Up @@ -351,11 +351,25 @@ for authentication with GCS, you will need a JSON service account key file.
`/path/to/my/key.json`), and give it appropriate permissions. Ensure at
least the user running the zeppelin daemon can read it.

Then, point `GOOGLE_APPLICATION_CREDENTIALS` at your new key file in **zeppelin-env.sh**. For example:
If you wish to set this as your default credential file to access Google Services,
point `GOOGLE_APPLICATION_CREDENTIALS` at your new key file in **zeppelin-env.sh**. For example:

```bash
export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json
```
If you do not want to use this key file as default credential file and want to specify a custom key
file for authentication with GCS, update the following property :

```xml
<property>
<name>zeppelin.notebook.google.credentialsJsonFilePath</name>
<value>path/to/key.json</value>
<description>
Path to GCS credential key file for authentication with Google Storage.
</description>
</property>
```


</br>
## Notebook Storage in ZeppelinHub <a name="ZeppelinHub"></a>
Expand Down
1 change: 1 addition & 0 deletions zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file
Open in desktop
Expand Up @@ -738,6 +738,7 @@ public enum ConfVars {
// whether homescreen notebook will be hidden from notebook list or not
ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR("zeppelin.notebook.gcs.dir", ""),
ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE("zeppelin.notebook.google.credentialsJsonFilePath", null),
ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
ZEPPELIN_NOTEBOOK_S3_TIMEOUT("zeppelin.notebook.s3.timeout", "120000"),
Expand Down
15 changes: 13 additions & 2 deletions ...ins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
View file
Open in desktop
Expand Up @@ -17,6 +17,8 @@

package org.apache.zeppelin.notebook.repo;

import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
Expand All @@ -29,6 +31,8 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.gson.JsonParseException;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -54,7 +58,9 @@
* object store, so this "directory" should not itself be an object. Instead, it represents the base
* path for the note.json files.
*
* Authentication is provided by google-auth-library-java.
* Authentication is provided by google-auth-library-java. A custom json key file path
* can be specified by zeppelin.notebook.google.credentialsJsonFilePath to connect with GCS
* If not specified the GOOGLE_APPLICATION_CREDENTIALS will be used to connect to GCS.
* @see <a href="https://github.com/google/google-auth-library-java">
* google-auth-library-java</a>.
*/
Expand Down Expand Up @@ -113,7 +119,12 @@ public void init(ZeppelinConfiguration zConf) throws IOException {
this.notePathPattern = Pattern.compile("^(.+\\.zpln)$");
}

this.storage = StorageOptions.getDefaultInstance().getService();
Credentials credentials = GoogleCredentials.getApplicationDefault();
String credentialJsonPath = zConf.getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE);
if (credentialJsonPath != null) {
credentials = GoogleCredentials.fromStream(new FileInputStream(credentialJsonPath));
}
this.storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();
}

private BlobId makeBlobId(String noteId, String notePath) throws IOException {
Expand Down
15 changes: 13 additions & 2 deletions .../notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
View file
Open in desktop
Expand Up @@ -17,6 +17,8 @@

package org.apache.zeppelin.notebook.repo;

import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
Expand All @@ -39,6 +41,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -56,7 +59,9 @@
* object store, so this "directory" should not itself be an object. Instead, it represents the base
* path for the note.json files.
*
* Authentication is provided by google-auth-library-java.
* Authentication is provided by google-auth-library-java. A custom json key file path
* can be specified by zeppelin.notebook.google.credentialsJsonFilePath to connect with GCS
* If not specified the GOOGLE_APPLICATION_CREDENTIALS will be used to connect to GCS.
* @see <a href="https://github.com/google/google-auth-library-java">
* google-auth-library-java</a>.
*/
Expand Down Expand Up @@ -115,7 +120,13 @@ public void init(ZeppelinConfiguration zConf) throws IOException {
this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$");
}

this.storage = StorageOptions.getDefaultInstance().getService();

Credentials credentials = GoogleCredentials.getApplicationDefault();
String credentialJsonPath = zConf.getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE);
if (credentialJsonPath != null) {
credentials = GoogleCredentials.fromStream(new FileInputStream(credentialJsonPath));
}
this.storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();
}

private BlobId makeBlobId(String noteId) {
Expand Down