Skip to content

[ZEPPELIN-4219] User can start interpreter when interpreter dependencies jars are not downloaded #3398

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zjffdu wants to merge 1 commit into from
Closed

[ZEPPELIN-4219] User can start interpreter when interpreter dependencies jars are not downloaded #3398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public List<File> load(String artifact, Collection<String> excludes, File destPa
File destFile = new File(destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.debug("copy {} to {}", srcFile.getAbsolutePath(), destPath);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
}
}
}
Expand All @@ -114,7 +114,7 @@ public synchronized void copyLocalDependency(String srcPath, File destPath)

if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.debug("copy {} to {}", srcFile.getAbsolutePath(), destPath);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
}
}

Expand Down Expand Up @@ -142,7 +142,7 @@ private List<File> loadFromMvn(String artifact, Collection<String> excludes)
List<File> files = new LinkedList<>();
for (ArtifactResult artifactResult : listOfArtifact) {
files.add(artifactResult.getArtifact().getFile());
logger.debug("load {}", artifactResult.getArtifact().getFile().getAbsolutePath());
logger.info("load {}", artifactResult.getArtifact().getFile().getAbsolutePath());
}

return files;
Expand Down
6 changes: 4 additions & 2 deletions zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public class InterpreterSetting {
*/
private Object properties = new Properties();

private Status status;
private Status status = Status.READY;
private String errorReason;

@SerializedName("interpreterGroup")
Expand Down Expand Up @@ -265,7 +265,6 @@ public InterpreterSetting() {
}

void postProcessing() {
this.status = Status.READY;
this.id = this.name;
if (this.lifecycleManager == null) {
this.lifecycleManager = new NullLifecycleManager(conf);
Expand Down Expand Up @@ -657,6 +656,7 @@ public Status getStatus() {
}

public void setStatus(Status status) {
LOGGER.info(String.format("Set interpreter %s status to %s", name, status.name()));
this.status = status;
}

Expand Down Expand Up @@ -867,6 +867,7 @@ public void run() {
// load dependencies
List<Dependency> deps = getDependencies();
if (deps != null) {
LOGGER.info("Start to download dependencies for interpreter: " + name);
for (Dependency d : deps) {
File destDir = new File(
conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
Expand All @@ -879,6 +880,7 @@ public void run() {
.load(d.getGroupArtifactVersion(), new File(destDir, id));
}
}
LOGGER.info("Finish downloading dependencies for interpreter: " + name);
}

setStatus(Status.READY);
Expand Down
52 changes: 24 additions & 28 deletions ...elin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -639,35 +639,30 @@ public void removeResourcesBelongsToNote(String noteId) {
* changed
*/
private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
final Thread t = new Thread() {
public void run() {
try {
List<Dependency> deps = setting.getDependencies();
if (deps != null) {
for (Dependency d : deps) {
File destDir = new File(
conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));

int numSplits = d.getGroupArtifactVersion().split(":").length;
if (!(numSplits >= 3 && numSplits <= 6)) {
dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
new File(destDir, setting.getId()));
}
}
try {
List<Dependency> deps = setting.getDependencies();
if (deps != null) {
LOGGER.info("Start to copy dependencies for interpreter: " + setting.getName());
for (Dependency d : deps) {
File destDir = new File(
conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));

int numSplits = d.getGroupArtifactVersion().split(":").length;
if (!(numSplits >= 3 && numSplits <= 6)) {
dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
new File(destDir, setting.getId()));
}
} catch (Exception e) {
LOGGER.error(String.format("Error while copying deps for interpreter group : %s," +
" go to interpreter setting page click on edit and save it again to make " +
"this interpreter work properly.",
setting.getGroup()), e);
setting.setErrorReason(e.getLocalizedMessage());
setting.setStatus(InterpreterSetting.Status.ERROR);
} finally {

}
LOGGER.info("Finish copy dependencies for interpreter: " + setting.getName());
}
};
t.start();
} catch (Exception e) {
LOGGER.error(String.format("Error while copying deps for interpreter group : %s," +
" go to interpreter setting page click on edit and save it again to make " +
"this interpreter work properly.",
setting.getGroup()), e);
setting.setErrorReason(e.getLocalizedMessage());
setting.setStatus(InterpreterSetting.Status.ERROR);
}
}

/**
Expand Down Expand Up @@ -777,8 +772,8 @@ public void removeRepository(String id) throws IOException {
dependencyResolver.delRepo(id);
saveToFile();
}

/** Change interpreter properties and restart */
/** restart in interpreter setting page */
private InterpreterSetting inlineSetPropertyAndRestart(
String id,
InterpreterOption option,
Expand Down Expand Up @@ -838,6 +833,7 @@ public void restart(String settingId, String noteId, String user) throws Interpr
}
}

@VisibleForTesting
public void restart(String id) throws InterpreterException {
InterpreterSetting setting = interpreterSettings.get(id);
copyDependenciesFromLocalPath(setting);
Expand Down