Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes several issue #47 #48

Merged
merged 3 commits into from Sep 4, 2019
Merged

Fixes several issue #47 #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1daf60
1. Removed several operations from blacklist.
maxwong Sep 3, 2019
0362b15
1. Removes more blacklists.
maxwong Sep 3, 2019
5419c4b
1. Fixes configuration bugs in codes.
maxwong Sep 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 13 additions & 11 deletions api/utils/eggroll_serdes.py
View file
Open in desktop
Expand Up @@ -63,19 +63,21 @@ def deserialize(_bytes):
return p_loads(_bytes)


deserialize_blacklist = [b'eval', b'execfile', b'compile', b'open', b'file', b'system', b'popen', b'popen2', b'popen3',
b'popen4', b'fdopen', b'tmpfile', b'fchmod', b'fchown', b'open', b'openpty', b'read', b'pipe',
b'chdir', b'fchdir', b'chroot', b'chmod', b'chown', b'link', b'lchown', b'listdir', b'lstat',
deserialize_blacklist = [b'eval', b'execfile', b'compile', b'system', b'popen', b'popen2', b'popen3',
b'popen4', b'fdopen', b'tmpfile', b'fchmod', b'fchown', b'openpty',
b'chdir', b'fchdir', b'chroot', b'chmod', b'chown', b'lchown', b'listdir', b'lstat',
b'mkfifo', b'mknod', b'access', b'mkdir', b'makedirs', b'readlink', b'remove', b'removedirs',
b'rename', b'renames', b'rmdir', b'tempnam', b'tmpnam', b'unlink', b'walk', b'execl',
b'execle', b'execlp', b'execv', b'execve', b'dup', b'dup2', b'execvp', b'execvpe', b'fork',
b'forkpty', b'kill', b'spawnl', b'spawnle', b'spawnlp', b'spawnlpe', b'spawnv', b'spawnve',
b'spawnvp', b'spawnvpe', b'load', b'loads', b'load', b'loads', b'call', b'check_call',
b'rename', b'renames', b'rmdir', b'tempnam', b'tmpnam', b'unlink', b'execl',
b'execle', b'execlp', b'execv', b'execve', b'dup2', b'execvp', b'execvpe',
b'forkpty', b'spawnl', b'spawnle', b'spawnlp', b'spawnlpe', b'spawnv', b'spawnve',
b'spawnvp', b'spawnvpe', b'load', b'loads', b'call', b'check_call',
b'check_output', b'Popen', b'getstatusoutput', b'getoutput', b'getstatus',
b'getline', b'copyfileobj', b'copyfile', b'copy', b'copy2', b'move', b'make_archive',
b'listdir', b'opendir', b'open', b'popen2', b'popen3', b'popen4', b'timeit', b'repeat',
b'call_tracing', b'interact', b'compile_command', b'compile_command', b'spawn', b'open',
b'fileopen', b'popen']
b'getline', b'copyfileobj', b'copyfile', b'copy', b'copy2', b'make_archive',
b'listdir', b'opendir', b'timeit', b'repeat',
b'call_tracing', b'interact', b'compile_command', b'spawn',
b'fileopen']

future_blacklist = [b'read', b'dup', b'fork', b'walk', b'file', b'move', b'link', b'kill', b'open', b'pipe']

serdes_cache = {}
for cls in ABCSerdes.__subclasses__():
Expand Down
5 changes: 0 additions & 5 deletions core/src/main/java/com/webank/ai/eggroll/core/utils/PropertyGetter.java
View file
Open in desktop
@@ -1,13 +1,8 @@
package com.webank.ai.eggroll.core.utils;

import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import java.util.List;
import java.util.Properties;

@Component
@Scope("prototype")
public interface PropertyGetter {
public boolean addSource(Properties prop);
public List<Properties> getAllSources();
Expand Down
26 changes: 21 additions & 5 deletions core/src/main/java/com/webank/ai/eggroll/core/utils/impl/PriorityPropertyGetter.java
View file
Open in desktop
Expand Up @@ -4,20 +4,30 @@
import com.webank.ai.eggroll.core.constant.StringConstants;
import com.webank.ai.eggroll.core.utils.PropertyGetter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import java.util.*;

@Component
@Scope("prototype")
@Service("propertyGetter")
public class PriorityPropertyGetter implements PropertyGetter {

private final List<Properties> propertiesPriorityList;
private static final Logger LOGGER = LogManager.getLogger();

public PriorityPropertyGetter() {
this.propertiesPriorityList = Collections.synchronizedList(Lists.newArrayList());
}

@PostConstruct
public void init() {
/* if (propertiesPriorityList.isEmpty() && serverConf.getProperties() != null) {
propertiesPriorityList.add(serverConf.getProperties());
}*/
}

@Override
public boolean addSource(Properties prop) {
return propertiesPriorityList.add(prop);
Expand Down Expand Up @@ -45,9 +55,15 @@ public String getPropertyWithTemporarySource(String key, Properties... props) {

@Override
public String getPropertyWithTemporarySource(String key, String defaultValue, Properties... props) {
if (props == null) {
props = new Properties[0];
}

ArrayList<Properties> appended = Lists.newArrayListWithCapacity(props.length + propertiesPriorityList.size());

appended.addAll(Arrays.asList(props));
appended.addAll(propertiesPriorityList);

return getPropertyInIterable(key, defaultValue, appended);
}

Expand Down Expand Up @@ -100,7 +116,7 @@ public String getPropertyInIterableInternal(CharSequence delimiter, String key,
int curMatch = 0;
String value;
for (Properties prop : propsIter) {
if (prop == null) {
if (prop == null || prop.isEmpty()) {
continue;
}

Expand Down
5 changes: 5 additions & 0 deletions ...work/egg/src/main/java/com/webank/ai/eggroll/framework/egg/manager/EggSessionManager.java
View file
Open in desktop
Expand Up @@ -8,6 +8,7 @@
import com.webank.ai.eggroll.core.retry.factory.StopStrategies;
import com.webank.ai.eggroll.core.retry.factory.WaitStrategies;
import com.webank.ai.eggroll.core.retry.factory.WaitTimeStrategies;
import com.webank.ai.eggroll.core.server.ServerConf;
import com.webank.ai.eggroll.core.utils.PropertyGetter;
import com.webank.ai.eggroll.core.utils.ToStringUtils;
import com.webank.ai.eggroll.core.utils.TypeConversionUtils;
Expand Down Expand Up @@ -40,6 +41,8 @@ public class EggSessionManager {
private EngineStatusTracker engineStatusTracker;
@Autowired
private ToStringUtils toStringUtils;
@Autowired
private ServerConf serverConf;

@GuardedBy("sessionIdToResourceLock")
private final Map<String, EggrollSession> sessionIdToResource;
Expand Down Expand Up @@ -89,6 +92,8 @@ public boolean getOrCreateSession(BasicMeta.SessionInfo sessionInfo) {
int finalMaxSessionEngineCount = Math.max(maxSessionEngineCountInConf, 1);
finalMaxSessionEngineCount = Math.min(finalMaxSessionEngineCount, 512);

LOGGER.info("[EGG][SESSIONMANAGER] maxSessionEngineCountInConf: {}, finalMaxSessionEngineCount: {}", maxSessionEngineCountInConf, finalMaxSessionEngineCount);

ComputingEngine typeParamComputingEngine = new ComputingEngine(ComputingEngine.ComputingEngineType.EGGROLL);
for (int i = 0; i < finalMaxSessionEngineCount; ++i) {
ComputingEngine engine = engineOperator.start(typeParamComputingEngine, sessionProperties);
Expand Down
17 changes: 8 additions & 9 deletions ...proxy/src/main/java/com/webank/ai/eggroll/networking/proxy/factory/GrpcServerFactory.java
View file
Open in desktop
Expand Up @@ -115,15 +115,14 @@ public Server createServer(ProxyServerConf proxyServerConf) {
.maxConnectionAge(24, TimeUnit.HOURS)
.maxConnectionAgeGrace(24, TimeUnit.HOURS);

if (proxyServerConf.isCompatibleEnabled()) {
AccessRedirector accessRedirector = new AccessRedirector();

serverBuilder.addService(accessRedirector.redirect(dataTransferPipedServer,
"com.webank.ai.eggroll.api.networking.proxy.DataTransferService",
"com.webank.ai.fate.api.networking.proxy.DataTransferService"))
.addService(accessRedirector.redirect(routeServer, "com.webank.ai.eggroll.api.networking.proxy.RouteService",
"com.webank.ai.fate.api.networking.proxy.RouteService"));
}
AccessRedirector accessRedirector = new AccessRedirector();

serverBuilder.addService(accessRedirector.redirect(dataTransferPipedServer,
"com.webank.ai.eggroll.api.networking.proxy.DataTransferService",
"com.webank.ai.fate.api.networking.proxy.DataTransferService"))
.addService(accessRedirector.redirect(routeServer, "com.webank.ai.eggroll.api.networking.proxy.RouteService",
"com.webank.ai.fate.api.networking.proxy.RouteService"));


if (proxyServerConf.isSecureServer()) {
String serverCrtPath = proxyServerConf.getServerCrtPath().replaceAll("\\.\\./", "");
Expand Down
2 changes: 1 addition & 1 deletion storage/storage-service-cxx/src/SKVServicer.cc
View file
Open in desktop
Expand Up @@ -133,7 +133,7 @@ Status SKVServicer::get(ServerContext *context, const Operand *request, Operand
string_view result = skvStore->get(request, value);

response->set_key(request->key());
response->set_value(value.data(), value.size());
response->set_value(result.data(), result.size());

LOG(INFO) << "get finished" << endl;
cout << "get finished" << ", value: " << value << endl;
Expand Down