1. Fixes configuration bugs in codes.

2. Removes confilct black lists.
FederatedAI · Sep 4, 2019 · 5419c4b · 5419c4b
1 parent 0362b15
commit 5419c4b
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 20 deletions.
diff --git a/api/utils/eggroll_serdes.py b/api/utils/eggroll_serdes.py
@@ -63,21 +63,21 @@ def deserialize(_bytes):
         return p_loads(_bytes)
 
 
-deserialize_blacklist = [b'eval', b'execfile', b'compile', b'open', b'file', b'system', b'popen', b'popen2', b'popen3',
-                          b'popen4', b'fdopen', b'tmpfile', b'fchmod', b'fchown', b'open', b'openpty', b'pipe',
-                          b'chdir', b'fchdir', b'chroot', b'chmod', b'chown', b'link', b'lchown', b'listdir', b'lstat',
+deserialize_blacklist = [b'eval', b'execfile', b'compile', b'system', b'popen', b'popen2', b'popen3',
+                          b'popen4', b'fdopen', b'tmpfile', b'fchmod', b'fchown', b'openpty',
+                          b'chdir', b'fchdir', b'chroot', b'chmod', b'chown', b'lchown', b'listdir', b'lstat',
                           b'mkfifo', b'mknod', b'access', b'mkdir', b'makedirs', b'readlink', b'remove', b'removedirs',
                           b'rename', b'renames', b'rmdir', b'tempnam', b'tmpnam', b'unlink', b'execl',
                           b'execle', b'execlp', b'execv', b'execve', b'dup2', b'execvp', b'execvpe',
-                          b'forkpty', b'kill', b'spawnl', b'spawnle', b'spawnlp', b'spawnlpe', b'spawnv', b'spawnve',
-                          b'spawnvp', b'spawnvpe', b'load', b'loads', b'load', b'loads', b'call', b'check_call',
+                          b'forkpty', b'spawnl', b'spawnle', b'spawnlp', b'spawnlpe', b'spawnv', b'spawnve',
+                          b'spawnvp', b'spawnvpe', b'load', b'loads', b'call', b'check_call',
                           b'check_output', b'Popen', b'getstatusoutput', b'getoutput', b'getstatus',
-                          b'getline', b'copyfileobj', b'copyfile', b'copy', b'copy2', b'move', b'make_archive',
-                          b'listdir', b'opendir', b'open', b'popen2', b'popen3', b'popen4', b'timeit', b'repeat',
-                          b'call_tracing', b'interact', b'compile_command', b'compile_command', b'spawn', b'open',
-                          b'fileopen', b'popen']
+                          b'getline', b'copyfileobj', b'copyfile', b'copy', b'copy2', b'make_archive',
+                          b'listdir', b'opendir', b'timeit', b'repeat',
+                          b'call_tracing', b'interact', b'compile_command', b'spawn',
+                          b'fileopen']
 
-future_blacklist = [b'read', b'dup', b'fork', b'walk']
+future_blacklist = [b'read', b'dup', b'fork', b'walk', b'file', b'move', b'link', b'kill', b'open', b'pipe']
 
 serdes_cache = {}
 for cls in ABCSerdes.__subclasses__():

diff --git a/core/src/main/java/com/webank/ai/eggroll/core/utils/PropertyGetter.java b/core/src/main/java/com/webank/ai/eggroll/core/utils/PropertyGetter.java
@@ -1,13 +1,8 @@
 package com.webank.ai.eggroll.core.utils;
 
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
 import java.util.List;
 import java.util.Properties;
 
-@Component
-@Scope("prototype")
 public interface PropertyGetter {
     public boolean addSource(Properties prop);
     public List<Properties> getAllSources();

diff --git a/core/src/main/java/com/webank/ai/eggroll/core/utils/impl/PriorityPropertyGetter.java b/core/src/main/java/com/webank/ai/eggroll/core/utils/impl/PriorityPropertyGetter.java
@@ -4,20 +4,30 @@
 import com.webank.ai.eggroll.core.constant.StringConstants;
 import com.webank.ai.eggroll.core.utils.PropertyGetter;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import java.util.*;
 
-@Component
-@Scope("prototype")
+@Service("propertyGetter")
 public class PriorityPropertyGetter implements PropertyGetter {
+
     private final List<Properties> propertiesPriorityList;
+    private static final Logger LOGGER = LogManager.getLogger();
 
     public PriorityPropertyGetter() {
         this.propertiesPriorityList = Collections.synchronizedList(Lists.newArrayList());
     }
 
+    @PostConstruct
+    public void init() {
+/*        if (propertiesPriorityList.isEmpty() && serverConf.getProperties() != null) {
+            propertiesPriorityList.add(serverConf.getProperties());
+        }*/
+    }
+
     @Override
     public boolean addSource(Properties prop) {
         return propertiesPriorityList.add(prop);
@@ -45,9 +55,15 @@ public String getPropertyWithTemporarySource(String key, Properties... props) {
 
     @Override
     public String getPropertyWithTemporarySource(String key, String defaultValue, Properties... props) {
+        if (props == null) {
+            props = new Properties[0];
+        }
+
         ArrayList<Properties> appended = Lists.newArrayListWithCapacity(props.length + propertiesPriorityList.size());
+
         appended.addAll(Arrays.asList(props));
         appended.addAll(propertiesPriorityList);
+
         return getPropertyInIterable(key, defaultValue, appended);
     }
 
@@ -100,7 +116,7 @@ public String getPropertyInIterableInternal(CharSequence delimiter, String key,
         int curMatch = 0;
         String value;
         for (Properties prop : propsIter) {
-            if (prop == null) {
+            if (prop == null || prop.isEmpty()) {
                 continue;
             }
 

diff --git a/...work/egg/src/main/java/com/webank/ai/eggroll/framework/egg/manager/EggSessionManager.java b/...work/egg/src/main/java/com/webank/ai/eggroll/framework/egg/manager/EggSessionManager.java
@@ -8,6 +8,7 @@
 import com.webank.ai.eggroll.core.retry.factory.StopStrategies;
 import com.webank.ai.eggroll.core.retry.factory.WaitStrategies;
 import com.webank.ai.eggroll.core.retry.factory.WaitTimeStrategies;
+import com.webank.ai.eggroll.core.server.ServerConf;
 import com.webank.ai.eggroll.core.utils.PropertyGetter;
 import com.webank.ai.eggroll.core.utils.ToStringUtils;
 import com.webank.ai.eggroll.core.utils.TypeConversionUtils;
@@ -40,6 +41,8 @@ public class EggSessionManager {
     private EngineStatusTracker engineStatusTracker;
     @Autowired
     private ToStringUtils toStringUtils;
+    @Autowired
+    private ServerConf serverConf;
 
     @GuardedBy("sessionIdToResourceLock")
     private final Map<String, EggrollSession> sessionIdToResource;
@@ -89,6 +92,8 @@ public boolean getOrCreateSession(BasicMeta.SessionInfo sessionInfo) {
         int finalMaxSessionEngineCount = Math.max(maxSessionEngineCountInConf, 1);
         finalMaxSessionEngineCount = Math.min(finalMaxSessionEngineCount, 512);
 
+        LOGGER.info("[EGG][SESSIONMANAGER] maxSessionEngineCountInConf: {}, finalMaxSessionEngineCount: {}", maxSessionEngineCountInConf, finalMaxSessionEngineCount);
+
         ComputingEngine typeParamComputingEngine = new ComputingEngine(ComputingEngine.ComputingEngineType.EGGROLL);
         for (int i = 0; i < finalMaxSessionEngineCount; ++i) {
             ComputingEngine engine = engineOperator.start(typeParamComputingEngine, sessionProperties);