if catch 409 error code when delete a unempty caontainer, will clean …

…up the container and try delete it again
intel-cloud · Dec 17, 2014 · 6d259d2 · 6d259d2
1 parent 1d313b6
commit 6d259d2
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 31 deletions.
diff --git a/dev/cosbench-api/src/com/intel/cosbench/api/auth/AuthAPI.java b/dev/cosbench-api/src/com/intel/cosbench/api/auth/AuthAPI.java
@@ -38,6 +38,10 @@ public interface AuthAPI {
      *            from execution engine.
      */
     public void init(Config config, Logger logger);
+
+    /**
+     * Initializes a new HttpClient for WorkAgent to relogin. 
+     */
     public void init();
 
     /**

diff --git a/dev/cosbench-api/src/com/intel/cosbench/api/storage/NoneStorage.java b/dev/cosbench-api/src/com/intel/cosbench/api/storage/NoneStorage.java
@@ -61,6 +61,7 @@ public void init(Config config, Logger logger) {
 
     @Override
     public void setAuthContext(AuthContext info) {
+    	setAuthFlag(true);
         /* empty */
     }
 

diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/agent/WorkAgent.java
@@ -295,7 +295,6 @@ public void reLogin() {
     		workerContext.getAuthApi().init();
     		authContext = workerContext.getAuthApi().login();
     		workerContext.getStorageApi().setAuthContext(authContext);
-    		workerContext.getStorageApi().setAuthFlag(true);
     		synchronized (AuthCachePool.getInstance()) {
 				AuthCachePool.getInstance().put(authContext.getID(), authContext);
 			}

diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/model/MissionContext.java
@@ -22,6 +22,7 @@
 
 import com.intel.cosbench.bench.*;
 import com.intel.cosbench.config.*;
+import com.intel.cosbench.config.common.KVConfigParser;
 import com.intel.cosbench.driver.util.OperationPicker;
 import com.intel.cosbench.log.LogManager;
 import com.intel.cosbench.model.*;
@@ -41,7 +42,7 @@ public class MissionContext implements MissionInfo {
     private StateRegistry stateHistory = new StateRegistry();
     private transient XmlConfig config;
     private transient volatile Future<?> future;
-
+    
     private Mission mission;
     private LogManager logManager;
     private ErrorStatistics errorStatistics;
@@ -54,7 +55,10 @@ public class MissionContext implements MissionInfo {
     private volatile Report report = null; // will be merged from worker reports
 
     private transient List<MissionListener> listeners = new ArrayList<MissionListener>();
-
+
+    private static final String GENERATE_HISTOGRAM_KEY = "histogram";
+    private static final boolean DEFAULT_GENERATE_HISTOGRAM = true;
+
     public MissionContext() {
         errorStatistics = new ErrorStatistics();
     }
@@ -95,7 +99,7 @@ private void fireMissionStopped() {
         for (MissionListener listener : listeners)
             listener.missionStopped(this);
     }
-
+    
     private Report mergeReport() {
         ReportMerger merger = new ReportMerger();
         for (WorkerContext worker : workerRegistry)
@@ -109,6 +113,27 @@ private Report mergeReport() {
         return report;
     }
 
+/*    private Report mergeReport() {
+        ReportMerger merger = new ReportMerger();
+        for (WorkerContext worker : workerRegistry)
+            merger.add(worker.getReport());
+        Report report = merger.merge();
+        Config missionConfig = KVConfigParser.parse(mission.getConfig());
+        boolean histogram = missionConfig.getBoolean(GENERATE_HISTOGRAM_KEY, DEFAULT_GENERATE_HISTOGRAM);
+        if(histogram) {
+        	generateHistogram(report);
+        }
+        return report;
+    }
+    
+    private void generateHistogram(Report report) {
+        OperatorRegistry registry = operatorRegistry;
+        for (Metrics metrics : report) {
+            OperatorContext op = registry.getOperator(metrics.getOpId());
+            metrics.setLatency(Histogram.convert(op.getCounter()));
+        }
+    }
+*/
     @Override
     public StateInfo[] getStateHistory() {
         return stateHistory.getAllStates();

diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/AbstractOperator.java
@@ -17,11 +17,11 @@
 
 package com.intel.cosbench.driver.operator;
 
+
 import java.util.HashMap;
 
-import com.intel.cosbench.config.Config;
-import com.intel.cosbench.api.storage.StorageException;
 import com.intel.cosbench.bench.ErrorStatistics;
+import com.intel.cosbench.config.Config;
 import com.intel.cosbench.log.LogFactory;
 import com.intel.cosbench.log.Logger;
 
@@ -136,9 +136,8 @@ public static void isUnauthorizedException(Exception e, Session session) {
     				LOGGER.debug("catch 401 error from storage backend, set auth flag to false");
     			}
     		}catch(NumberFormatException ne) {
-    			ne.printStackTrace(); // mask ignore
+    			ne.printStackTrace();// mask ignore
     		}
     }
 
-
 }
diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Cleaner.java
@@ -19,8 +19,13 @@
 
 import static com.intel.cosbench.driver.operator.Deleter.doDelete;
 
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 
 import com.intel.cosbench.api.storage.StorageException;
@@ -94,24 +99,82 @@ protected void operate(int idx, int all, Session session) {
         session.getListener().onOperationCompleted(result);
     }
 
-    public static void doDispose(String conName, Config config, Session session) {
+    public  void doDispose(String conName, Config config, Session session) {
         if (Thread.interrupted())
             throw new AbortedException();
-
-        try {
-            session.getApi().deleteContainer(conName, config);
-        } catch (StorageInterruptedException sie) {
-            throw new AbortedException();
-        } catch (StorageException se) {
-            String msg = "Error deleting container " +  conName; 
-            doLogWarn(session.getLogger(), msg);
-            // ignored
-        } catch (Exception e) {
-            doLogErr(session.getLogger(), "fail to perform clean operation " + conName, e);
-            throw new AgentException(); // mark error
-        }
-
-        /* no sample is provided for this operation */
+
+        boolean isEmpty = true;
+        boolean tryAgain = false;
+        do {
+        	if(isEmpty) {
+        		 try {
+        	            session.getApi().deleteContainer(conName, config);
+        	            isEmpty = true;
+        	            tryAgain = false;
+        	        } catch (StorageInterruptedException sie) {
+        	            throw new AbortedException();
+        	        } catch (StorageException se) {
+        	            String msg = "Error deleting container " +  conName; 
+        	            doLogErr(session.getLogger(), msg,se);
+        	            if(isConflictException(session,se)) {
+        	            	isEmpty = false;
+        	            	tryAgain = true;
+        	            }else{
+        	            	tryAgain = false;
+        	            }
+        	        } catch (Exception e) {
+        	            doLogErr(session.getLogger(), "fail to perform clean operation " + conName, e);
+        	            throw new AgentException(); // mark error
+        	        }
+        	}else {
+        		try{
+        			for(String objName: getObjectsList(conName, config, session)) {
+        				doDelete(conName, objName, config, session, this);
+        			}
+        		  }catch(StorageException se) {
+        			  doLogErr(session.getLogger(), "fail to get : "+conName+" objects list ",se);
+        			  tryAgain = false;
+        		  }catch (IOException ioe) {
+        			  doLogErr(session.getLogger(), "fail to convert objects stream to string",ioe);
+        			  tryAgain = false;
+        		  }catch (Exception e) {
+        			  doLogErr(session.getLogger(), "unexpected error",e);
+        			  tryAgain = false;
+        		  }
+        		isEmpty = true;
+        		tryAgain = true;
+        	}
+        }while(tryAgain == true);
+
+    }
+
+    private static String[] getObjectsList(String conName, Config config, Session session) throws IOException {
+    	String[] objects = {};
+    	StringWriter stringWriter = new StringWriter();
+    	try {
+			IOUtils.copy(session.getApi().getList(conName, "", config), stringWriter);
+		}catch(StorageException se) {
+			throw se;
+		}catch (IOException e) {
+			throw e;
+		}
+    	String objectString = stringWriter.toString();
+    	objects = objectString.split("\n");
+    	return objects;
+    }
+
+    private static boolean isConflictException(Session session, Exception e) {
+    	if(e != null && e.getMessage() != null)
+    		try{
+    			if(409 == Integer.valueOf(e.getMessage().substring(9, 12))){
+    				doLogDebug(session.getLogger(),"catch 409 error, will clean up the unempty container and try again");
+    				return true;
+    			}
+    		}catch(NumberFormatException ne) {
+    			ne.printStackTrace(); // mask ignore
+    			return false;
+    		}
+    	return false;
     }
 
 }
diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/service/COSBDriverService.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/service/COSBDriverService.java
@@ -43,16 +43,16 @@
 class COSBDriverService implements DriverService, MissionListener {
 
     private static final Logger LOGGER = LogFactory.getSystemLogger();
-
+    
     private DriverContext context;
     private Map<String, MissionHandler> handlers;
-
+    
     private AuthAPIService authAPIs;
     private StorageAPIService storageAPIs;
 
     private ExecutorService executor;
     private MissionRepository memRepo = new RAMMissionRepository();
-
+    
     public COSBDriverService() {
         /* empty */
     }
@@ -74,7 +74,7 @@ public void init() {
         handlers = Collections.synchronizedMap(handlers);
         executor = Executors.newCachedThreadPool();
     }
-
+    
     @Override
     public String submit(XmlConfig config) {
         LOGGER.debug("submitting mission ... ");

diff --git a/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java b/dev/cosbench-swift/src/com/intel/cosbench/client/swift/SwiftClient.java
@@ -27,7 +27,6 @@
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.*;
 import org.apache.http.entity.*;
-
 import com.intel.cosbench.client.http.HttpClientUtil;
 import com.intel.cosbench.log.*;
 

diff --git a/release/conf/controller.conf b/release/conf/controller.conf
@@ -1,6 +1,6 @@
 [controller]
 drivers = 1
-log_level = INFO
+log_level = DEBUG
 log_file = log/system.log
 archive_dir = archive
 

diff --git a/release/conf/driver.conf b/release/conf/driver.conf
@@ -1,2 +1,2 @@
 [driver]
-log_level = INFO
+log_level = DEBUG