Implement Operator to pick up files from disk and put them into the

storage

dev/cosbench-controller-web/src/com/intel/cosbench/controller/web/WorkloadConfigurationController.java

@@ -308,9 +308,34 @@ private Stage constructNormalStage(HttpServletRequest req)
     		wOp.setConfig(wconfig);
 
     		work.addOperation(wOp);
-
-
-    		// delete operation
+
+            // filewrite operation
+            String fwconfig = "";
+            Operation fwOp = new Operation("filewrite");
+
+            int fwRatio = getParmInt(req, "filewrite.ratio", 0);
+            fwOp.setRatio(fwRatio);
+
+            // "containers" section in config
+            String fwcselector = getParm(req, "filewrite.containers");
+            String fwcmin = getParm(req, "filewrite.containers.min");
+            String fwcmax = getParm(req, "filewrite.containers.max");
+            fwconfig += "containers=" + fwcselector + "(" + fwcmin + "," + fwcmax + ");";
+
+            // "objects" section in config
+            String fwoselector = getParm(req, "filewrite.fileselection");
+            fwconfig += "fileselection=" + fwoselector + ";";
+
+
+            // "files" section in config
+            String fwfselector = getParm(req, "filewrite.files");
+            fwconfig += "files=" + fwfselector;
+
+            fwOp.setConfig(fwconfig);
+
+            work.addOperation(fwOp);
+
+            // delete operation
     		String dconfig = "";
     		Operation dOp = new Operation("delete");    		
 

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java

@@ -0,0 +1,142 @@
+/** 
+ 
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. 
+ */
+
+package com.intel.cosbench.driver.operator;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.CountingInputStream;
+
+import com.intel.cosbench.api.storage.StorageInterruptedException;
+import com.intel.cosbench.bench.Result;
+import com.intel.cosbench.bench.Sample;
+import com.intel.cosbench.config.Config;
+import com.intel.cosbench.driver.util.ContainerPicker;
+import com.intel.cosbench.driver.util.FilePicker;
+import com.intel.cosbench.service.AbortedException;
+
+/**
+ * This class represents a WRITE operation that reads a specified folder and writes the files contained.
+ * 
+ * @author Niklas Goerke niklas974@github
+ * 
+ */
+class FileWriter extends AbstractOperator {
+
+    public static final String OP_TYPE = "filewrite";
+
+    private ContainerPicker contPicker = new ContainerPicker();
+    private FilePicker filePicker = new FilePicker();
+
+    private File folder;
+
+    private File[] listOfFiles;
+
+    public FileWriter() {
+        /* empty */
+    }
+
+    @Override
+    protected void init(String division, Config config) {
+        super.init(division, config);
+        contPicker.init(division, config);
+        String filepath = config.get("files");
+        folder = new File(filepath);
+        listOfFiles = folder.listFiles();
+        System.out.println("listOfFiles.length: " + listOfFiles.length);
+        String range = "(1," + listOfFiles.length + ")";
+        System.out.println("initialising filePicker, range: " + range);
+        filePicker.init(range, config);
+    }
+
+    @Override
+    public String getOpType() {
+        return OP_TYPE;
+    }
+
+    @Override
+    protected void operate(int idx, int all, Session session) {
+        Sample sample;
+        if (!folder.canRead()) {
+            doLogErr(session.getLogger(), "fail to perform file filewrite operation, can not read " + folder.getAbsolutePath());
+            sample = new Sample(new Date(), OP_TYPE, false);
+        }
+
+        // looks kinda ugly, but does make sense…
+
+        Random random = session.getRandom();
+        String containerName = contPicker.pickContName(random, idx, all);
+
+        // as I said, the 1 from init() is being removed here
+        Integer rand = (filePicker.pickObjKey(random) - 1);
+        String filename = null;
+        try {
+            filename = listOfFiles[rand].getName();
+        } catch (ArrayIndexOutOfBoundsException e) {
+            doLogErr(session.getLogger(), "fail to perform file Write operation, tried to put more files than exist", e);
+            sample = new Sample(new Date(), OP_TYPE, false);
+            session.getListener().onSampleCreated(sample);
+            Date now = sample.getTimestamp();
+            Result result = new Result(now, OP_TYPE, sample.isSucc());
+            session.getListener().onOperationCompleted(result);
+            return;
+        }
+
+        try {
+            FileInputStream fis = new FileInputStream(listOfFiles[rand]);
+            sample = doWrite(fis, listOfFiles[rand].length(), containerName, filename, config, session);
+            System.out.println("rand: " + rand + " filename: " + filename + " container " + containerName);
+        } catch (FileNotFoundException e) {
+            doLogErr(session.getLogger(), "fail to perform file Write operation", e);
+            System.out.println("fail to perform file Write operation " + e.getMessage());
+            sample = new Sample(new Date(), OP_TYPE, false);
+        }
+        session.getListener().onSampleCreated(sample);
+        Date now = sample.getTimestamp();
+        Result result = new Result(now, OP_TYPE, sample.isSucc());
+        session.getListener().onOperationCompleted(result);
+    }
+
+    public static Sample doWrite(InputStream in, long length, String conName, String objName, Config config, Session session) {
+        if (Thread.interrupted())
+            throw new AbortedException();
+
+        CountingInputStream cin = new CountingInputStream(in);
+
+        long start = System.currentTimeMillis();
+
+        try {
+            session.getApi().createObject(conName, objName, cin, length, config);
+        } catch (StorageInterruptedException sie) {
+            throw new AbortedException();
+        } catch (Exception e) {
+            session.getLogger().error("fail to perform write operation", e);
+            return new Sample(new Date(), OP_TYPE, false);
+        } finally {
+            IOUtils.closeQuietly(cin);
+        }
+
+        long end = System.currentTimeMillis();
+
+        Date now = new Date(end);
+        return new Sample(now, OP_TYPE, true, end - start, cin.getByteCount());
+    }
+}

dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Operators.java

@@ -35,6 +35,8 @@ private static AbstractOperator createOperator(String type) {
             return new Reader();
         if (StringUtils.equals(type, Writer.OP_TYPE))
             return new Writer();
+        if (StringUtils.equals(type, FileWriter.OP_TYPE))
+            return new FileWriter();
         if (StringUtils.equals(type, Preparer.OP_TYPE))
             return new Preparer();
         if (StringUtils.equals(type, Cleaner.OP_TYPE))

dev/cosbench-driver/src/com/intel/cosbench/driver/random/NameGenerator.java

@@ -25,4 +25,6 @@ public interface NameGenerator {
 
     public String next(Random random, int idx, int all);
 
+    int nextKey(Random random);
+
 }

dev/cosbench-driver/src/com/intel/cosbench/driver/random/NumericNameGenerator.java

@@ -48,6 +48,11 @@ public String next(Random random) {
         int value = generator.next(random);
         return StringUtils.join(new Object[] { prefix, value, suffix });
     }
+
+    @Override
+    public int nextKey(Random random) {
+        return generator.next(random);
+    }    
 
     @Override
     public String next(Random random, int idx, int all) {

dev/cosbench-driver/src/com/intel/cosbench/driver/util/ContainerPicker.java

@@ -0,0 +1,64 @@
+/** 
+ 
+Copyright 2013 Intel Corporation, All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. 
+ */
+
+package com.intel.cosbench.driver.util;
+
+import static com.intel.cosbench.driver.util.Defaults.CONTAINER_PREFIX;
+import static com.intel.cosbench.driver.util.Defaults.CONTAINER_SUFFIX;
+import static com.intel.cosbench.driver.util.Division.CONTAINER;
+
+import java.util.Random;
+
+import com.intel.cosbench.config.Config;
+import com.intel.cosbench.driver.random.Generators;
+import com.intel.cosbench.driver.random.NameGenerator;
+
+/**
+ * This class encapsulates logic to pick names for the container
+ * 
+ * @author Niklas Goerke - niklas974@github
+ * 
+ */
+public class ContainerPicker {
+
+    private Division division;
+    private NameGenerator conNmGen;
+
+    public ContainerPicker() {
+        /* empty */
+    }
+
+    public void init(String division, Config config) {
+        conNmGen = getConNmGen(config);
+        this.division = Division.getDivision(division);
+    }
+
+    private static NameGenerator getConNmGen(Config config) {
+        String pattern = config.get("containers");
+        String prefix = config.get("cprefix", CONTAINER_PREFIX);
+        String suffix = config.get("csuffix", CONTAINER_SUFFIX);
+        return Generators.getNameGenerator(pattern, prefix, suffix);
+    }
+
+    public String pickContName(Random random, int idx, int all) {
+        if (division.equals(CONTAINER)) {
+            return conNmGen.next(random, idx, all);
+        } else {
+            return conNmGen.next(random);
+        }
+    }
+}

dev/cosbench-driver/src/com/intel/cosbench/driver/util/FilePicker.java

@@ -0,0 +1,48 @@
+/** 
+ 
+Copyright 2013 Intel Corporation, All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. 
+ */
+
+package com.intel.cosbench.driver.util;
+
+import java.util.Random;
+
+import com.intel.cosbench.config.Config;
+import com.intel.cosbench.driver.random.Generators;
+import com.intel.cosbench.driver.random.NameGenerator;
+
+/**
+ * This class encapsulates logic to pick integers for identififaction of files / objects
+ * 
+ * @author Niklas Goerke - niklas974@github
+ * 
+ */
+public class FilePicker {
+
+    private NameGenerator objNmGen;
+
+    public FilePicker() {
+        /* empty */
+    }
+
+    public void init(String range, Config config) {
+        String selector = config.get("fileselection").substring(0, 1);
+        objNmGen = Generators.getNameGenerator(selector + range, "", "");
+    }
+
+    public int pickObjKey(Random random) {
+        return objNmGen.nextKey(random);
+    }
+}