Merge branch 'develop' for v0.2.0

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v0.2.0
+
+- Add Spack package for pdwfs [b6cb1f2]
+- Update README.md [1e062df]
+- Add an example for using pdwfs with SLURM job scheduler [b2839c1]
+- Update SLURM helpers scripts [3aca9ae]
+- Increase default stripe size to 50MB [9907704]
+- Merge branch 'feature/custom-redigo-client' into develop [6705d0c]
+- Fix performance issues on write and read [8a54135]
+- Optimize write with Redis Set cmd if whole buffer is to be written (faster) [bd34304]
+- Refactor tests to improve isolation and avoid need for running Redis beforehand [9ea7596]
+- Refactor the C layer to move all "triage" into C layer and minimize CGo cross-layer calls [1f53f0c]
+
 ## v0.1.2
 
 - Refactor the inode layer to remove a lock [0b573de]

Makefile

@@ -3,7 +3,7 @@ BUILDDIR=build
 #install dir
 PREFIX ?= /usr/local
 
-all: dirs pdwfslibc scripts
+all: dirs pdwfslibc tools
 
 dirs:
 	mkdir -p $(BUILDDIR)
@@ -14,13 +14,17 @@ pdwfslibc: pdwfsgo
 pdwfsgo:
 	make -C src/go
 
-.PHONY: scripts
-scripts: scripts/pdwfs 
+.PHONY: tools
+tools: tools/pdwfs 
 	mkdir -p $(BUILDDIR)/bin
-	install scripts/pdwfs $(BUILDDIR)/bin/
-	install scripts/pdwfs-redis $(BUILDDIR)/bin/	 
+	install tools/pdwfs $(BUILDDIR)/bin/
+	install tools/pdwfs-slurm $(BUILDDIR)/bin/
+	install tools/redis.srun $(BUILDDIR)/bin/
+	#cd tools/pv && ./configure && make  # if pv needs rebuild
+	install tools/pv/pv $(BUILDDIR)/bin/
 
-test: scripts pdwfslibc
+
+test: tools pdwfslibc
 	make -C src/go test
 	make -C src/c test
 
@@ -32,12 +36,16 @@ install: pdwfslibc
 	install -d $(PREFIX)/lib $(PREFIX)/bin
 	install $(BUILDDIR)/lib/libpdwfs_go.so $(PREFIX)/lib
 	install $(BUILDDIR)/lib/pdwfs.so $(PREFIX)/lib
-	install scripts/pdwfs $(PREFIX)/bin
-	install scripts/pdwfs-redis $(PREFIX)/bin
+	install tools/pdwfs $(PREFIX)/bin
+	install tools/pdwfs-slurm $(PREFIX)/bin
+	install tools/redis.srun $(PREFIX)/bin
 	chmod +x $(PREFIX)/bin/*
 
 tag:
 	git tag -a $(TAG) -m "Version $(TAG)"
 
 dist:
-	scripts/create_tarballs.sh
+	tools/create_tarballs.sh
+
+bench:
+	make -C examples/ior_benchmark bench

docker/Dockerfile

@@ -1,4 +1,20 @@
-FROM golang:1.11.5 
+FROM centos:latest 
+
+RUN yum -y update && yum -y install \
+	wget \
+	gcc \
+	make \
+	strace \
+	git \
+	glib2-devel \
+	yum clean all
+
+# Go language
+RUN wget -O go.tar.gz 'https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz' && \
+	tar xf go.tar.gz -C /usr/local && \
+	rm go.tar.gz
+
+ENV PATH "/usr/local/go/bin:$PATH"
 
 ENV GO111MODULE=on
 

docker/Makefile

@@ -8,6 +8,9 @@ build:
 run: build
 	docker run $(DOCKER_RUN_OPT) -it --rm -v $(shell pwd)/..:/home/dev/pdwfs -w /home/dev/pdwfs --name pdwfs-dev pdwfs
 
+connect:
+	docker exec -it pdwfs-dev bash
+
 clean:
 	docker rm $(shell docker ps -qa --no-trunc --filter "status=exited"); \
 	docker rmi $(shell docker images --filter "dangling=true" -q --no-trunc)

docker/banner.sh

@@ -4,7 +4,6 @@ echo "*                                                           *"
 echo "* To build pdwfs:                                           *"
 echo "* $ make                                                    *"
 echo "*                                                           *"  
-echo "* To run the test suite you need a running Redis instance:  *"
-echo "* $ scripts/pdwfs-redis start                               *"
+echo "* To run the test suite:                                    *"
 echo "* $ make test                                               *"
 echo "*************************************************************"

examples/base_dockerfile/Dockerfile

@@ -8,6 +8,7 @@ RUN yum -y update && yum -y install \
 	make \
 	strace \
 	git \
+	glib2-devel \
 	yum clean all
 
 # Go language

examples/hydroC_ParaView/Dockerfile

@@ -42,9 +42,9 @@ RUN cd src && git clone 'https://github.com/cea-hpc/pdwfs' && \
 
 ENV PATH "${HOME}/opt/pdwfs/bin:${PATH}"
 
-# uncomment to build pdwfs from a volume mounted on the pdwfs source directory on the host (for debug/development)
-# (a modification is needed in the Makefile as well)
-#ENV PATH "/pdwfs/build/bin:${PATH}"
+# pdwfs bin will be first search in /pdwfs/build which is a local build directory (on the host, not in the container, /pdwfs is a mounted volume)
+# if no bin is found, it will look into the container installed version checked out from the GitHub repo
+ENV PATH "/pdwfs/build/bin:${PATH}"
 
 COPY banner.sh /tmp/
 RUN cat /tmp/banner.sh >> ${HOME}/.bashrc

examples/hydroC_ParaView/Makefile

@@ -1,7 +1,7 @@
 
 # uncomment to mount pdwfs sources folder into the container (for debug/development)
 # a modification is needed in the Dockerfile as well
-#DOCKER_RUN_OPT = -v $(shell pwd)/../../:/pdwfs
+DOCKER_RUN_OPT = -v $(shell pwd)/../../:/pdwfs
 
 # uncomment to allow stracing in docker (for debug)
 #DOCKER_RUN_OPT = --security-opt seccomp:unconfined $(DOCKER_RUN_OPT)
@@ -16,19 +16,6 @@ run: build
 connect:
 	docker exec -it hydro-run bash
 
-
-define BROWSER_PYSCRIPT
-import os, webbrowser, sys
-try:
-	from urllib import pathname2url
-except:
-	from urllib.request import pathname2url
-webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
-endef
-export BROWSER_PYSCRIPT
-
-BROWSER := python -c "$$BROWSER_PYSCRIPT"
-
 watch:
-	@echo "<video preload controls autoplay loop src=hydro.mp4></video>" > output/video.html && $(BROWSER) output/video.html
+	@echo "<video preload controls autoplay loop src=hydro.mp4></video>" > output/video.html && python -m webbrowser -t "output/video.html"
 

examples/hydroC_ParaView/README.md

@@ -1,17 +1,16 @@
 # Workflow Example: HydroC + ParaView + FFmpeg
 
-This example illustrates the use of pdwfs to produce movie images  from a simulation without writing any simulation output data on disk.
+This example illustrates the use of pdwfs to produce movie images  from an [HydroC](https://github.com/HydroBench/Hydro) simulation without writing any simulation data on disk.
 
-It can run on a laptop and only requires Docker:
+The example runs on a laptop and everything you need is packaged in a Dockerfile.
 
+We also provide a Makefile which does the plumbing for you. So, to build and run the container, just type:
 ```
 $ git clone https://github.com/cea-hpc/pdwfs
 $ cd pdwfs
 $ make -C examples/hydroC_ParaView run
 ```
-The last command will build the container the first time it is executed. As it is compiling OpenMPI and downloading Go and ParaView, you'll have time to grab some coffee...
-
-Once the build is complete, it runs the container. Just follow the help message to run the workflow with pdwfs.
+Once you are in the container, just follow the help message to run the workflow with pdwfs.
 
 Once the workflow has run, go back to your host (not the container) and type the following command to watch the movie you just produced:
 

examples/hydroC_ParaView/run_on_pdwfs.sh

@@ -1,7 +1,11 @@
 #!/bin/bash
 
-# Start a Redis instance in background to stage data (+deactivate Redis snapshotting)
-redis-server --daemonize yes --save ""
+# Start two Redis instances in background to stage data (+deactivate Redis snapshotting)
+redis-server --port 6379 --daemonize yes --save ""
+redis-server --port 6380 --daemonize yes --save ""
+
+# Export server addresses to pdwfs
+export PDWFS_REDIS=localhost:6379,localhost:6380
 
 # Wrap the simulation command with pdwfs intercepting IO in Dep/ directory
 # Note: redirecting stderr to /dev/null as a read error appears at each iteration 
@@ -16,4 +20,5 @@ pdwfs -p Dep -- ./process_all.py
 ffmpeg -i images/test_%02d.jpg -vcodec libx264 -pix_fmt yuv420p -profile:v baseline -level 3 /output/hydro.mp4
 
 # No data to clean up on disk, just shutdown Redis
-#redis-cli shutdown
+#redis-cli -p 6379 shutdown
+#redis-cli -p 6380 shutdown

examples/ior_benchmark/Dockerfile

@@ -40,19 +40,18 @@ RUN mkdir -p ${HOME}/run
 RUN git clone 'https://github.com/cea-hpc/pdwfs' && \
 	cd pdwfs && make PREFIX=${HOME}/opt/pdwfs install
 
-ENV PATH "${HOME}/opt/pdwfs:${PATH}"
-
-# uncomment to build pdwfs from a volume mounted on the pdwfs source directory on the host (for debug/development)
-# (a modification is needed in the Makefile as well)
-#ENV PATH "/pdwfs/build/bin:${PATH}"
-
+# pdwfs bin will be first search in /pdwfs/build which is a local build directory (on the host, not in the container, /pdwfs is a mounted volume)
+# if no bin is found, it will look into the container installed version checked out from the GitHub repo
+ENV PATH "/pdwfs/build/bin:${HOME}/opt/pdwfs/bin:${PATH}"
 
 COPY banner.sh /tmp/
 RUN cat /tmp/banner.sh >> ${HOME}/.bashrc
 
 COPY --chown=luke:rebels jupyter_notebook_config.py ${HOME}/.jupyter/
+COPY --chown=luke:rebels ior_script.jinja2 .
+COPY --chown=luke:rebels utils.py .
+COPY --chown=luke:rebels bench.py .
 COPY --chown=luke:rebels notebook/ior_example.ipynb .
-COPY --chown=luke:rebels notebook/ior_script.jinja2 .
 
 CMD bash
 

examples/ior_benchmark/Makefile

@@ -1,17 +1,22 @@
 
 # uncomment to mount pdwfs sources folder into the container (for debug/development)
-# a modification is needed in the Dockerfile as well
 DOCKER_RUN_OPT = -v $(shell pwd)/../../:/pdwfs
 
 # uncomment to allow stracing in docker (for debug)
 #DOCKER_RUN_OPT = --security-opt seccomp:unconfined $(DOCKER_RUN_OPT)
 
+DOCKER_RUN = docker run $(DOCKER_RUN_OPT) -it --rm -p 8888:8888 -v $(shell pwd)/output:/output -v $(shell pwd)/notebook:/notebook --name ior-run ior
+
 build:
 	docker build -t pdwfs-base ../base_dockerfile
 	docker build -t ior .
 
 run: build
-	docker run $(DOCKER_RUN_OPT) -it --rm -p 8888:8888 -v $(shell pwd)/output:/output -v $(shell pwd)/notebook:/notebook --name ior-run ior
+	$(DOCKER_RUN) jupyter-notebook
+
+bench: build
+	$(DOCKER_RUN) ./bench.py $(shell git describe)	
+	python -m webbrowser -t "output/bench.html"
 
 connect:
 	docker exec -it ior-run bash

examples/ior_benchmark/README.md

@@ -0,0 +1,21 @@
+# IOR Benchmark Example
+
+This example illustrates the use of pdwfs with the HPC I/O benchmark tool [IOR](https://github.com/hpc/ior).
+
+The example runs in a Jupyter notebook and everything you need is packaged in a Dockerfile. 
+
+We also provide a Makefile which does the plumbing for you. So, to build and run the container, just type:
+
+```
+$ git clone https://github.com/cea-hpc/pdwfs
+$ cd pdwfs
+$ make -C examples/ior_benchmark run
+```
+
+When the Jupyter notebook server is up and running, open your browser on your host at http://localhost:8888, open the notebook *ior_example.ipynb* and follow the steps.
+
+This example also allows to run the benchmark in a non-interactive way and save the results in the output directory. To run the benchmark this way, just type:
+```
+$ make -C examples/ior_benchmark bench
+```
+Once finished it will show you the results in your browser. 

examples/ior_benchmark/bench.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import glob
+import subprocess
+from datetime import datetime
+
+# override the matplotlib import in utils.py
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot
+
+# load local utils.py module
+import utils
+
+bench_script = """
+#!/bin/bash
+
+cd run
+
+echo "    Executing the benchmark on disk"
+mpirun ior -f ../ior_script > /output/ior_disk.out
+
+echo "    Executing the benchmark with pdwfs"
+# start a new Redis instance in the background
+redis-cli shutdown 2> /dev/null
+redis-server --daemonize yes --save "" > /dev/null
+
+# wrap the previous command with pdwfs -p . (indicating the current directory as the intercepted directory)
+pdwfs -p . -- mpirun ior -f ../ior_script > /output/ior_pdwfs.out
+"""
+
+def run_bench(api, version):
+
+    bench_title = api + " IOR benchmark - pdwfs " + version + " - " + str(datetime.utcnow()) + " UTC"
+    print "Running:", bench_title
+
+    read = "1"        # 1: perform read benchmark
+    numTasks="2"      # number of parallel processes
+    filePerProc="0"   # 1: write one file per processes
+    collective="1"    # 1: enable collective IO operations (MPIIO, HDF5 only)
+    segmentCount="1"  # see previous schematic
+    transferSize = ["512k", "1m", "3m", "5m", "7m", "10m","25m","35m", "50m","60m","75m","85m", "100m","115m","125m","150m","175m","200m", "225m", "250m"]
+    utils.build_ior_script(api, read, numTasks, filePerProc, collective, segmentCount, transferSize)
+
+    with open("run/bench.sh", "w") as f:
+        f.write(bench_script)
+
+    subprocess.check_call(["bash", "run/bench.sh"])
+
+    print "    Parsing and saving the results in a plot"
+    df_disk = utils.parse_ior_results("/output/ior_disk.out")
+    df_pdwfs = utils.parse_ior_results("/output/ior_pdwfs.out")
+
+    os.rename("/output/ior_disk.out", "/output/ior_" + api + "_disk.out")
+    os.rename("/output/ior_pdwfs.out", "/output/ior_" + api + "_pdwfs-" + version + ".out")
+
+    matplotlib.use('Agg')
+
+    for readOrWrite in ["write", "read"]:
+        filename = readOrWrite + "_ior_" + api + "_pdwfs-" + version + ".png"
+        utils.plot_results(
+            readOrWrite, 
+            df_disk[df_disk["Operation"] == readOrWrite], 
+            df_pdwfs[df_pdwfs["Operation"] == readOrWrite], 
+            title=bench_title, 
+            filename="/output/" + filename)
+        with open("/output/bench.html", "a") as f:
+            f.write("<img src=" + filename + ">\n")
+
+if __name__ == '__main__':
+
+    for f in glob.glob("/output/*"):
+        os.remove(f)
+
+    version = sys.argv[1]
+    run_bench("POSIX", version)
+    run_bench("MPIIO", version)
+    run_bench("HDF5", version)

examples/ior_benchmark/ior_script.jinja2

@@ -0,0 +1,16 @@
+IOR START
+  api={{ api }}
+  verbose=0
+  testFile=testFile
+  writeFile=1
+  readFile={{ read }}
+  numTasks={{ numTasks }}
+  filePerProc={{ filePerProc }}
+  collective={{ collective }}
+  segmentCount={{ segmentCount }}
+{% for size in transferSize %}
+RUN
+  blockSize={{ size }}
+  transferSize={{ size }}
+{% endfor %}
+IOR STOP