Permalink
Browse files

Merge branch 'master' into sikich1/sync

  • Loading branch information...
dsikich committed Jun 1, 2017
2 parents 86cec57 + be89be5 commit e9655a9259240b6f697ff5644d1162638fe45a30
View
@@ -1,4 +1,5 @@
Bringhurst, Jon <jonb@lanl.gov>
Di Natale, Giuseppe <dinatale2@llnl.gov>
Fuller, Douglas <dfuller@redhat.com>
Groff, Todd <groffta@ornl.gov>
LaFon, Jharrod <jlafon@lanl.gov>
View
@@ -23,9 +23,16 @@ MAINTAINERCLEANFILES = \
ltconfig ltmain.sh missing mkinstalldirs \
stamp-h.in
BUILD_DIRS = BUILD BUILDROOT RPMS SOURCES SPECS SRPMS
$(BUILD_DIRS):
rm -rf $@
mkdir $@
distclean-local:
-(cd $(top_srcdir) && rm -rf autom4te*.cache autoscan.*)
-(cd $(top_srcdir) && rm -rf $(PACKAGE)-*)
-(cd $(top_srcdir) && rm -rf $(BUILD_DIRS))
mrproper: distclean-local clean
-(cd $(top_srcdir) && rm -rf autom4te.cache config.h config.log)
@@ -41,4 +48,9 @@ mrproper: distclean-local clean
-find $(top_srcdir)/ -name ".dirstamp" -exec rm {} \;
-find $(top_srcdir) -depth -name ".deps" -exec rm -rf {} \;
rpm: dist ${BUILD_DIRS}
mv $(distdir).tar.gz SOURCES
rpmbuild -ba --define="_topdir `pwd`" mpifileutils.spec
.PHONY: test
View
21 buildme
@@ -7,9 +7,19 @@ set -x
topdir=`pwd`
installdir=$topdir/install
export PATH="${topdir}/autotools/install/bin:$PATH"
# if EA is set to true then add configure option
EA=false
build_type=""
CC=mpicc
if $EA ; then
# build line option required if building for EA systems
build_type="powerpc64le-redhat-linux-gnu"
CC=mpigcc
export CXX=mpig++
fi
./autogen.sh
rm -rf build
@@ -20,15 +30,16 @@ export PKG_CONFIG_PATH="${installdir}/lib/pkgconfig"
# TODO: avoid this step
# necessary so configure test of dtcmp links with MPI
export CC=mpicc
export CC=${CC}
# hack to get things to build after common library
export CFLAGS="-I${topdir}/src/common -DDCOPY_USE_XATTRS"
export CFLAGS="-g -O0 -I${topdir}/src/common -DDCOPY_USE_XATTRS"
export LDFLAGS="-Wl,-rpath,${topdir}/install/lib -L${topdir}/install/lib -lcircle"
# --enable-experimental \
../configure \
../configure --build=${build_type} \
--enable-lustre \
--enable-experimental \
--prefix=$installdir \
--disable-silent-rules \
--with-dtcmp=$installdir && \
View
@@ -4,6 +4,14 @@ set -x
topdir=`pwd`
# if EA is set to true then add configure option
EA=false
build_type=""
if $EA ; then
# build line option required if building for EA systems
build_type="powerpc64le-redhat-linux-gnu"
fi
## create directory to download and build autotools
mkdir autotools
pushd autotools
@@ -21,7 +29,8 @@ pushd autotools
rm -rf autoconf-2.69
tar -zxf autoconf-2.69.tar.gz
pushd autoconf-2.69
./configure --prefix=$toolsinstalldir && \
./configure --build=${build_type} \
--prefix=$toolsinstalldir && \
make && \
make install
if [ $? -ne 0 ]; then
@@ -37,7 +46,8 @@ pushd autotools
rm -rf automake-1.13
tar -zxf automake-1.13.tar.gz
pushd automake-1.13
./configure --prefix=$toolsinstalldir && \
./configure --build=${build_type} \
--prefix=$toolsinstalldir && \
make && \
make install
if [ $? -ne 0 ]; then
@@ -53,7 +63,8 @@ pushd autotools
rm -rf libtool-2.4
tar -zxf libtool-2.4.tar.gz
pushd libtool-2.4
./configure --prefix=$toolsinstalldir && \
./configure --build=${build_type} \
--prefix=$toolsinstalldir && \
make && \
make install
if [ $? -ne 0 ]; then
@@ -69,7 +80,9 @@ pushd autotools
rm -rf pkg-config-0.28
tar -zxf pkg-config-0.28.tar.gz
pushd pkg-config-0.28
./configure --prefix=$toolsinstalldir && \
./configure --with-internal-glib \
--build=${build_type} \
--prefix=$toolsinstalldir && \
make && \
make install
if [ $? -ne 0 ]; then
View
@@ -28,6 +28,14 @@ OPT="-g -O0"
installdir=`pwd`/install
# if EA is set to true then add configure option
EA=false
build_type=""
if $EA ; then
# build line option required if building for EA systems
build_type="powerpc64le-redhat-linux-gnu"
fi
mkdir -p deps
cd deps
@@ -57,7 +65,7 @@ rm -rf ${libcircle}
tar -zxf ${libcircle}.tar.gz
pushd ${libcircle}
export CFLAGS=${OPT}
./configure \
./configure --build=${build_type} \
--prefix=$installdir \
--disable-silent-rules && \
make VERBOSE=1 && \
@@ -72,7 +80,7 @@ rm -rf ${lwgrp}
tar -zxf ${lwgrp}.tar.gz
pushd ${lwgrp}
export CFLAGS=${OPT}
./configure \
./configure --build=${build_type} \
--prefix=${installdir} \
--disable-silent-rules && \
make && \
@@ -87,7 +95,7 @@ rm -rf ${dtcmp}
tar -zxf ${dtcmp}.tar.gz
pushd ${dtcmp}
export CFLAGS=${OPT}
./configure \
./configure --build=${build_type} \
--prefix=${installdir} \
--disable-silent-rules \
--with-lwgrp=${installdir} && \
@@ -103,7 +111,7 @@ rm -rf ${libarchive}
tar zxvf ${libarchive}.tar.gz
pushd ${libarchive}
export CFLAGS=${OPT}
./configure \
./configure --build=${build_type} \
--prefix=${installdir} && \
make && \
make install
View
@@ -0,0 +1,36 @@
#!/bin/bash
. /usr/local/tools/dotkit/init.sh
use mvapich2-gnu
set -x
export topdir=`pwd`
export installdir=$topdir/install
export PATH="${topdir}/autotools/install/bin:$PATH"
./autogen.sh
export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:${installdir}/lib/pkgconfig"
# TODO: avoid this step
# necessary so configure test of dtcmp links with MPI
export CC=mpicc
# hack to get things to build after common library
export CFLAGS="-I${topdir}/src/common -DDCOPY_USE_XATTRS"
export LDFLAGS="-Wl,-rpath,${topdir}/install/lib -L${topdir}/install/lib -lcircle"
#./configure \
# --enable-lustre \
# --prefix=$installdir \
# --disable-silent-rules \
# --with-dtcmp=$installdir && \
./configure --enable-dist && \
make dist && \
make rpm
if [ $? -ne 0 ] ; then
echo "failed to configure, build, or build file utils rpm"
exit 1
fi
View
@@ -1,4 +1,4 @@
AC_INIT([mpifileutils],[0.6],[mpifileutils-discuss@googlegroups.com],[mpifileutils],[http://fileutils.io])
AC_INIT([mpifileutils],[0.7a],[mpifileutils-discuss@googlegroups.com],[mpifileutils],[http://fileutils.io])
# Keep artifacts in directories as much as possible.
AC_CONFIG_AUX_DIR(auxdir)
@@ -101,10 +101,7 @@ AC_CONFIG_FILES([Makefile \
X_AC_EXPERIMENTAL
if test [EXPERIMENTAL_ENABLED]; then
AC_CONFIG_FILES([experimental/Makefile \
experimental/dfind/Makefile \
experimental/dgrep/Makefile \
experimental/dparallel/Makefile \
experimental/dtar/Makefile \
experimental/dsh/Makefile \
])
fi
View
@@ -6,33 +6,57 @@ dstripe - restripe files on underlying storage
# SYNOPSIS
**dstripe STRIPE_COUNT STRIPE_SIZE SRC_FILE DEST_FILE**
**dstripe [OPTION] PATH...**
# DESCRIPTION
Parallel MPI application to restripe a given file.
This tool is in active development. It will eventually report striping information and it will also support recursive operation on directories. It currently only works on Lustre.
This tool is in active development. It currently only works on Lustre.
dstripe enables one to restripe a file across the underlying storage devices. dstripe will make a full copy of the source file with the new striping parameters and write it to the destination file.
One must specify the integer number of stripes as the first parameter in STRIPE_COUNT. One can specify -1 to use all available stripes. One must specify the stripe size in bytes in STRIPE_SIZE. It is possible to use units like "MB" or "GB" after the number. The units should come immediately after the number without spaces. Then one must give the source file in SRC_FILE and the name for the new file in DEST_FILE.
dstripe enables one to restripe a file across the underlying storage devices. One must specify a list of paths to recursively walk. By default, stripe size is 1MB and stripe count is -1 allowing dstripe to use all available stripes.
# OPTIONS
-c, \--count STRIPE_COUNT
: The number of stripes to use during file restriping. If STRIPE_COUNT is -1, then all available stripes are used. If STRIPE_COUNT is 0, the lustre file system default is used. The default stripe count is -1.
-s, \--size STRIPE_SIZE
: The stripe size to use during file restriping. It is possible to use units like "MB" and "GB" after the number, which should be immediately follow the number without spaces (ex. 2MB). The default stripe size is 1MB.
-m, \--minsize SIZE
: The minimum size a file must be to be a candidate for restriping. It is possible to use units like "MB" and "GB" after the number, which should be immediately follow the number without spaces (ex. 2MB). The default minimum file size is 0MB.
-r, \--report
: Display the file size, stripe count, and stripe size of all files found in PATH. No restriping is performed when using this option.
-v, \--verbose
: Run in verbose mode.
-h, \--help
: Print the command usage, and the list of options available.
# EXAMPLES
1. To stripe a file on all storage devices using a 1MB stripe size:
mpirun -np 128 dstripe -1 1MB /path/to/file /path/to/file2
mpirun -np 128 dstripe -s 1MB /path/to/file
2. To stripe a file across 20 storage devices with a 1GB stripe size:
mpirun -np 128 dstripe 20 2GBB /path/to/file /path/to/file2
mpirun -np 128 dstripe -c 20 -s 1GB /path/to/file
3. To restripe all files in /path/to/files/ that are at least 1GB in size:
mpirun -np 128 dstripe -m 1GB /path/to/files/
4. To restripe all files in /path/to/files/ across 10 storage devices with 2MB stripe size:
mpirun -np 128 dstripe -c 10 -s 2MB /path/to/files/
3. To read the current striping parameters of a file on Lustre:
5. To display the current stripe count and stripe size of all files in /path/to/files/:
lfs getstripe /path/to/file
mpirun -np 128 dstripe -r /path/to/files/
# SEE ALSO
View
@@ -1 +1,2 @@
SUBDIRS = dgrep dparallel dtar dfind dsh
#SUBDIRS = dgrep dparallel dtar dfind dsh
SUBDIRS = dparallel dsh
@@ -3,6 +3,6 @@ include $(top_srcdir)/common.mk
bin_PROGRAMS = dsh
dsh_SOURCES = dsh.c
dsh_CPPFLAGS = -I../common/ $(MPI_CFLAGS) $(libcircle_CFLAGS)
dsh_CPPFLAGS = -I../../src/common/ $(MPI_CFLAGS) $(libcircle_CFLAGS)
dsh_LDFLAGS = $(MPI_CLDFLAGS)
dsh_LDADD = ../common/libmfu.la $(MPI_CLDFLAGS) $(libcircle_LIBS)
dsh_LDADD = ../../src/common/libmfu.la $(MPI_CLDFLAGS) $(libcircle_LIBS)
Oops, something went wrong.

0 comments on commit e9655a9

Please sign in to comment.