Permalink
Browse files

QFS files - initial commite

  • Loading branch information...
1 parent 9708523 commit f7e0bab3c9f1889ef1b9d0fc6c704473e0596a89 thilee committed Aug 31, 2012
Showing with 26,992 additions and 0 deletions.
  1. +179 −0 CMakeLists.txt
  2. +202 −0 LICENSE.txt
  3. +55 −0 Makefile
  4. +445 −0 benchmarks/mstress/MStress_Client.java
  5. +57 −0 benchmarks/mstress/Makefile
  6. +312 −0 benchmarks/mstress/README
  7. +445 −0 benchmarks/mstress/mstress.py
  8. +69 −0 benchmarks/mstress/mstress_cleanup.py
  9. +582 −0 benchmarks/mstress/mstress_client.cc
  10. +77 −0 benchmarks/mstress/mstress_initialize.sh
  11. +169 −0 benchmarks/mstress/mstress_plan.py
  12. +62 −0 benchmarks/mstress/mstress_prepare_master_clients.sh
  13. +144 −0 benchmarks/mstress/mstress_run.py
  14. +85 −0 benchmarks/mstress/mstress_sample_run.sh
  15. +85 −0 build.xml
  16. +64 −0 examples/README
  17. +40 −0 examples/cc/CMakeLists.txt
  18. +254 −0 examples/cc/kfssample_main.cc
  19. +219 −0 examples/java/KfsSample.java
  20. +32 −0 examples/python/kfssample.cfg
  21. +174 −0 examples/python/kfssample.py
  22. +50 −0 examples/sampleservers/sample_setup.cfg
  23. +493 −0 examples/sampleservers/sample_setup.py
  24. +30 −0 package/conf/ChunkServer.prp
  25. +24 −0 package/conf/KfsClient.prp
  26. +28 −0 package/conf/MetaServer.prp
  27. 0 package/deb/.gitignore
  28. +35 −0 package/rpm/README
  29. +75 −0 package/rpm/specs/qfs-chunkserver.spec
  30. +78 −0 package/rpm/specs/qfs-client.spec
  31. +78 −0 package/rpm/specs/qfs-metaserver.spec
  32. +153 −0 scripts/kfsprune.py
  33. +82 −0 scripts/metalogprune.py
  34. +44 −0 src/cc/access/CMakeLists.txt
  35. +1,268 −0 src/cc/access/kfs_access_jni.cc
  36. +1,181 −0 src/cc/access/kfs_module_py.cc
  37. +80 −0 src/cc/access/kfs_setup.py
  38. +3,079 −0 src/cc/chunk/AtomicRecordAppender.cc
  39. +208 −0 src/cc/chunk/AtomicRecordAppender.h
  40. +455 −0 src/cc/chunk/BufferManager.cc
  41. +253 −0 src/cc/chunk/BufferManager.h
  42. +74 −0 src/cc/chunk/CMakeLists.txt
  43. +263 −0 src/cc/chunk/Chunk.h
  44. +4,373 −0 src/cc/chunk/ChunkManager.cc
  45. +824 −0 src/cc/chunk/ChunkManager.h
  46. +125 −0 src/cc/chunk/ChunkServer.cc
  47. +98 −0 src/cc/chunk/ChunkServer.h
  48. +41 −0 src/cc/chunk/ClientManager.cc
  49. +202 −0 src/cc/chunk/ClientManager.h
  50. +790 −0 src/cc/chunk/ClientSM.cc
  51. +195 −0 src/cc/chunk/ClientSM.h
  52. +743 −0 src/cc/chunk/DirChecker.cc
  53. +124 −0 src/cc/chunk/DirChecker.h
  54. +2,104 −0 src/cc/chunk/DiskIo.cc
  55. +317 −0 src/cc/chunk/DiskIo.h
  56. +2,871 −0 src/cc/chunk/KfsOps.cc
  57. +1,849 −0 src/cc/chunk/KfsOps.h
  58. +286 −0 src/cc/chunk/LeaseClerk.cc
  59. +110 −0 src/cc/chunk/LeaseClerk.h
  60. +76 −0 src/cc/chunk/Logger.cc
  61. +82 −0 src/cc/chunk/Logger.h
Sorry, we could not display the entire diff because too many files (367) changed.
View
179 CMakeLists.txt
@@ -0,0 +1,179 @@
+#
+# $Id$
+#
+# Created 2006/10/20
+# Author: Sriram Rao (Kosmix Corp)
+#
+# Copyright 2008-2012 Quantcast Corp.
+# Copyright 2006 Kosmix Corp.
+#
+# This file is part of Kosmos File System (KFS).
+#
+# Licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+#
+
+set(CMAKE_LEGACY_CYGWIN_WIN32 0) # Remove when CMake >= 2.8.4 is required
+cmake_minimum_required(VERSION 2.4.6)
+
+project (KFS)
+
+if (DEFINED KFS_DIR_PREFIX)
+ message ("Kfs source dir prefix: ${KFS_DIR_PREFIX}")
+ set(CMAKE_MODULE_PATH ${KFS_DIR_PREFIX}cmake)
+else (DEFINED KFS_DIR_PREFIX)
+ set(KFS_DIR_PREFIX "")
+ set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
+endif (DEFINED KFS_DIR_PREFIX)
+
+# Locate Boost
+# set(Boost_LIB_DIAGNOSTIC_DEFINITIONS "-DBOOST_LIB_DIAGNOSTIC")
+
+if (NOT CYGWIN)
+ set(Boost_USE_STATIC_LIBS ON)
+endif (NOT CYGWIN)
+
+set(Boost_USE_MULTITHREADED ON)
+
+IF (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ find_package(Boost COMPONENTS regex system REQUIRED)
+ELSE (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ find_package(Boost COMPONENTS regex REQUIRED)
+ENDIF (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+
+message(STATUS "Boost-includes = ${Boost_INCLUDE_DIRS}")
+message(STATUS "Boost-libs = ${Boost_LIBRARIES}")
+
+# Locate the path to jni.h
+find_package(JNI)
+
+ENABLE_TESTING()
+
+# Change this to where the install directory is located
+if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+ set(CMAKE_INSTALL_PREFIX "." CACHE PATH "installation directory prefix" FORCE)
+endif (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+
+# Build with statically linked libraries; the value for this variable has to be defined here
+# overwriting whatever is in the cache.
+# When set to ON, we build with statically linked libraries; when off we
+# link with dynamically linked libs
+
+IF (CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ IF (BUILD_CPU_MODE STREQUAL "32")
+ message (STATUS "Building 32-bit mode on Solaris")
+ # If we are asked to build 32 bit mode
+ add_definitions (-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGE_FILES)
+ ELSE (BUILD_CPU_MODE STREQUAL "32")
+ # On solaris, use 64-bit mode
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m64")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m64")
+ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m64")
+ ENDIF (BUILD_CPU_MODE STREQUAL "32")
+ # Statically linked binaries don't work on solaris
+ set (USE_STATIC_LIB_LINKAGE OFF CACHE BOOL "Build binaries with statically linked libraries" FORCE)
+ # Cmake does whacky relink on solaris and messes things up; avoid this
+ set (CMAKE_SKIP_RPATH ON)
+ELSE (CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ set (USE_STATIC_LIB_LINKAGE ON CACHE BOOL "Build binaries with statically linked libraries" FORCE)
+ IF (CMAKE_SIZEOF_VOID_P MATCHES "4" AND NOT CYGWIN)
+ message (STATUS "Enabling largefile source flags")
+ add_definitions (-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_LARGE_FILES)
+ ENDIF (CMAKE_SIZEOF_VOID_P MATCHES "4" AND NOT CYGWIN)
+ENDIF (CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+
+IF (ENABLE_PROFILING)
+ message (STATUS "Enabling profiling with gprof")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg")
+ set(CMAKE_SHAREDBoost_USE_MULTITHREADED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg")
+ set(CMAKE_EXE_FLAGS "${CMAKE_EXE_FLAGS} -pg")
+ENDIF (ENABLE_PROFILING)
+
+# Darwin compilers need to be told about ports
+IF (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/opt/local/include")
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L/opt/local/lib")
+ENDIF (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+
+
+# Change the line to Release to build release binaries
+# For servers, build with debugging info; for tools, build Release
+#
+
+IF (NOT CMAKE_BUILD_TYPE)
+ message (STATUS "Setting build type to Debug")
+ set (CMAKE_BUILD_TYPE "Debug")
+ENDIF (NOT CMAKE_BUILD_TYPE)
+
+IF (CMAKE_BUILD_TYPE STREQUAL "Release")
+ message(STATUS "Enabling -D NDEBUG flag")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D NDEBUG -g3")
+ENDIF(CMAKE_BUILD_TYPE STREQUAL "Release")
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -DBOOST_SP_USE_QUICK_ALLOCATOR")
+string(TOUPPER KFS_OS_NAME_${CMAKE_SYSTEM_NAME} KFS_OS_NAME)
+add_definitions (-D${KFS_OS_NAME})
+
+#
+# Find the path to libfuse.so
+#
+
+SET(Fuse_LIBRARY_DIR "")
+IF (EXISTS "/lib64/libfuse.so")
+ SET(Fuse_LIBRARY_DIR "/lib64")
+ELSEIF (EXISTS "/opt/local/lib/libfuse.dylib")
+ SET(Fuse_LIBRARY_DIR "/opt/local/lib")
+ELSEIF (EXISTS "/usr/local/lib/libfuse.dylib" OR EXISTS "/usr/local/lib/libfuse_ino64.dylib")
+ SET(Fuse_LIBRARY_DIR "/usr/local/lib")
+ SET(Fuse_INCLUDE_DIR "/usr/local/include/osxfuse")
+ELSEIF (EXISTS "/usr/lib/libfuse.a" OR EXISTS "/usr/lib/libfuse.so")
+ SET(Fuse_LIBRARY_DIR "/usr/local/lib")
+ELSEIF (EXISTS "/lib/libfuse.a" OR EXISTS "/lib/libfuse.so")
+ SET(Fuse_LIBRARY_DIR "/lib")
+ENDIF (EXISTS "/lib64/libfuse.so")
+
+if(COMMAND cmake_policy)
+ cmake_policy(SET CMP0003 NEW)
+endif(COMMAND cmake_policy)
+
+# include dirs
+include_directories( ${Boost_INCLUDE_DIRS} ${KFS_DIR_PREFIX}src/cc)
+
+# get the subdirs we want
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/common src/cc/common)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/meta src/cc/meta)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/chunk src/cc/chunk)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/libclient src/cc/libclient)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/kfsio src/cc/kfsio)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/tools src/cc/tools)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/devtools src/cc/devtools)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/tests src/cc/tests)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/emulator src/cc/emulator)
+add_subdirectory (${KFS_DIR_PREFIX}src/test-scripts src/test-scripts)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/qcdio src/cc/qcdio)
+add_subdirectory (${KFS_DIR_PREFIX}src/cc/qcrs src/cc/qcrs)
+
+add_subdirectory (${KFS_DIR_PREFIX}examples/cc examples/cc)
+
+IF (NOT ${JAVA_INCLUDE_PATH} STREQUAL "")
+ message(STATUS "Found JNI...building kfs_access")
+ include_directories ( ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2} )
+ add_subdirectory (${KFS_DIR_PREFIX}src/cc/access src/cc/access)
+ENDIF (NOT ${JAVA_INCLUDE_PATH} STREQUAL "")
+
+IF (NOT ${Fuse_LIBRARY_DIR} STREQUAL "")
+ message(STATUS "Found fuse")
+ include_directories ( ${Fuse_INCLUDE_DIR} )
+ add_subdirectory (${KFS_DIR_PREFIX}src/cc/fuse src/cc/fuse)
+ENDIF (NOT ${Fuse_LIBRARY_DIR} STREQUAL "")
View
202 LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
View
55 Makefile
@@ -0,0 +1,55 @@
+# $Id$
+#
+# Created 2012/07/27
+# Author: Mike Ovsiannikov
+#
+# Copyright 2012 Quantcast Corp.
+#
+# This file is part of Kosmos File System (KFS).
+#
+# Licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# Do not assume gnumake -- keep it as simple as possible
+
+release:
+ cd build && \
+ { test -d release || mkdir release; } && \
+ cd release && \
+ cmake -D CMAKE_BUILD_TYPE=RelWithDebInfo ../.. && \
+ make install
+ if test -x "`which ant 2>/dev/null`"; then ant jar; fi
+ if test -x "`which python 2>/dev/null`"; then \
+ cd build/release && python ../../src/cc/access/kfs_setup.py build; fi
+ # cd build/release && make test
+ cd build/release && ../../src/test-scripts/kfstest.sh
+
+tarball: release
+ cd build && \
+ tar -cvf kfs.tar -C ./release ./bin ./lib ./include && \
+ tar -rvf kfs.tar -C ../ ./scripts ./webui ./examples ./benchmarks && \
+ gzip qfs.tar
+
+debug:
+ cd build && \
+ { test -d debug || mkdir debug; } && \
+ cd debug && \
+ cmake ../.. && \
+ make install
+ if test -x "`which ant 2>/dev/null`"; then ant jar; fi
+ if test -x "`which python 2>/dev/null`"; then \
+ cd build/debug && python ../../src/cc/access/kfs_setup.py build; fi
+ # cd build/debug && make test
+ cd build/debug && ../../src/test-scripts/kfstest.sh
+
+clean:
+ rm -rf build/release build/debug build/classes build/kfs-*.jar build/*.tar.gz
View
445 benchmarks/mstress/MStress_Client.java
@@ -0,0 +1,445 @@
+/**
+ * $Id$
+ *
+ * Author: Thilee Subramaniam
+ *
+ * Copyright 2012 Quantcast Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * This Java client performs filesystem meta opetarions on the Hadoop namenode
+ * using HDFS DFSClient.
+ */
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+
+public class MStress_Client
+{
+ static final String TEST_BASE_DIR = new String("/mstress");
+
+ static DFSClient dfsClient_ = null;
+ static StringBuilder path_ = new StringBuilder(4096);
+ static int pathLen_ = 0;
+ static int totalCreateCount = 0;
+ static final int COUNT_INCR = 500;
+
+ //From commandline
+ static String dfsServer_ = "";
+ static int dfsPort_ = 0;
+ static String testName_ = "";
+ static String prefix_ = "";
+ static int prefixLen_ = 0;
+ static String planfilePath_ = "";
+ static String hostName_ = "";
+ static String processName_ = "";
+
+ //From plan file
+ static String type_ = "";
+ static int levels_ = 0;
+ static int inodesPerLevel_ = 0;
+ static int pathsToStat_ = 0;
+
+ private static void pathPush(String leafStr) {
+ int leafLen = leafStr.length();
+ if (leafLen == 0) {
+ return;
+ }
+ if (leafStr.charAt(0) != '/') {
+ path_.insert(pathLen_, "/");
+ System.out.printf("Leaf = %s, path_ = [%s]\n", leafStr, path_.toString());
+ pathLen_ ++;
+ }
+ path_.insert(pathLen_, leafStr);
+ System.out.printf("After push Leaf = %s, path_ = [%s]\n", leafStr, path_.toString());
+ pathLen_ += leafLen;
+ }
+
+ private static void pathPop(String leafStr) {
+ int leafLen = leafStr.length();
+ if (leafLen > pathLen_ - 1) {
+ System.out.printf("Error in pop: %s from %s, leafLen = %d, pathLen_ = %d\n", leafStr, path_.toString(), leafLen, pathLen_);
+ return;
+ }
+ String lastPart = path_.substring(pathLen_ - leafLen, pathLen_);
+ System.out.printf("lastPart = [%s - %s] leafStr = [%s - %s]\n", lastPart, lastPart.getClass().getName(), leafStr, leafStr.getClass().getName());
+
+ if (!leafStr.equals(lastPart)) {
+ System.out.printf("Error in pop: %s from %s\n", leafStr, path_.toString());
+ System.exit(1);
+ return;
+ }
+ pathLen_ -= leafLen + 1;
+ path_.insert(pathLen_, '\0');
+ System.out.printf("After pop, path_ = [%s]\n", path_.toString());
+ }
+
+ private static void pathReset() {
+ path_.insert(0, '\0');
+ pathLen_ = 0;
+ }
+
+
+ public static void main(String args[]) {
+ parseOptions(args);
+
+ try {
+ Configuration conf = new Configuration(true);
+ String confSet = "hdfs://" + dfsServer_ + ":" + dfsPort_;
+ conf.set("fs.default.name", confSet);
+ conf.set("fs.trash.interval", "0");
+ InetSocketAddress inet = new InetSocketAddress(dfsServer_, dfsPort_);
+ dfsClient_ = new DFSClient(inet, conf);
+
+ if (parsePlanFile() < 0) {
+ return;
+ }
+
+ if (testName_.equals("create")) {
+ createDFSPaths();
+ } else if (testName_.equals("stat")) {
+ statDFSPaths();
+ } else if (testName_.equals("readdir")) {
+ listDFSPaths();
+ } else if (testName_.equals("delete")) {
+ removeDFSPaths();
+ } else {
+ System.out.printf("Error: unrecognized test \'%s\'\n", testName_);
+ }
+ } catch( IOException e) {
+ e.printStackTrace();
+ }
+ return;
+ }
+
+ private static void parseOptions(String args[])
+ {
+ if (!(args.length == 14 || args.length == 12 || args.length == 5)) {
+ usage();
+ }
+
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("-s") && i+1 < args.length) {
+ dfsServer_ = args[i+1];
+ System.out.println(args[i+1]);
+ i++;
+ } else if (args[i].equals("-p") && i+1 < args.length) {
+ dfsPort_ = Integer.parseInt(args[i+1]);
+ System.out.println(args[i+1]);
+ i++;
+ } else if (args[i].equals("-t") && i+1 < args.length) {
+ testName_ = args[i+1];
+ System.out.println(args[i+1]);
+ i++;
+ } else if (args[i].equals("-a") && i+1 < args.length) {
+ planfilePath_ = args[i+1];
+ System.out.println(args[i+1]);
+ i++;
+ } else if (args[i].equals("-c") && i+1 < args.length) {
+ hostName_ = args[i+1];
+ System.out.println(args[i+1]);
+ i++;
+ } else if (args[i].equals("-n") && i+1 < args.length) {
+ processName_ = args[i+1];
+ System.out.println(args[i+1]);
+ i++;
+ } else if (args[i].equals("-P") && i+1 < args.length) {
+ prefix_ = args[i+1];
+ System.out.println(args[i+1]);
+ i++;
+ }
+ }
+
+ if (dfsServer_.length() == 0 ||
+ testName_.length() == 0 ||
+ planfilePath_.length() == 0 ||
+ hostName_.length() == 0 ||
+ processName_.length() == 0 ||
+ dfsPort_ == 0) {
+ usage();
+ }
+ if (prefix_ == null) {
+ prefix_ = new String("PATH_");
+ }
+ prefixLen_ = prefix_.length();
+ }
+
+ private static void usage()
+ {
+ String className = MStress_Client.class.getName();
+ System.out.printf("Usage: java %s -s dfs-server -p dfs-port [-t [create|stat|readdir|rmdir] -a planfile-path -c host -n process-name -P prefix]\n",
+ className);
+ System.out.printf(" -t: this option requires -a, -c, and -n options.\n");
+ System.out.printf(" -P: default prefix is PATH_.\n");
+ System.out.printf("eg:\n");
+ System.out.printf(" java %s -s <metaserver-host> -p <metaserver-port> -t create -a <planfile> -c localhost -n Proc_00\n", className);
+ System.exit(1);
+ }
+
+ private static int parsePlanFile()
+ {
+ int ret = -1;
+ try {
+ FileInputStream fis = new FileInputStream(planfilePath_);
+ DataInputStream dis = new DataInputStream(fis);
+ BufferedReader br = new BufferedReader(new InputStreamReader(dis));
+
+ if (prefix_.isEmpty()) {
+ prefix_ = "PATH_";
+ }
+
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.length() == 0 || line.startsWith("#")) {
+ continue;
+ }
+ if (line.startsWith("type=")) {
+ type_ = line.substring(5);
+ continue;
+ }
+ if (line.startsWith("levels=")) {
+ levels_ = Integer.parseInt(line.substring(7));
+ continue;
+ }
+ if (line.startsWith("inodes=")) {
+ inodesPerLevel_ = Integer.parseInt(line.substring(7));
+ continue;
+ }
+ if (line.startsWith("nstat=")) {
+ pathsToStat_ = Integer.parseInt(line.substring(6));
+ continue;
+ }
+ }
+ dis.close();
+ if (levels_ > 0 && !type_.isEmpty() && inodesPerLevel_ > 0 && pathsToStat_ > 0) {
+ ret = 0;
+ }
+ } catch (Exception e) {
+ System.out.println("Error: " + e.getMessage());
+ }
+ return ret;
+ }
+
+ private static long timeDiffMilliSec(Date alpha, Date zigma)
+ {
+ return zigma.getTime() - alpha.getTime();
+ }
+
+ private static void CreateDFSPaths(int level, String parentPath) {
+ Boolean isLeaf = false;
+ Boolean isDir = false;
+ if (level + 1 >= levels_) {
+ isLeaf = true;
+ }
+ if (isLeaf) {
+ if (type_.equals("dir")) {
+ isDir = true;
+ } else {
+ isDir = false;
+ }
+ } else {
+ isDir = true;
+ }
+
+ for (int i = 0; i < inodesPerLevel_; i++) {
+ String path = parentPath + "/" + prefix_ + Integer.toString(i);
+ //System.out.printf("Creating (isdir=%b) [%s]\n", isDir, path.toString());
+
+ if (isDir) {
+ try {
+ dfsClient_.mkdirs(path);
+ totalCreateCount ++;
+ if (totalCreateCount % COUNT_INCR == 0) {
+ System.out.printf("Created paths so far: %d\n", totalCreateCount);
+ }
+ if (!isLeaf) {
+ CreateDFSPaths(level+1, path);
+ }
+ } catch( IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+ } else {
+ try {
+ dfsClient_.create(path, true);
+ totalCreateCount ++;
+ if (totalCreateCount % COUNT_INCR == 0) {
+ System.out.printf("Created paths so far: %d\n", totalCreateCount);
+ }
+ } catch( IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+ }
+ }
+ }
+
+ private static int createDFSPaths()
+ {
+ String basePath = new String(TEST_BASE_DIR) + "/" + hostName_ + "_" + processName_;
+ try {
+ Boolean ret = dfsClient_.mkdirs(basePath);
+ if (!ret) {
+ System.out.printf("Error: failed to create test base dir [%s]\n", basePath);
+ return -1;
+ }
+ } catch( IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+
+ Date alpha = new Date();
+
+ CreateDFSPaths(0, basePath);
+
+ Date zigma = new Date();
+ System.out.printf("Client: %d paths created in %d msec\n", totalCreateCount, timeDiffMilliSec(alpha, zigma));
+ return 0;
+ }
+
+ private static int statDFSPaths()
+ {
+ String basePath = new String(TEST_BASE_DIR) + "/" + hostName_ + "_" + processName_;
+
+ Date alpha = new Date();
+ Random random = new Random(alpha.getTime());
+
+ for (int count = 0; count < pathsToStat_; count++) {
+ String path = basePath;
+ for (int d = 0; d < levels_; d++) {
+ int randIdx = random.nextInt(inodesPerLevel_);
+ String name = new String(prefix_) + Integer.toString(randIdx);
+ path = path + "/" + name;
+ }
+
+ //System.out.printf("Doing stat on [%s]\n", path);
+ HdfsFileStatus stat = null;
+ try {
+ stat = dfsClient_.getFileInfo(path);
+ } catch(IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+ if (count % COUNT_INCR == 0) {
+ System.out.printf("Stat paths so far: %d\n", count);
+ }
+ }
+ Date zigma = new Date();
+ System.out.printf("Client: Stat done on %d paths in %d msec\n", pathsToStat_, timeDiffMilliSec(alpha, zigma));
+ return 0;
+ }
+
+ private static int listDFSPaths()
+ {
+ Date alpha = new Date();
+ int inodeCount = 0;
+
+ String basePath = new String(TEST_BASE_DIR) + "/" + hostName_ + "_" + processName_;
+ Queue<String> pending = new LinkedList<String>();
+ pending.add(basePath);
+
+ while (!pending.isEmpty()) {
+ String parent = pending.remove();
+ DirectoryListing thisListing;
+ try {
+ thisListing = dfsClient_.listPaths(parent, HdfsFileStatus.EMPTY_NAME);
+ if (thisListing == null || thisListing.getPartialListing().length == 0) {
+ //System.out.println("Empty directory");
+ continue;
+ }
+ do {
+ HdfsFileStatus[] children = thisListing.getPartialListing();
+ for (int i = 0; i < children.length; i++) {
+ String localName = children[i].getLocalName();
+ //System.out.printf("Readdir going through [%s/%s]\n", parent, localName);
+ if (localName.equals(".") || localName.equals("..")) {
+ continue;
+ }
+ inodeCount ++;
+ if (inodeCount % COUNT_INCR == 0) {
+ System.out.printf("Readdir paths so far: %d\n", inodeCount);
+ }
+ if (children[i].isDir()) {
+ pending.add(parent + "/" + localName);
+ }
+ }
+ if (!thisListing.hasMore()) {
+ break;
+ } else {
+ //System.out.println("Remaining entries " + Integer.toString(thisListing.getRemainingEntries()));
+ }
+ thisListing = dfsClient_.listPaths(parent, thisListing.getLastName());
+ } while (thisListing != null);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+ }
+
+ Date zigma = new Date();
+ System.out.printf("Client: Directory walk done over %d inodes in %d msec\n", inodeCount, timeDiffMilliSec(alpha, zigma));
+ return 0;
+ }
+
+ private static int removeDFSPaths()
+ {
+ String rmPath = new String(TEST_BASE_DIR) + "/" + hostName_ + "_" + processName_;
+
+ System.out.printf("Deleting %s ...\n", rmPath);
+
+ int countLeaf = (int) Math.round(Math.pow(inodesPerLevel_, levels_));
+ int[] leafIdxRangeForDel = new int[countLeaf];
+ for(int i=0;i<countLeaf;i++)
+ leafIdxRangeForDel[i] = i;
+ Collections.shuffle(Arrays.asList(leafIdxRangeForDel));
+
+ Date alpha = new Date();
+ try {
+ for(int idx : leafIdxRangeForDel) {
+ String path = "";
+ for(int lev=0; lev < levels_; lev++) {
+ int delta = idx % inodesPerLevel_;
+ idx /= inodesPerLevel_;
+ if(path.length() > 0) {
+ path = prefix_ + delta + "/" + path;
+ } else {
+ path = prefix_ + delta;
+ }
+ }
+ dfsClient_.delete(rmPath + "/" + path,true);
+ }
+ dfsClient_.delete(rmPath, true);
+ } catch(IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException();
+ }
+ Date zigma = new Date();
+ System.out.printf("Client: Deleted %s. Delete took %d msec\n", rmPath, timeDiffMilliSec(alpha, zigma));
+ return 0;
+ }
+}
View
57 benchmarks/mstress/Makefile
@@ -0,0 +1,57 @@
+# $Id$
+#
+# Author: Thilee Subramaniam
+#
+# Copyright 2012 Quantcast Corp.
+#
+# This file is part of Kosmos File System (KFS).
+#
+# Licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+#
+
+
+UNAME := $(shell uname)
+CC = g++
+KFS_BUILD_INCLUDE?=../../build/include
+KFS_BUILD_STATLIB?=../../build/lib/static
+INCLUDES = -I. -I$(KFS_BUILD_INCLUDE)/kfs
+CCFLAGS = -g -O2 $(INCLUDES)
+LIBS = -lkfs_tools -lkfs_client -lkfs_qcdio -lpthread -lkfs_io -lkfs_common -lkfs_qcdio -lpthread -lz -lcrypto -lkfs_qcrs -lboost_regex
+ifneq ($(UNAME), Darwin)
+LIBS += -lrt
+endif
+LDFLAGS = -L$(KFS_BUILD_STATLIB) -L$(BOOST_LIBRARY_DIR) $(LIBS)
+
+javaC = javac
+javaR = java
+javaCP = -cp
+
+default: ccclient javaclient
+
+ccclient: mstress_client.cc
+ $(CC) $(CCFLAGS) mstress_client.cc $(LDFLAGS) -o mstress_client
+
+javaclient: MStress_Client.java
+ $(javaC) $(javaCP) $(shell echo mstress_hdfs_client_jars/*.jar | sed 's/ /:/g') MStress_Client.java
+
+run_ccclient: ccclient
+ ./mstress_client -h
+
+run_javaclient: javaclient
+ $(javaR) $(javaCP) .:$(shell echo mstress_hdfs_client_jars/*.jar | sed 's/ /:/g') MStress_Client -h
+
+clean:
+ rm -f *.o mstress_client *.class
+ rm -rf *.dSYM
+
View
312 benchmarks/mstress/README
@@ -0,0 +1,312 @@
+#
+# $Id$
+#
+# Author: Thilee Subramaniam
+#
+# Copyright 2012 Quantcast Corp.
+#
+# This file is part of Kosmos File System (KFS).
+#
+# Licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+
+MSTRESS : A framework for metaserver/namenode benchmarking
+==========================================================
+
+Contents:
+ [1] Framework description
+ [2] Files in this direcotry
+ [3] Running benchmark
+ [4] Setting up DFS metaserver/namenode
+
+
+[1] Framework
+=============
+
+The mstress master would invoke mstress.py in slave mode on the client hosts
+through SSH.
+
+Each mstress slave would invoke the necessary number of load-generating clients,
+which would stress the meta server.
+
+ +-----------------------------+
+ | +-------------------+ |
+ | | mstress.py +-----+----------------------+
+ | | (--mode master) +-----+------------------+ |
+ | +-------------------+ | | |
+ | (master host) | | |
+ +-----------------------------+ | |
+ | |
+ +--------------------------------------+ | |
+ | | | |
+ +-----------+ | +--------------+ +--------------+ | | |
+ | |<---------+-|mstress_client|<--| mstress.py |<-+--+ |
+ | | | +--------------+ |(--mode slave)| | |
+ | DFS meta | | +--------------+ | |
+ | server | | (client host 1) | |
+ | | +--------------------------------------+ |
+ | | |
+ | | |
+ | | +--------------------------------------+ |
+ | | | +--------------+ +--------------+ | |
+ | |<-----------|mstress_client|<--| mstress.py |<-+------+
+ +-----------+ | +--------------+ |(--mode slave)| |
+ | +--------------+ |
+ | (client host 2) |
+ +--------------------------------------+
+
+The clients will do file or directory tree creation, stat, or directory walk as
+specified by the benchmark plan.
+
+
+
+[2] Files
+=========
+
+ - mstress_initialize.sh
+ Helper script to be used before compiling the source and deploying the
+ mstress bundle.
+ Do ./mstress_initialize.sh --h to see options.
+
+ - Makefile
+ Used to build the KFS stress client (C++) and HDFS stress client (Java).
+ Ensure that $JAVA_HOME is set correctly.
+
+ - mstress_client.cc
+ Produces the mstress_client binary that actually drives the KFS metaserver.
+ Build using the provided Makefile ('make ccclient')
+ See 'Benchmarking Procedure' below for details.
+
+ - MStress_Client.java
+ Produces the java MStress_Client for HDFS namenode.
+ Build using the provided Makefile ('make javaclient')
+ See 'Benchmarking Procedure' below for details.
+
+ - mstress_prepare_master_clients.sh
+ Helper script used to copy the mstress directory to a list of hosts. To be
+ used after running make.
+
+ - mstress_plan.py
+ Used to generate a plan file for benchmarking.
+ Args: client hosts list, number of clients per client host, file tree depth,
+ nodes per level etc.
+ The generated plan file is also copied to the /tmp firectory of the
+ participating client hosts.
+ Do ./mstress_plan.py --help to see all options.
+
+ - mstress.py
+ Used to run the metaserver test with the help of the plan file.
+ Args: dfs server host & port, planfile etc.
+ This script invokes mstress.py on the remote host through SSH. For this
+ reason, the mstress path should be the same on the participating hosts.
+ Do ./mstress.py --help to see all options.
+
+ - mstress_run.py
+ Essentially a wrapper around mstress_plan.py and mstress.py
+ Args: client hosts list and DFS server:port information.
+ Do mstress_run.py --help to see usage.
+
+ - mstress_sample_run.sh
+ Used to run sample benchmarks on given KFS and HDFS servers by launching
+ clients on localhost. Essentially a wrapper around mstress_initialize.sh,
+ make, mstress_prepare_master_clients.sh, and mstress.run.py.
+
+ - mstress_cleanup.py
+ Used to clean up the plan files and log files created on participating
+ hosts.
+ Do ./mstress_cleanup.py --help to see usage.
+
+
+
+[3] Benchmarking Procedure
+==========================
+
+In reality, benchmark would use separate physical machines each for compiling,
+running the DFS server, running mstress master, and load generating clients.
+The procedure below assumes different machines, but one can also run all
+on the same box, "localhost".
+
+
+(1) Setup the KFS metaserver and HDFS namenode with the help of
+ section [4] "Setting up DFS metaserver/namenode" below.
+
+
+(2) You should have SSH key authentication set up on the hosts involved so
+ that the scripts can do password/passphrase-less login.
+
+
+(3) On the build machine, ensure that you have the Cloudera HDFS client jars.
+ This is typically at /usr/lib/hadoop/client/*.jars.
+ If you don't have them, install them by,
+ 1. Add the following to /etc/yum.repos.d/thirdparty.repo (sudo needed)
+ -----------------------------------
+ [cloudera-cdh4]
+ name=Cloudera's Distribution for Hadoop, Version 4
+ baseurl=http://archive.cloudera.com/cdh4/redhat/6/x86_64/cdh/4/
+ gpgkey = http://archive.cloudera.com/cdh4/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera
+ gpgcheck = 1
+ -----------------------------------
+ 2. sudo yum install hadoop-client
+
+
+(4) On the build host, execute 'mstress_initialize.sh' to set up jar paths.
+ ./mstress_initialize.sh /usr/lib/hadoop/client/
+
+
+(5) On the build host, compile and install KFS using the steps described in the
+ DeveloperDoc in top-level 'doc' directory. Then change directory to
+ bench/mstress, and just issuing 'make' should build the Java/C++ clients.
+
+ To manually build C++ client:
+ - Assuming the kfs code is in ~/code/kfs, compile and install KFS using
+ the steps described in the DeveloperDoc in top-level 'doc' directory.
+ - cd ~/code/kfs/bench/mstress
+ - KFS_BUILD_INCLUDE=~/code/kfs/build/include \
+ KFS_BUILD_STATLIB=~/code/kfs/build/lib/static \
+ BOOST_LIBRARY_DIR=/opt/local/lib \
+ make
+ If you encounter any build problem, ensure that your KFS_BUILD_INCLUDE etc.
+ refer to valid paths.
+
+
+ To manually build MStress_Client.java
+ - Compile MStress_client.java with hadoop-client jars in the class path.
+ theCP=$(echo mstress_hdfs_client_jars/*.jar | sed 's/ /:/g')
+ javac -cp $theCP MStress_Client.java
+
+
+(6) Determine the master and load generating client hosts that you want to use
+ to connect to the DFS server. This could just be "localhost" if you want to
+ run the benchmark locally.
+
+
+(7) From the build host, use "mstress_prepare_master_clients.sh" to copy your
+ mstress directory to the participating hosts.
+ Note: Do './mstress_prepare_master_clients.sh localhost' localhost-only run.
+ The mstress directory paths should be the same on master and client hosts.
+
+
+(8) On the master host change directory to ~/mstress
+ Create a plan file using mstress_plan.py.
+ Do ./mstress_plan.py --help to see example usage.
+ Eg:
+ ./mstress_plan.py -c localhost,127.0.0.1 -n 3 -t file -l 2 -i 10 -n 139
+
+ This will create a plan that creates 2 levels of 10 inodes each by 3
+ processes on 2 hosts. Since each client creates 110 inodes (10 directories
+ with 10 files each) and since there are 6 clients (3 x 2), this plan is to
+ create 660 inodes on the DFS server.
+
+ The planfile will pick N files to stat per client such that
+ (N x client-host-count x clients-per-host) is just enough to meet 139.
+
+ The plan file gets copied to the /tmp directory where you run it. It will
+ also get copied to the participating client hosts in the '-c' option.
+
+
+(9) Checklist: check the presence of,
+ - the plan file on master host and client hosts (step 8 does this for you)
+ - the mstress_client binaries (KFS and HDFS clients) on master and all
+ client hosts (step 7).
+
+(10) Run the benchmark from the master with mstress.py.
+ Do ./mstress.py --help to see options.
+ Eg:
+ ./mstress.py -f kfs -s <metahost> -p <metaport> -a </tmp/something.plan>
+ ./mstress.py -f hdfs -s <namehost> -p <nameport> -a </tmp/something.plan>
+
+(11) The benchmark name, progress, and time taken will be printed out.
+
+
+[4] DFS Server Setup
+====================
+
+[4.1] KFS Metaserver Setup
+-------------------------
+
+You can setup the KFS metaserver using the steps described in AdminDoc in the
+top-level 'doc' directory.
+
+If you want to set up a simple metaserver for local testing, please use the
+script ~/code/kfs/examples/sampleservers/sample_setup.py.
+
+
+[4.2] HDFS Namenode Setup
+-------------------------
+
+This will setup the HDFS namenode to listen on port 40000.
+The webUI will run on default port 50070.
+The installation used here is based on Cloudera's CDH4 release.
+
+(1) Ensure java is installed, and $JAVA_HOME is set.
+
+(2) Add the following to /etc/yum.repos.d/thirdparty.repo (sudo needed)
+ -----------------------------------
+ [cloudera-cdh4]
+ name=Cloudera's Distribution for Hadoop, Version 4
+ baseurl=http://archive.cloudera.com/cdh4/redhat/6/x86_64/cdh/4/
+ gpgkey = http://archive.cloudera.com/cdh4/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera
+ gpgcheck = 1
+ -----------------------------------
+
+(3) Install hadoop-hdfs-namenode and update the configs.
+ sudo yum install hadoop-hdfs-namenode
+ sudo mv /etc/hadoop/conf /etc/hadoop/conf.orig
+ sudo cp -r /etc/hadoop/conf.empty /etc/hadoop/conf
+
+(4) Update /etc/hadoop/conf/core-site.xml (enter your server name instead
+ of 10.20.30.255)
+ ----------------------------------
+ <configuration>
+ <property>
+ <name>fs.default.name</name>
+ <value>hdfs://10.20.30.255:40000</value>
+ </property>
+ </configuration>
+ ----------------------------------
+
+(5) Edit /etc/hadoop/conf/hdfs-site.xml, fix or ensure that there is
+ a "file://" prefix to avoid warnings.
+ ----------------------------------
+ <configuration>
+ <property>
+ <name>dfs.name.dir</name>
+ <value>file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
+ </property>
+ </configuration>
+ ----------------------------------
+
+(6) Format the namenode:
+ sudo service hadoop-hdfs-namenode init
+
+(7) Start namenode.
+ sudo service hadoop-hdfs-namenode start
+
+(8) Now namenode should be running. Confirm this by running,
+ ps aux | grep java
+ sudo netstat -pan | grep 40000
+
+(9) To administer the files and directories,
+ /usr/lib/hadoop/bin/hadoop fs -ls /
+
+(10) The user with write access on this namenode is "hdfs". Therefore, give
+ write permission to "/" folder (for mstress benchmark to use) by logging
+ in as "hdfs" user.
+ sudo bash
+ su hdfs
+ JAVA_HOME=<java-home> /usr/lib/hadoop/bin/hadoop fs -chmod 777 /
+ exit
+
+(11) Now the namenode is ready for running benchmarks.
+
View
445 benchmarks/mstress/mstress.py
@@ -0,0 +1,445 @@
+#!/usr/bin/env python
+
+# $Id$
+#
+# Author: Thilee Subramaniam
+#
+# Copyright 2012 Quantcast Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# This program invokes given number of client processes on the given set of
+# remote clients (Java and C++) and makes use of the plan file to apply load
+# on the DFS server.
+
+import optparse
+import sys
+import subprocess
+import time
+import os
+import signal
+import datetime
+import commands
+import resource
+import re
+
+class Globals:
+ MASTER_PATH = ''
+ CLIENT_PATH = ''
+ MSTRESS_LOCK = '/tmp/mstress_master.lock'
+ SIGNALLED = False
+ SERVER_CMD = ""
+ SERVER_KEYWORD = ""
+ KFS_SERVER_CMD = "metaserver"
+ KFS_SERVER_KEYWORD = "metaserver"
+ HDFS_SERVER_CMD = "java"
+ HDFS_SERVER_KEYWORD = "NameNode"
+
+
+def ParseCommandline():
+ parser = optparse.OptionParser()
+ parser.add_option('-m', '--mode',
+ action='store',
+ default='master',
+ type='string',
+ help='Run as master or slave')
+ parser.add_option('-f', '--filesystem',
+ action='store',
+ default=None,
+ type='string',
+ help='Filesystem whose metaserver to test. kfs or hdfs.')
+ parser.add_option('-s', '--server',
+ action='store',
+ default=None,
+ type='string',
+ help='Metaserver or Namenode hostname.')
+ parser.add_option('-p', '--port',
+ action='store',
+ default=None,
+ type='int',
+ help='Metaserver or Namenode port')
+ parser.add_option('-c', '--client-hostname',
+ action='store',
+ default=None,
+ type='string',
+ help='mstress slave\'s hostname (slave only option).')
+ parser.add_option('-k', '--client-lookup-key',
+ action='store',
+ default=None,
+ type='string',
+ help='mstress slave\'s lookup key to be used (slave only option).')
+ parser.add_option('-t', '--client-testname',
+ action='store',
+ default=None,
+ type='string',
+ help='Test to run on mstress slave (slave only option).')
+ parser.add_option('-a', '--plan',
+ action='store',
+ default=None,
+ type='string',
+ help='Plan file containing client instructions.')
+ parser.add_option('-l', '--leave-files', action='store_true',
+ default=False, help='Leave files. Does not perform delete test.')
+
+ opts, args = parser.parse_args()
+ if args:
+ sys.exit('Unexpected arguments: %s.' % str(args))
+
+ if not opts.filesystem or not opts.server or not opts.port or not opts.plan:
+ sys.exit('Missing mandatory arguments.')
+ if opts.mode not in ('master', 'slave'):
+ sys.exit('Invalid mode.')
+ if opts.mode == 'master':
+ # master should not have -c option
+ if opts.client_hostname is not None:
+ sys.exit('Master: does not support -c option.')
+ if opts.client_testname is not None:
+ sys.exit('Master: does not support -t option.')
+ else:
+ # for slave, this is the slave host name.
+ hosts = opts.client_hostname.split(',')
+ if len(hosts) != 1:
+ sys.exit('Slave: Error in client host name.')
+ if opts.client_testname is None or opts.client_lookup_key is None:
+ sys.exit('Slave: Error in client test name or lookup key.')
+
+ return opts
+
+
+def PrintMemoryUsage(opts):
+ proc = subprocess.Popen(['ssh', opts.server,
+ 'ps -C %s -o rss,pid,cmd |grep %s'%(Globals.SERVER_CMD,Globals.SERVER_KEYWORD)],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ print "Memory usage %dKB" % int(re.search(r'^(\d+)\s+', proc.communicate()[0]).group(1))
+
+
+def RunMStressMaster(opts, hostsList):
+ """ Called when run in master mode. Calls master funcions for 'create',
+ 'stat', and 'readdir'.
+
+ Args:
+ opts: options object, from parsed commandine options.
+ hostsList: list of hosts obtained from plan file.
+
+ Returns:
+ None
+ """
+
+ # print 'Master: called with %r, %r' % (opts, hostsList)
+
+ startTime = datetime.datetime.now()
+ if RunMStressMasterTest(opts, hostsList, 'create') == False:
+ return
+ deltaTime = datetime.datetime.now() - startTime
+ print '\nMaster: Create test took %d.%d sec' % (deltaTime.seconds, deltaTime.microseconds/1000000)
+ PrintMemoryUsage(opts)
+ print '=========================================='
+
+ startTime = datetime.datetime.now()
+ if RunMStressMasterTest(opts, hostsList, 'stat') == False:
+ return
+ deltaTime = datetime.datetime.now() - startTime
+ print '\nMaster: Stat test took %d.%d sec' % (deltaTime.seconds, deltaTime.microseconds/1000000)
+ print '=========================================='
+
+ startTime = datetime.datetime.now()
+ if RunMStressMasterTest(opts, hostsList, 'readdir') == False:
+ return
+ deltaTime = datetime.datetime.now() - startTime
+ print '\nMaster: Readdir test took %d.%d sec' % (deltaTime.seconds, deltaTime.microseconds/1000000)
+ print '=========================================='
+
+ if opts.leave_files:
+ print "\nNot deleting files because of -l option"
+ return
+
+ startTime = datetime.datetime.now()
+ if RunMStressMasterTest(opts, hostsList, 'delete') == False:
+ return
+ deltaTime = datetime.datetime.now() - startTime
+ print '\nMaster: Delete test took %d.%d sec' % (deltaTime.seconds, deltaTime.microseconds/1000000)
+ print '=========================================='
+
+
+def RunMStressMasterTest(opts, hostsList, test):
+ """ Called when run in master mode. Invokes the slave version of the same
+ program on the provided hosts list with the given test name.
+
+ Args:
+ opts: parsed commandline options.
+ hostsList: list of hosts obtained from plan file.
+ test: string: test name to call.
+
+ Returns:
+ False on error, True on success
+ """
+ if Globals.SIGNALLED:
+ return False
+
+ # invoke remote master client.
+ ssh_cmd = '%s -m slave -f %s -s %s -p %d -t %s -a %s'%(
+ Globals.MASTER_PATH,
+ opts.filesystem,
+ opts.server,
+ opts.port,
+ test,
+ opts.plan)
+ clientHostMapping = MapHostnameForTest(hostsList, test)
+ running_procs = {}
+
+ for client in hostsList:
+ slaveLogfile = opts.plan + '_' + client + '_' + test + '_' + opts.filesystem + '.slave.log'
+ p = subprocess.Popen(['/usr/bin/ssh', client,
+ '%s -c %s -k %s >& %s' % (ssh_cmd, client, clientHostMapping[client], slaveLogfile)],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ running_procs[p] = client
+
+ isLine1 = True
+ while running_procs:
+ tobedelkeys = []
+ for proc in running_procs.iterkeys():
+ client = running_procs[proc]
+ retcode = proc.poll()
+ if retcode is not None:
+ sout,serr = proc.communicate()
+ if sout:
+ print '\nMaster: output of slave (%s):%s' % (client, sout)
+ if serr:
+ print '\nMaster: err of slave (%s):%s' % (client, serr)
+ tobedelkeys.append(proc)
+ else:
+ if Globals.SIGNALLED:
+ proc.terminate()
+
+ for k in tobedelkeys:
+ del running_procs[k]
+
+ if running_procs:
+ if isLine1:
+ sys.stdout.write('Master: remote slave running \'%s\'' % test)
+ isLine1 = False
+ else:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ time.sleep(0.5)
+ return True
+
+def MapHostnameForTest(clients, test):
+ """ Determines the '-c' argument to use for slave invocation. This argument
+ is passed to the C++/Java client so that the client can use it as a key
+ to read the plan file.
+
+ For 'create', this name is the same as the client name. But for doing
+ a 'stat' or a 'readdir' we want to run the tests on a client different
+ from the one that created the path.
+ Args:
+ clients: list of strings, clients.
+ test: string, the name of the test.
+
+ Returns:
+ map of strings, client name to '-c' argument.
+ """
+ mapping = {}
+ length = len(clients)
+ for i in range(0, length):
+ if test == 'stat' or test == 'readdir':
+ mapping[clients[i]] = clients[(i+1)%length]
+ else:
+ mapping[clients[i]] = clients[i]
+
+ return mapping
+
+def RunMStressSlave(opts, clientsPerHost):
+ """ Called when the code is run in slave mode, on each slave.
+ Invokes number of client processes equal to 'clientsPerHost'.
+
+ Args:
+ opts: parsed commandline options.
+ clientsPerHost: integer, number of processes to run on each host.
+
+ Returns:
+ None
+ """
+
+ print 'Slave: called with %r, %d' % (opts, clientsPerHost)
+ os.putenv('KFS_CLIENT_DEFAULT_FATTR_REVALIDATE_TIME',"-1")
+
+ running_procs = []
+ for i in range(0, clientsPerHost):
+ clientLogfile = '%s_%s_proc_%02d_%s_%s.client.log' % (opts.plan, opts.client_hostname, i, opts.client_testname, opts.filesystem)
+ args = ["%s -s %s -p %s -a %s -c %s -t %s -n proc_%02d >& %s" % (
+ Globals.CLIENT_PATH,
+ opts.server,
+ str(opts.port),
+ opts.plan,
+ opts.client_lookup_key,
+ opts.client_testname,
+ i,
+ clientLogfile)]
+ print 'Slave: args = %r' % args
+ p = subprocess.Popen(args,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ running_procs.append(p)
+
+ isLine1 = True
+ while running_procs:
+ for proc in running_procs:
+ ret = proc.poll()
+ if ret is not None:
+ sout,serr = proc.communicate()
+ if sout:
+ print '\nSlave: output of (ClientHost %s, ClientNo %r):%s' % (opts.client_hostname, proc, sout)
+ if serr:
+ print '\nSlave: err of (ClientHost %s, ClientNo %r):%s' % (opts.client_hostname, proc, serr)
+ running_procs.remove(proc)
+ else:
+ if Globals.SIGNALLED:
+ proc.terminate()
+
+ if running_procs:
+ if isLine1:
+ sys.stdout.write('Slave: load client \'%s\' running' % opts.client_testname)
+ isLine1 = False
+ else:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ time.sleep(0.5)
+
+
+def ReadPlanFile(opts):
+ """ Reads the given plan file to extract the list of client-hosts and
+ process-count per client-host.
+
+ Args:
+ opts: parsed commandline options.
+
+ Returns:
+ hostslist: list of client host names
+ clientsPerHost: integer: client processes per client host.
+ """
+
+ hostsList = None
+ clientsPerHost = None
+ leafType = None
+ numLevels = None
+ numToStat = None
+ nodesPerLevel = None
+
+ planfile = open(opts.plan, 'r')
+ for line in planfile:
+ if line.startswith('#'):
+ continue
+ if line.startswith('hostslist='):
+ hostsList = line[len('hostslist='):].strip().split(',')
+ elif line.startswith('clientsperhost='):
+ clientsPerHost = int(line[len('clientsperhost='):].strip())
+ elif line.startswith('type='):
+ leafType = line[len('type='):].strip()
+ elif line.startswith('levels='):
+ numLevels = int(line[len('levels='):].strip())
+ elif line.startswith('nstat='):
+ numToStat = int(line[len('nstat='):].strip())
+ elif line.startswith('inodes='):
+ nodesPerLevel = int(line[len('inodes='):].strip())
+ planfile.close()
+ if None in (hostsList, clientsPerHost, leafType, numLevels, numToStat, nodesPerLevel):
+ sys.exit('Failed to read plan file')
+
+ nodesPerProcess = 0
+ leafNodesPerProcess = 0
+ for l in range(1,numLevels+1):
+ nodesPerProcess += pow(nodesPerLevel,l)
+ if l == numLevels:
+ leafNodesPerProcess = pow(nodesPerLevel,l)
+ inters = nodesPerProcess - leafNodesPerProcess
+ overallNodes = nodesPerProcess * len(hostsList) * clientsPerHost
+ overallLeafs = leafNodesPerProcess * len(hostsList) * clientsPerHost
+ intermediateNodes = inters * len(hostsList) * clientsPerHost + len(hostsList) * clientsPerHost + 1
+ totalNumToStat = numToStat * len(hostsList) * clientsPerHost
+
+ print ('Plan:\n' +
+ ' o %d client processes on each of %d hosts will generate load.\n' % (clientsPerHost, len(hostsList)) +
+ ' o %d levels of %d nodes (%d leaf nodes, %d total nodes) will be created by each client process.\n' % (numLevels, nodesPerLevel, leafNodesPerProcess, nodesPerProcess) +
+ ' o Overall, %d leaf %ss will be created, %d intermediate directories will be created.\n' % (overallLeafs, leafType, intermediateNodes) +
+ ' o Stat will be done on a random subset of %d leaf %ss by each client process, totalling %d stats.\n' % (numToStat, leafType, totalNumToStat) +
+ ' o Readdir (non-overlapping) will be done on the full file tree by all client processes.\n')
+ return hostsList, clientsPerHost
+
+
+def SetGlobalPaths(opts):
+ mydir = os.path.dirname(os.path.realpath(__file__))
+ Globals.MASTER_PATH = os.path.join(mydir, 'mstress.py')
+
+ if opts.filesystem == 'kfs':
+ Globals.CLIENT_PATH = os.path.join(mydir, 'mstress_client')
+ Globals.SERVER_CMD = Globals.KFS_SERVER_CMD
+ Globals.SERVER_KEYWORD = Globals.KFS_SERVER_KEYWORD
+ elif opts.filesystem == 'hdfs':
+ hdfsjars = commands.getoutput("echo %s/mstress_hdfs_client_jars/*.jar | sed 's/ /:/g'" % mydir)
+ Globals.CLIENT_PATH = 'java -Xmx256m -cp %s:%s MStress_Client' % (mydir,hdfsjars)
+ Globals.SERVER_CMD = Globals.HDFS_SERVER_CMD
+ Globals.SERVER_KEYWORD = Globals.HDFS_SERVER_KEYWORD
+ else:
+ sys.exit('Invalid filesystem option')
+
+def CreateLock(opts):
+ if opts.mode != 'master':
+ return
+ if os.path.exists(Globals.MSTRESS_LOCK):
+ sys.exit('Program already running. Please wait till it finishes')
+ f = open(Globals.MSTRESS_LOCK, 'w')
+ f.write(str(os.getpid()))
+ f.close()
+
+def RemoveLock(opts):
+ if opts.mode != 'master':
+ return
+ if os.path.exists(Globals.MSTRESS_LOCK):
+ f = open(Globals.MSTRESS_LOCK, 'r')
+ pid = f.read()
+ f.close()
+ if int(pid) == os.getpid():
+ os.unlink(Globals.MSTRESS_LOCK)
+
+def HandleSignal(signum, frame):
+ print "Received signal, %d" % signum
+ Globals.SIGNALLED = True
+
+def main():
+ signal.signal(signal.SIGTERM, HandleSignal)
+ signal.signal(signal.SIGINT, HandleSignal)
+ signal.signal(signal.SIGHUP, HandleSignal)
+
+ opts = ParseCommandline()
+
+ SetGlobalPaths(opts)
+
+ CreateLock(opts)
+
+ try:
+ (hostsList,clientsPerHost) = ReadPlanFile(opts)
+
+ if opts.mode == 'master':
+ RunMStressMaster(opts, hostsList)
+ else:
+ RunMStressSlave(opts, clientsPerHost)
+ finally:
+ RemoveLock(opts)
+
+if __name__ == '__main__':
+ main()
+
View
69 benchmarks/mstress/mstress_cleanup.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+#
+# $Id$
+#
+# Copyright 2012 Quantcast Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# This is a helper script to cleanup the planfile and the logs from all
+# participating hosts.
+#
+
+import optparse
+import sys
+import subprocess
+import time
+import os
+import signal
+import datetime
+import commands
+
+if len(sys.argv) < 2 or sys.argv[1].startswith('-'):
+ print 'Usage: %s <planfile>\nThis will cleanup the planfile and the logs from all participating hosts.' % sys.argv[0]
+ sys.exit(0)
+
+if not sys.argv[1].startswith('/tmp'):
+ print 'Planfile is typically in the /tmp directory. Are you sure?'
+ sys.exit(1)
+
+planFile = sys.argv[1]
+hostsList = None
+f = None
+
+try:
+ f = open(planFile, 'r')
+except IOError, e:
+ print 'Planfile not found'
+ sys.exit(1)
+
+for line in f:
+ if line.startswith('#'):
+ continue
+ if line.startswith('hostslist='):
+ hostsList = line[len('hostslist='):].strip().split(',')
+ break
+f.close()
+
+if len(hostsList) == 0:
+ print 'No hosts list found in plan file. Exiting.'
+ sys.exit(1)
+
+for host in hostsList:
+ cmd = 'ssh %s "rm -f %s*"' % (host, planFile)
+ print 'Executing "%s"' % cmd
+ print commands.getoutput(cmd)
+
+print 'Done'
+
View
582 benchmarks/mstress/mstress_client.cc
@@ -0,0 +1,582 @@
+/**
+ * $Id$
+ *
+ * Author: Thilee Subramaniam
+ *
+ * Copyright 2012 Quantcast Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * This C++ client performs filesystem meta opetarions on the KFS metaserver
+ * using kfsClient.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <math.h>
+#include <string.h>
+#include <sys/time.h>
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <vector>
+#include <queue>
+#include <algorithm>
+
+using namespace std;
+
+#include "KfsClient.h"
+
+FILE* logFile = stdout;
+
+#define TEST_BASE_DIR "/mstress"
+#define COUNT_INCR 500
+
+/*
+ This program is invoked with the following arguments:
+ - kfs server/port
+ - test name ('create', 'stat', or 'readdir')
+ - a planfile
+ - keys to read the planfile (hostname and process name)
+
+ eg: For the following plan file,
+ ---------------------------------------------
+ #File or directory
+ type=file
+ #Number of levels in created tree
+ levels=2
+ #Number of inodes per level
+ inodes=3
+ #Number of random leaf paths to stat, per client
+ nstat=17
+ ---------------------------------------------
+ if 'create' testname, '127.0.0.1' hostname, and 'proc_00' processname
+ 'PPP' path_prefix are given, then the created tree would be:
+ /mstress/127.0.0.1_proc_00/PPP_0/PPP_0
+ /mstress/127.0.0.1_proc_00/PPP_0/PPP_1
+ /mstress/127.0.0.1_proc_00/PPP_0/PPP_2
+ /mstress/127.0.0.1_proc_00/PPP_1/PPP_0
+ /mstress/127.0.0.1_proc_00/PPP_1/PPP_1
+ /mstress/127.0.0.1_proc_00/PPP_1/PPP_2
+ /mstress/127.0.0.1_proc_00/PPP_2/PPP_0
+ /mstress/127.0.0.1_proc_00/PPP_2/PPP_1
+ /mstress/127.0.0.1_proc_00/PPP_2/PPP_2
+*/
+
+
+//Global datastructure to hold various options.
+struct Client {
+ static const size_t INITIAL_SIZE;
+ struct Path
+ {
+ char* actualPath_;
+ size_t actualSize_;
+ size_t len_;
+
+ Path() : actualSize_(INITIAL_SIZE), len_(0) {
+ actualPath_ = (char*)calloc(actualSize_, 1);
+ }
+
+ void Push(const char* leafStr) {
+ size_t leafLen = strlen(leafStr);
+ if (leafLen == 0) {
+ return;
+ }
+ if (len_ + 1 + leafLen + 1 > actualSize_) {
+ actualSize_ *= 2;
+ actualPath_ = (char*)realloc(actualPath_, actualSize_);
+ fprintf(logFile, "Reallocating to %zu bytes\n", actualSize_);
+ }
+ if (leafStr[0] != '/') {
+ strcpy(actualPath_ + len_, "/");
+ len_ ++;
+ }
+ strcpy(actualPath_ + len_, leafStr);
+ len_ += leafLen;
+ }
+
+ void Pop(const char* leafStr) {
+ size_t leafLen = strlen(leafStr);
+ if (leafLen > len_ - 1 ||
+ strncmp(actualPath_ + len_ - leafLen, leafStr, leafLen)) {
+ fprintf(logFile, "Error in pop %s from %s\n", leafStr, actualPath_);
+ exit(0);
+ }
+ len_ -= leafLen + 1;
+ *(actualPath_ + len_) = 0;
+ }
+
+ void Reset() {
+ actualPath_[0] = 0;
+ len_ = 0;
+ }
+ };
+
+ Path path_;
+
+ //from commadline
+ string dfsServer_;
+ int dfsPort_;
+ string testName_;
+ string planfilePath_;
+ string prefix_;
+ size_t prefixLen_;
+ string hostName_;
+ string processName_;
+
+ //from planfile
+ string type_;
+ int levels_;
+ int inodesPerLevel_;
+ int pathsToStat_;
+};
+const size_t Client::INITIAL_SIZE = 1 << 12;
+
+//A simple AutoDoUndo class to delete kfsClient automatically.
+class AutoCleanupKfsClient
+{
+public:
+ AutoCleanupKfsClient(Client* client) : initialized(false)
+ {
+ kfsClient = KFS::Connect(client->dfsServer_, client->dfsPort_);
+ if (kfsClient) {
+ initialized = true;
+ }
+ }
+ ~AutoCleanupKfsClient() {
+ delete kfsClient;
+ }
+ bool IsInitialized() { return initialized; }
+ KFS::KfsClient* GetClient() { return kfsClient; }
+
+private:
+ KFS::KfsClient* kfsClient;
+ bool initialized;
+};
+
+
+void Usage(const char* argv0)
+{
+ fprintf(logFile, "Usage: %s -s dfs-server -p dfs-port [-t [create|stat|readdir|delete] -a planfile-path -c host -n process-name -P path-prefix]\n", argv0);
+ fprintf(logFile, " -t: this option requires -a, -c, and -n options.\n");
+ fprintf(logFile, " -P: the default value is PATH_.\n");
+ fprintf(logFile, "eg:\n%s -s <metaserver-host> -p <metaserver-port> -t create -a <planfile> -c localhost -n Proc_00\n", argv0);
+ exit(0);
+}
+
+void hexout(char* str, int len) {
+ for (int i = 0; i < len; i++) {
+ printf("%02X ", str[i]);
+ }
+ printf("\n");
+ for (int i = 0; i < len; i++) {
+ printf("%2c ", str[i] < 30 ? '.' : str[i]);
+ }
+ printf("\n");
+}
+
+void myitoa(int n, char* buf)
+{
+ static char result[32];
+ snprintf(result, 32, "%d", n);
+ strcpy(buf, result);
+}
+
+//Return a random permutation of numbers in [0..range).
+void unique_random(vector<size_t>& result, size_t range)
+{
+ result.resize(range);
+ srand(time(NULL));
+
+ for(size_t i=0; i<range; ++i) {
+ result[i] = i;
+ }
+
+ random_shuffle(result.begin(), result.end() ) ;
+}
+
+void parse_options(int argc, char* argv[], Client* client)
+{
+ int c = 0;
+ char *dfs_server = NULL,
+ *dfs_port = NULL,
+ *test_name = NULL,
+ *planfile = NULL,
+ *host = NULL,
+ *prefix = NULL,
+ *process_name = NULL;
+
+ while ((c = getopt(argc, argv, "s:p:a:c:n:t:P:h")) != -1) {
+ switch (c) {
+ case 's':
+ dfs_server = optarg;
+ break;
+ case 'p':
+ dfs_port = optarg;
+ break;
+ case 'a':
+ planfile = optarg;
+ break;
+ case 'c':
+ host = optarg;
+ break;
+ case 'n':
+ process_name = optarg;
+ break;
+ case 't':
+ test_name = optarg;
+ break;
+ break;
+ case 'P':
+ prefix = optarg;
+ break;
+ case 'h':
+ case '?':
+ Usage(argv[0]);
+ break;
+ default:
+ fprintf (logFile, "?? getopt returned character code 0%o ??\n", c);
+ Usage(argv[0]);
+ break;
+ }
+ }
+
+ if (!client || !dfs_server || !dfs_port || !planfile || !host || !process_name || !test_name) {
+ Usage(argv[0]);
+ }
+ ifstream ifs(planfile, ifstream::in);
+ if (ifs.fail()) {
+ fprintf(stdout, "Error: planfile not found\n");
+ Usage(argv[0]);
+ }
+ ifs.close();
+
+ client->testName_ = test_name;
+ client->dfsServer_ = dfs_server;
+ client->dfsPort_ = atoi(dfs_port);
+ client->planfilePath_ = planfile;
+ client->hostName_ = host;
+ client->processName_ = process_name;
+ if (!prefix) {
+ client->prefix_ = "PATH_";
+ } else {
+ client->prefix_ = prefix;
+ }
+ client->prefixLen_ = client->prefix_.size();
+
+ fprintf(logFile, "server=%s\n", dfs_server);
+ fprintf(logFile, "port=%s\n", dfs_port);
+ fprintf(logFile, "planfile=%s\n", planfile);
+ fprintf(logFile, "host=%s\n", host);
+ fprintf(logFile, "process_name=%s\n", process_name);
+ fprintf(logFile, "test name=%s\n", test_name);
+}
+
+//Reads the plan file and add the level information to distribution_ vector.
+//Also set type_, prefix_, levels_, pathsToStat_ class variables.
+void ParsePlanFile(Client* client)
+{
+ string line;
+ ifstream ifs(client->planfilePath_.c_str(), ifstream::in);
+
+ while (ifs.good()) {
+ getline(ifs, line);
+ if (line.empty() || line[0] == '#') {
+ continue;
+ }
+ if (line.substr(0, 5) == "type=") {
+ client->type_ = line.substr(5);
+ continue;
+ }
+ if (line.substr(0, 7) == "levels=") {
+ client->levels_ = atoi(line.substr(7).c_str());
+ continue;
+ }
+ if (line.substr(0, 7) == "inodes=") {
+ client->inodesPerLevel_ = atoi(line.substr(7).c_str());
+ continue;
+ }
+ if (line.substr(0, 6) == "nstat=") {
+ client->pathsToStat_ = atoi(line.substr(6).c_str());
+ continue;
+ }
+ }
+ ifs.close();
+ if (client->prefix_.empty() || client->levels_ <= 0 && client->inodesPerLevel_ <= 0) {
+ fprintf(logFile, "Error parsing plan file\n");
+ exit(-1);
+ }
+}
+
+long TimeDiffMilliSec(struct timeval* alpha, struct timeval* zigma)
+{
+ long diff = 0;
+ diff += (zigma->tv_sec - alpha->tv_sec) * 1000;
+ diff += (zigma->tv_usec - alpha->tv_usec) / 1000;
+ return diff < 0 ? 0 : diff;
+}
+
+
+int CreateDFSPaths(Client* client, AutoCleanupKfsClient* kfs, int level, int* createdCount)
+{
+
+ KFS::KfsClient* kfsClient = kfs->GetClient();
+ int rc;
+ bool isLeaf = (level + 1 >= client->levels_);
+ bool isDir =isLeaf ? (client->type_ == "dir") : true;
+ char name[512];
+ strncpy(name, client->prefix_.c_str(), 512);
+ for (int i = 0; i < client->inodesPerLevel_; i++) {
+ myitoa(i, name + client->prefixLen_);
+ client->path_.Push(name);
+ //hexout(client->path_.actualPath_, client->path_.len_ + 3);
+
+ if (isDir) {
+ //fprintf(logFile, "Creating DIR [%s]\n", client->path_.actualPath_);
+ rc = kfsClient->Mkdir(client->path_.actualPath_);
+ (*createdCount)++;
+ if (*createdCount > 0 && (*createdCount) % COUNT_INCR == 0) {
+ fprintf(logFile, "Created paths so far: %d\n", *createdCount);
+ }
+ if (!isLeaf) {
+ CreateDFSPaths(client, kfs, level+1, createdCount);
+ }
+ } else {
+ //fprintf(logFile, "Creating file [%s]\n", client->path_.actualPath_);
+ rc = kfsClient->Create(client->path_.actualPath_);
+ (*createdCount)++;
+ if (*createdCount > 0 && (*createdCount) % COUNT_INCR == 0) {
+ fprintf(logFile, "Created paths so far: %d\n", *createdCount);
+ }
+ }
+ client->path_.Pop(name);
+ }
+}
+
+int CreateDFSPaths(Client* client, AutoCleanupKfsClient* kfs)
+{
+ KFS::KfsClient* kfsClient = kfs->GetClient();
+ ostringstream os;
+ os << TEST_BASE_DIR << "/" << client->hostName_ + "_" << client->processName_;
+ int err = kfsClient->Mkdirs(os.str().c_str());
+ //fprintf(logFile, "first mkdir err = %d\n", err);
+ if (err && err != EEXIST) {
+ fprintf(logFile, "Error: mkdir test base dir failed\n");
+ exit(-1);
+ }
+
+ int createdCount = 0;
+ struct timeval tvAlpha;
+ gettimeofday(&tvAlpha, NULL);
+
+ client->path_.Reset();
+ client->path_.Push(os.str().c_str());
+ if (CreateDFSPaths(client, kfs, 0, &createdCount) < 0) {
+ fprintf(logFile, "Error: failed to create DFS paths\n");
+ }
+
+ struct timeval tvZigma;
+ gettimeofday(&tvZigma, NULL);
+ fprintf(logFile, "Client: %d paths created in %ld msec\n", createdCount, TimeDiffMilliSec(&tvAlpha, &tvZigma));
+}
+
+int StatDFSPaths(Client* client, AutoCleanupKfsClient* kfs) {
+ KFS::KfsClient* kfsClient = kfs->GetClient();
+
+ ostringstream os;
+ os << TEST_BASE_DIR << "/" << client->hostName_ + "_" << client->processName_;
+
+ srand(time(NULL));
+ struct timeval tvAlpha;
+ gettimeofday(&tvAlpha, NULL);
+
+ for (int count = 0; count < client->pathsToStat_; count++) {
+ client->path_.Reset();
+ client->path_.Push(os.str().c_str());
+ char name[4096];
+ strncpy(name, client->prefix_.c_str(), client->prefixLen_);
+
+ for (int d = 0; d < client->levels_; d++) {
+ int randIdx = rand() % client->inodesPerLevel_;
+ myitoa(randIdx, name + client->prefixLen_);
+ client->path_.Push(name);
+ //fprintf(logFile, "Stat: path now is %s\n", client->path_.actualPath_);
+ }
+ //fprintf(logFile, "Stat: doing stat on [%s]\n", client->path_.actualPath_);
+
+ KFS::KfsFileAttr attr;
+ int err = kfsClient->Stat(os.str().c_str(), attr);
+ if (err) {
+ fprintf(logFile, "error doing stat on %s\n", os.str().c_str());
+ }
+
+ if (count > 0 && count % COUNT_INCR == 0) {
+ fprintf(logFile, "Stat paths so far: %d\n", count);
+ }
+ }
+
+ struct timeval tvZigma;
+ gettimeofday(&tvZigma, NULL);
+ fprintf(logFile, "Client: Stat done on %d paths in %ld msec\n", client->pathsToStat_, TimeDiffMilliSec(&tvAlpha, &tvZigma));
+
+ return 0;
+}
+
+int ListDFSPaths(Client* client, AutoCleanupKfsClient* kfs) {
+ KFS::KfsClient* kfsClient = kfs->GetClient();
+
+ srand(time(NULL));
+ struct timeval tvAlpha;
+ gettimeofday(&tvAlpha, NULL);
+ int inodeCount = 0;
+
+ queue<string> pending;
+ ostringstream os;
+ os << TEST_BASE_DIR << "/" << client->hostName_ + "_" << client->processName_;
+ pending.push(os.str());
+
+ while (!pending.empty()) {
+ string parent = pending.front();
+ pending.pop();
+ //fprintf(logFile, "readdir on parent [%s]\n", parent.c_str());
+ vector<KFS::KfsFileAttr> children;
+ int err = kfsClient->ReaddirPlus(parent.c_str(), children);
+ if (err) {
+ fprintf(logFile, "Error [err=%d] reading directory %s\n", err, parent.c_str());
+ continue;
+ }
+ while (!children.empty()) {
+ string child = children.back().filename;
+ bool isDir = children.back().isDirectory;
+ children.pop_back();
+ //fprintf(logFile, " Child = %s inodeCount=%d\n", child.c_str(), inodeCount);
+ if (child == "." ||
+ child == "..") {
+ continue;
+ }
+ inodeCount ++;
+ if (isDir) {
+ string nextParent = parent + "/" + child;
+ pending.push(nextParent);
+ //fprintf(logFile, " Adding next parent [%s]\n", nextParent.c_str());
+ }
+ if (inodeCount > 0 && inodeCount % COUNT_INCR == 0) {
+ fprintf(logFile, "Readdir paths so far: %d\n", inodeCount);
+ }
+ }
+ }
+
+ struct timeval tvZigma;
+ gettimeofday(&tvZigma, NULL);
+ fprintf(logFile, "Client: Directory walk done over %d inodes in %ld msec\n", inodeCount, TimeDiffMilliSec(&tvAlpha, &tvZigma));
+ return 0;
+}
+
+int RemoveDFSPaths(Client* client, AutoCleanupKfsClient* kfs) {
+ KFS::KfsClient* kfsClient = kfs->GetClient();
+ KFS::KfsFileAttr attr;
+
+ ostringstream os;
+ os << TEST_BASE_DIR << "/" << client->hostName_ + "_" << client->processName_;
+
+ // get a list of leaf indices to remove. Note that this is different from the nodename suffix.
+ // eg: if we have 3 levels of 4 inodes, then there will be pow(4,3) = 64 leaf nodes. We are
+ // interested in a subset of indices in (0..63). This gets filled in 'leafIdxRangeForDel'.
+ long double countLeaf = pow(client->inodesPerLevel_, client->levels_);
+ vector<size_t> leafIdxRangeForDel;
+ unique_random(leafIdxRangeForDel, countLeaf);
+ fprintf(logFile, "To delete %d paths\n", leafIdxRangeForDel.size());
+
+ struct timeval tvAlpha;
+ gettimeofday(&tvAlpha, NULL);
+ bool isLeafDir = client->type_=="dir";
+
+ char sfx[32];
+ string pathToDel;
+ string pathSoFar;
+ size_t idx = 0;
+ size_t pos = 0;
+ int delta = 0;
+ int lev = 0;
+
+ while (!leafIdxRangeForDel.empty()) {
+ idx = leafIdxRangeForDel.back();
+ leafIdxRangeForDel.pop_back();
+ pathSoFar.clear();
+ pos = 0;
+ delta = 0;
+ lev = 0;
+ while (lev < client->levels_) {
+ pos = idx / client->inodesPerLevel_;
+ delta = idx - (pos * client->inodesPerLevel_);
+ myitoa(delta, sfx);
+ if (pathSoFar.length()) {
+ pathSoFar = client->prefix_ + sfx + "/" + pathSoFar;
+ } else {
+ pathSoFar = client->prefix_ + sfx;
+ }
+ idx = pos;
+ lev ++;
+ }
+
+ pathToDel = os.str() + "/" + pathSoFar;
+ fprintf(logFile, "Client: Deleting %s ...\n", pathToDel.c_str());
+ if (isLeafDir) {
+ kfsClient->Rmdir(pathToDel.c_str());
+ } else {
+ kfsClient->Remove(pathToDel.c_str());
+ }
+ }
+
+ kfsClient->RmdirsFast(os.str().c_str());
+
+ struct timeval tvZigma;
+ gettimeofday(&tvZigma, NULL);
+ fprintf(logFile, "Client: Deleted %s. Delete took %ld msec\n", os.str().c_str(), TimeDiffMilliSec(&tvAlpha, &tvZigma));
+
+ return 0;
+}
+
+
+int main(int argc, char* argv[])
+{
+ Client client;
+
+ parse_options(argc, argv, &client);
+
+ AutoCleanupKfsClient kfs(&client);
+ if (!kfs.IsInitialized()) {
+ fprintf(logFile, "kfs client failed to initialize. exiting.\n");
+ exit(-1);
+ }
+
+ ParsePlanFile(&client);
+
+ if (client.testName_ == "create") {
+ CreateDFSPaths(&client, &kfs);
+ } else if (client.testName_ == "stat") {
+ StatDFSPaths(&client, &kfs);
+ } else if (client.testName_ == "readdir") {