Merge branch 'release/1.1.4'

leo-project · Oct 3, 2014 · 1ea7bd5 · 1ea7bd5
2 parents e004a01 + 2659fb5
commit 1ea7bd5
Show file tree

Hide file tree

Showing 15 changed files with 330 additions and 83 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,50 @@
 CHANGELOG
 =========
 
+1.1.4 (Oct 3, 2014)
+====================
+
+* Improvements
+    * Strictly checked routing-table between manager-node and other nodes
+    * Implemented the data diagnosis function of a storage-node
+    * [#249](https://github.com/leo-project/leofs/issues/249) ``leo_manager`` Improve whereis command
+* Fixed Bugs
+    * [#246](https://github.com/leo-project/leofs/issues/246) ``leo_storage`` Could not recursively creating directory
+    * [#247](https://github.com/leo-project/leofs/issues/247) ``leo_storage`` Start normally even if error occur
+    * [#250](https://github.com/leo-project/leofs/issues/250) ``leo_statistics`` leo_statistics_sampler can be down under very high load
+* Used libraries
+    * leo project
+        * [leo_backend-db v1.1.2](https://github.com/leo-project/leo_backend_db.git)
+        * [leo_cache v0.6.0](https://github.com/leo-project/leo_cache.git)
+        * [leo_commons v1.1.0](https://github.com/leo-project/leo_commons.git)
+        * [leo_dcerl v0.2.11](https://github.com/leo-project/leo_dcerl.git)
+        * [leo_logger v1.1.2](https://github.com/leo-project/leo_logger.git)
+        * [leo_mcerl v0.4.1](https://github.com/leo-project/leo_mcerl.git)
+        * [leo_mq v1.0.10](https://github.com/leo-project/leo_mq.git)
+        * [leo_object_storage v1.1.5](https://github.com/leo-project/leo_object_storage.git)
+        * [leo_ordning_reda v0.10.9](https://github.com/leo-project/leo_ordning_reda.git)
+        * [leo_redundant_manager 1.8.9](https://github.com/leo-project/leo_redundant_manager.git)
+        * [leo_rpc v0.8.10](https://github.com/leo-project/leo_rpc.git)
+        * [leo_pod v0.6.2](https://github.com/leo-project/leo_pod.git)
+        * [leo_s3_libs v1.1.3](https://github.com/leo-project/leo_s3_libs.git)
+        * [leo_statistics v1.0.7](https://github.com/leo-project/leo_statistics.git)
+        * [savanna_agent v0.4.4](https://github.com/leo-project/savanna_agent.git)
+        * [savanna_commons v0.8.6](https://github.com/leo-project/savanna_commons.git)
+        * [erpcgen v0.2.3](https://github.com/leo-project/erpcgen.git)
+        * [nfs_rpc_server v0.2.2](https://github.com/leo-project/nfs_rpc_server.git)
+        * [leo_gateway v1.1.4](https://github.com/leo-project/leo_gateway.git)
+        * [leo_manager v1.1.4](https://github.com/leo-project/leo_manager.git)
+        * [leo_storage v1.1.4](https://github.com/leo-project/leo_storage.git)
+    * others
+        * [bitcask v1.7.0](https://github.com/basho/bitcask.git)
+        * [cowboy v0.8.6](https://github.com/extend/cowboy.git)
+        * [cowlib v0.6.2](https://github.com/extend/cowboy.git)
+        * [eleveldb v1.4.10](https://github.com/basho/eleveldb.git)
+        * [folsom v0.8.1](https://github.com/boundary/folsom.git)
+        * [jiffy v0.8.5](https://github.com/davisp/jiffy.git)
+        * [lz4 v0.2.2](https://github.com/leo-project/erlang-lz4.git) - forked from [szktty/erlang-lz4](https://github.com/szktty/erlng-lz4)
+
+
 1.1.3 (Sep 12, 2014)
 ====================
 

diff --git a/leofs-adm b/leofs-adm
@@ -63,6 +63,9 @@ case $OS in
             NC_ARGS=""
         fi
         ;;
+    SunOS)
+        NC_ARGS=""
+        ;;
     *)
         NC_ARGS="-C"
 esac
@@ -315,6 +318,48 @@ usage_compact_status() {
     esac
 }
 
+usage_diagnose_start() {
+    USAGE="diagnose-start <node>"
+    case "$1" in
+        min)
+            output "$WHITESPACE $USAGE "
+            ;;
+        *)
+            output "Usage:"
+            output "  $SCRIPT $USAGE"
+            output "Description:"
+            output "  - Diagnose data of the storage-node"
+    esac
+}
+
+usage_diagnose_suspend() {
+    USAGE="diagnose-suspend $STORAGE_NODE"
+    case "$1" in
+        min)
+            output "$WHITESPACE $USAGE "
+            ;;
+        *)
+            output "Usage:"
+            output "  $SCRIPT $USAGE"
+            output "Description:"
+            output "  - Suspend the data-diagnosis"
+    esac
+}
+
+usage_diagnose_resume() {
+    USAGE="diagnose-resume $STORAGE_NODE"
+    case "$1" in
+        min)
+            output "$WHITESPACE $USAGE "
+            ;;
+        *)
+            output "Usage:"
+            output "  $SCRIPT $USAGE"
+            output "Description:"
+            output "  - Resume the data-diagnosis"
+    esac
+}
+
 usage_du() {
     USAGE="du [detail] $STORAGE_NODE"
     case "$1" in
@@ -982,6 +1027,49 @@ case "$1" in
             usage_compact_resume
         fi
         ;;
+
+    diagnose-start)
+        if [ $# -eq 2 ]; then
+            case "$2" in
+                --help)
+                    usage_diagnose_start
+                    exit 1
+                    ;;
+                *)
+                    send_command "diagnose-data $2"
+            esac
+        else
+            usage_diagnose_start
+        fi
+        ;;
+    compact-suspend)
+        if [ $# -eq 2 ]; then
+            case "$2" in
+                --help)
+                    usage_diagnose_suspend
+                    exit 1
+                    ;;
+                *)
+                    send_command "compact suspend $2"
+            esac
+        else
+            usage_diagnose_suspend
+        fi
+        ;;
+    diagnose-resume)
+        if [ $# -eq 2 ]; then
+            case "$2" in
+                --help)
+                    usage_diagnose_resume
+                    exit 1
+                    ;;
+                *)
+                    send_command "compact resume " $2
+            esac
+        else
+            usage_diagnose_resume
+        fi
+        ;;
     compact-status)
         if [ $# -eq 2 ]; then
             case "$2" in

diff --git a/leofs.mk b/leofs.mk
@@ -1 +1 @@
-LEOFS_VERSION=1.1.3
+LEOFS_VERSION=1.1.4
diff --git a/make_libs_docs.sh b/make_libs_docs.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+if [ $# -ne 1 ]; then
+  echo "Usage: ./git_checkout.sh <leo_libs_docs dir>"
+  exit 1
+fi
+
+for lib in leo_backend_db \
+           leo_cache \
+           leo_commons \
+           leo_dcerl \
+           leo_logger \
+           leo_mcerl \
+           leo_mq \
+           leo_object_storage \
+           leo_ordning_reda \
+           leo_pod \
+           leo_redundant_manager \
+           leo_rpc \
+           leo_s3_libs \
+           leo_statistics
+do
+    echo $lib
+    cd "deps/$lib" && sh ./make_rst_doc.sh && cp doc/rst/* $1/$lib/
+    cd -
+done
+echo "Finished"
diff --git a/pkg/leo_gateway/files/leo_gateway.conf b/pkg/leo_gateway/files/leo_gateway.conf
@@ -1,5 +1,5 @@
 #======================================================================
-# LeoFS - Gateway Configuration
+# LeoFS - Gateway Configuration v1.1.0
 #
 # See: http://www.leofs.org/docs/configuration.html#leofs-gateway
 #======================================================================
@@ -38,12 +38,17 @@
 ## Name of Manager node(s)
 managers = [manager_0@127.0.0.1, manager_1@127.0.0.1]
 
+
 ## --------------------------------------------------------------------
-## GATEWAY
+## GATEWAY Protocol
 ## --------------------------------------------------------------------
-## Gateway’s HTTP API to use: [s3 | rest | embed | nfs]
+## Gateway Protocol to use: [s3 | rest | embed | nfs]
 protocol = s3
 
+
+## --------------------------------------------------------------------
+## GATEWAY - HTTP-related configurations
+## --------------------------------------------------------------------
 ## Port number the Gateway uses for HTTP connections
 http.port = 8080
 
@@ -68,15 +73,13 @@ http.max_keepalive = 4096
 ## Synchronized time of a bucket property (second)
 bucket_prop_sync_interval = 300
 
+
 ## --------------------------------------------------------------------
 ## GATEWAY - Large Object
 ## --------------------------------------------------------------------
 ## Total number of chunked objects
 large_object.max_chunked_objs = 1000
 
-## Maximum length of an object
-large_object.max_len_of_obj = 524288000
-
 ## Length of a chunked object
 large_object.chunked_obj_len = 5242880
 
@@ -189,22 +192,6 @@ log.member_dir = /var/log/leo_gateway/ring
 ## Output log file(s) - ring
 log.ring_dir = /var/log/leo_gateway/ring
 
-## Is enable Elasticsearch for access-log
-## log.is_enable_esearch = false
-
-## Node of Elasticsearch
-## log.esearch.host = 127.0.0.1
-
-## Elasticsearch listening port
-## log.esearch.port = 9200
-
-## Elasticsearch receive timeout
-## log.esearch.timeout = 5000
-
-## Duration of stack objects
-## log.esearch.esearch_bulk_duration = 3000
-
-
 ## --------------------------------------------------------------------
 ## GATEWAY - Other Directories
 ## --------------------------------------------------------------------
@@ -228,6 +215,22 @@ queue_dir  = /var/db/leo_gateway/queue
 ## qos.managers = [savanna_manager_0@127.0.0.1, savanna_manager_1@127.0.0.1]
 
 
+## --------------------------------------------------------------------
+## Mount
+## --------------------------------------------------------------------
+## Port number the Mount server use
+mount.port = 22050
+## Numbers of processes listening for Mount server connections
+mount.num_of_acceptors = 128
+
+## --------------------------------------------------------------------
+## NFS
+## --------------------------------------------------------------------
+## Port number the NFS server use
+nfs.port = 2049
+## Numbers of processes listening for NFS server connections
+nfs.num_of_acceptors = 128
+
 ## --------------------------------------------------------------------
 ## Other Libs
 ## --------------------------------------------------------------------
@@ -247,19 +250,6 @@ queue_dir  = /var/db/leo_gateway/queue
 ## leo_statistics.profile = false
 
 
-## --------------------------------------------------------------------
-## Mnesia
-##     * Store the info storage-cluster and the info of gateway(s)
-##     * Store the RING and the command histories
-## --------------------------------------------------------------------
-## The write threshold for transaction log dumps
-## as the number of writes to the transaction log
-mnesia.dump_log_write_threshold = 50000
-
-## Controls how often disc_copies tables are dumped from memory
-mnesia.dc_dump_limit = 40
-
-
 #======================================================================
 # For vm.args
 #======================================================================

diff --git a/pkg/leo_manager/Makefile b/pkg/leo_manager/Makefile
@@ -3,16 +3,20 @@ VERSION=$(LEOFS_VERSION)
 COMPONENT_INTERNAL=leo_manager
 COMPONENT=leo_manager
 COMPONENT_PATH=../../package/$(COMPONENT_INTERNAL)_0
-DIR=../bootstrap/$(COMPONENT)
+BASE=../bootstrap/$(COMPONENT)
+DIR=../bootstrap/$(COMPONENT)/$(COMPONENT)
 
 .PHONY: build_info package package_list all bootstrap
 
 all: package
 
 bootstrap:
 	-rm -r $(DIR)
+	mkdir -p $(BASE)
 	cp -r $(COMPONENT_PATH) $(DIR)
-	mkdir $(DIR)/share
+	mkdir -p $(DIR)/share
+	mkdir -p $(BASE)/sbin
+	cp ../../leofs-adm $(BASE)/sbin
 	cp files/$(COMPONENT) $(DIR)/bin/
 	cp files/$(COMPONENT).xml $(DIR)/share
 	cp files/start $(DIR)/erts-*/bin/start
@@ -24,7 +28,7 @@ bootstrap:
 
 package_list: bootstrap
 	cat pre_pkg > packlist
-	(cd $(DIR); find * -type f | sort) >> packlist
+	(cd $(BASE); find * -type f | sort) >> packlist
 	cat post_pkg >> packlist
 
 build_info:
@@ -37,4 +41,4 @@ clean:
 	-rm *.tgz build-info packlist
 
 package: package_list build_info
-	pkg_create -i install.sh -k deinstall.sh -D displayfile -B build-info -c comment -d description -f packlist -I /opt/local/$(COMPONENT) -p $(DIR) -U $(COMPONENT)-$(VERSION)$(SUFFIX).tgz
+	pkg_create -i install.sh -k deinstall.sh -D displayfile -B build-info -c comment -d description -f packlist -I /opt/local/ -p $(BASE) -U $(COMPONENT)-$(VERSION)$(SUFFIX).tgz
diff --git a/pkg/leo_manager/files/leo_manager.conf.master b/pkg/leo_manager/files/leo_manager.conf.master
@@ -1,8 +1,8 @@
-#======================================================================
-# LeoFS - Manager Configuration
-#
-# See: http://www.leofs.org/docs/configuration.html#leofs-manager-master
-# See: http://www.leofs.org/docs/configuration.html#leofs-manager-slave
+##======================================================================
+## LeoFS - Manager Configuration (MASTER)
+##
+## See: http://www.leofs.org/docs/configuration.html#leofs-manager-master
+## See: http://www.leofs.org/docs/configuration.html#leofs-manager-slave
 #======================================================================
 ## --------------------------------------------------------------------
 ## SASL
@@ -53,6 +53,16 @@ console.acceptors.json = 16
 ## # of histories to display at once
 console.histories.num_of_display = 200
 
+## --------------------------------------------------------------------
+## MANAGER - System
+##     * Only set its configurations to **Manager-master**
+## --------------------------------------------------------------------
+## DC Id
+system.dc_id = dc_1
+
+## Cluster Id
+system.cluster_id = leofs_1
+
 ## --------------------------------------------------------------------
 ## MANAGER - Consistency Level
 ##     * Only set its configurations to **Manager-master**
@@ -70,17 +80,17 @@ consistency.read = 1
 ## A number of replicas needed for a successful DELETE operation
 consistency.delete = 1
 
-## A number of replicas in other racks
-## consistency.rack_aware_replicas = 0
+## A number of rack-aware replicas
+consistency.rack_aware_replicas = 0
 
 
 ## --------------------------------------------------------------------
 ## MANAGER - Multi DataCenter Settings
 ## --------------------------------------------------------------------
-## A max number of clusters that the manager can connect
+## A number of replication targets
 ## mdc_replication.max_targets = 2
 
-## A number of replicas in each other DC
+## A number of replicas a DC
 ## mdc_replication.num_of_replicas_a_dc = 1