Permalink
Browse files

Added storage to service

  • Loading branch information...
1 parent 80c47e5 commit 69062891533bb5c91db669bba551b3c47ba6fd38 Ignacio Perez committed Jan 4, 2012
View
Binary file not shown.
View
@@ -0,0 +1,23 @@
+About Storage
+=============
+
+The Storage component is intended to guarantee the durability of the indexes data. It works as a binary log for your indexes keeping track of all operations. This allows for you to regenerate indexes from scratch (this is what the Nebulizer component does when it 'moves' an index).
+
+### Components
+
+The Storage component comprises 2 servers:
+
+* Indexes. The IndexesServer fulfills different functionalities:
+** - Serves the records to the IndexRecoverer (Reader)
+** - Takes records from the raw files and deals them to the different indexes files (Dealer)
+** - Optimizes log files. Merges all mergable operations (Optimizer)
+
+* Writer
+** The writer gets the call from the API when a new doc is added. It writes down the log storage raw files.
+
+### Running
+
+There are bash scripts in the storage folder for running both components (start_indexes, start_writer). In order for them to work you need to copy the engine jar with dependencies (indextank-engine-1.0.0-jar-with-dependencies.jar) in the lib folder.
+
+
+
View
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# back up log storage
+
+while true; do
+ echo starting sync at `/bin/date`
+ ionice -c 2 -n 5 rsync -av --append-verify --delete /data/storage/live /archive/
+ ionice -c 2 -n 5 rsync -av --append-verify /data/storage/history /archive/
+ echo finished sync at `/bin/date`
+ sleep 2
+done
@@ -0,0 +1,15 @@
+log4j.rootLogger=DEBUG, console
+log4j.logger.ALERTS=DEBUG, alerts
+log4j.logger.org.apache = ERROR
+log4j.logger.httpclient = WARN
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%-5p [%t] %c - [%m] %d{ISO8601}%n
+
+# BEGIN APPENDER: ALERTS LOG
+log4j.appender.alerts=org.apache.log4j.FileAppender
+log4j.appender.alerts.File=/data/logs/alerts.log
+log4j.appender.alerts.layout=org.apache.log4j.PatternLayout
+log4j.appender.alerts.layout.ConversionPattern=%-5p [%t] %c - [%m] %d{ISO8601}%n
+# END APPENDER: ALERTS LOG
View
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+kill `cat indexes.pid`
+
View
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+kill `cat writer.pid`
+
@@ -0,0 +1,6 @@
++ /indexes
++ /indexes/*
++ /indexes/*/optimized/
++ /indexes/*/optimized/*___sorted_*
++ /indexes/*/optimized/*___index
+- *
@@ -0,0 +1,6 @@
++ /indexes
++ /indexes/*
++ /indexes/*/segments/
++ /indexes/*/segments/*___sorted_*
++ /indexes/*/segments/*___index
+- *
View
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Starts the IndexesServer process.
+# The IndexesServer fulfills different functionalities:
+# - Serves the records to the IndexRecoverer (Reader)
+# - Takes records from the raw files and deals them to the different indexes files (Dealer)
+# - Optimizes log files. Merges all mergable operations (Optimizer)
+
+/usr/bin/nohup ionice -c 2 -n 3 /usr/bin/java -cp conf:lib/indextank-engine-1.0.0-jar-with-dependencies.jar com.flaptor.indextank.storage.IndexesLogServer -rp 15100 -mp 16000 2>&1 > /data/logs/indexes.log &
+echo $! > indexes.pid
View
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Starts WriterServer process.
+# The writer gets the call from the API when a new doc is added. It writes down the log storage raw files.
+
+/usr/bin/nohup ionice -c 2 -n 1 /usr/bin/java -cp conf:lib/indextank-engine-1.0.0-jar-with-dependencies.jar com.flaptor.indextank.storage.LogWriterServer -p 15000 2>&1 > /data/logs/st_writer.log &
+echo $! > writer.pid
View
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+# Keeps slave server up to date with merged files.
+
+output_rsync() {
+ lastcode="?"
+ while read data
+ do
+ code=`echo "$data" | sed -r "s;indexes/([^/]*?)/.*$;\1;"`
+ unsorted=`echo "$data" | egrep "___unsorted"`
+ index=`echo "$data" | egrep "___index"`
+ segm=`echo "$data" | egrep "indexes/$code/segments/(.+)$"`
+ opti=`echo "$data" | egrep "indexes/$code/optimized/(.+)$"`
+ count=`echo $data | cut -d "_" -f 7`
+ if [ -n "$code" ]; then
+ if [ -n "$segm$opti" ]; then
+ if [ "$code" != "$lastcode" ]; then
+ echo
+ echo -n "$code"
+ lastcode="$code"
+ fi
+ fi
+ if [ -z "$index" ]; then
+ if [ -n "$segm" ]; then
+ if [ -n "$unsorted" ]; then
+ echo -n " [unsorted]"
+ else
+ echo -n " [$count]"
+ fi
+ fi
+ if [ -n "$opti" ]; then
+ echo -n " <$count>"
+ fi
+ fi
+ else
+ echo "$data" | egrep "^sent"
+ echo "$data" | egrep "^total size"
+ fi
+ done
+}
+
+if [[ ! -f "/data/master" || ! -f "/data/storage/safe_to_read" ]]; then
+
+ if [ -d "/data/storage/migrated" ]; then
+ # the dealer hasn't switched the migrated data. nothing to do yet
+ echo "[ ] The IndexesLogServer hasn't picked up last synchronization yet. Waiting."
+ exit 0
+ else
+ MASTER="ec2-174-129-33-117.compute-1.amazonaws.com"
+
+ echo "[+] Preparing directory..."
+ rm -fr /data/storage/migrating
+ mkdir -p /data/storage/migrating
+
+ echo "[+] Synchronizing dealer info to /data/storage/migrating..."
+ rsync -av $MASTER:/data/storage/dealer*.info /data/storage/migrating > /dev/null
+ if [[ $? != 0 ]]; then echo ">>>>>>>>>>>>>>>>>> FAILED to rsync dealer info :("; exit 1; fi
+
+ CURRENT=`cat /data/storage/dealer.next_timestamp.info`
+ NEW=`cat /data/storage/migrating/dealer.next_timestamp.info`
+
+ if [ "$NEW" != "$CURRENT" ]; then
+ #echo "[+] Locally hard-linking unsorted segments to /data/storage/migrating..."
+ #rsync -ahv --link-dest /data/storage/ --include-from unsorted_files_pattern /data/storage/indexes /data/storage/migrating
+ #if [[ $? != 0 ]]; then echo "Failed to ??? :("; exit 1; fi
+
+ #echo "[+] Synchronizing unsorted segments to /data/storage/migrating..."
+ #rsync -ahbv --suffix "" --backup-dir "." --append-verify --include-from unsorted_files_pattern $MASTER:/data/storage/indexes /data/storage/migrating
+ #if [[ $? != 0 ]]; then echo "Failed to ??? :("; exit 1; fi
+
+ echo "[+] Synchronizing unsorted segments to /data/storage/migrating..."
+ rsync -ahv --link-dest /data/storage/ --include-from unsorted_files_pattern $MASTER:/data/storage/indexes /data/storage/migrating
+ if [[ $? != 0 ]]; then echo ">>>>>>>>>>>>>>>>>> FAILED to synchronize unsorted segments :("; exit 1; fi
+
+ echo "[+] Synchronizing sorted segments to /data/storage/migrating..."
+ rsync -ahv --link-dest /data/storage/ --include-from sorted_files_pattern $MASTER:/data/storage/indexes /data/storage/migrating
+ if [[ $? != 0 ]]; then echo ">>>>>>>>>>>>>>>>>> FAILED to synchronize sorted segments :("; exit 1; fi
+
+ echo "[+] Synchronizing optimized segments to /data/storage/migrating..."
+ rsync -ahv --link-dest /data/storage/ --include-from optimized_files_pattern $MASTER:/data/storage/indexes /data/storage/migrating
+ if [[ $? != 0 ]]; then echo ">>>>>>>>>>>>>>>>>> FAILED to synchronize optimized segments :("; exit 1; fi
+
+ echo "[+] Removing duplicates between sorted and unsorted segments..."
+ find /data/storage/migrating/indexes -name *_sorted_* | cut -d_ -f1 | awk '{printf "%s_*_unsorted_*\n",$1}' | xargs -L 100 rm -f
+
+ echo "[+] Fetching time delta with master..."
+ MASTER_DATE=`ssh $MASTER date "+%s"`
+ LOCAL_DATE=`date "+%s"`
+ echo $(( LOCAL_DATE - MASTER_DATE )) > /data/storage/migrating/master_delta.info 2> /dev/null
+
+ echo "[+] Moving complete snapshot to /data/storage/migrated"
+ mv /data/storage/migrating /data/storage/migrated
+ fi
+ fi
+
+ echo "Synchronization complete"
+
+fi
+
+
+#echo "[+] Synchronizing segments to /data/migrating..."
+#rsync -ahv --link-dest /data/storage/ $MASTER:/data/storage/indexes /data/migrating
+#echo
+#if [[ $? != 0 ]]; then echo "Failed to create /data/previous :("; exit 1; fi
+#if [[ ${PIPESTATUS[0]} != 0 ]]; then echo "Failed to rsync segments :("; exit 1; fi
+#echo
+
+#if [ -d "/data/previous" ]; then
+# echo "Removing /data/previous..."
+# rm -fr /data/previous
+#fi
+#mkdir -p /data/previous/
+#if [[ $? != 0 ]]; then echo "Failed to create /data/previous :("; exit 1; fi
+#
+#echo "Moving /data/storage/indexes to /data/previous..."
+#mv /data/storage/indexes /data/previous/
+
+#echo "Copying dealer info files from /data/storage to /data/previous..."
+#cp /data/storage/dealer*.info /data/previous/
+#
+#echo "Moving migrated data to /data/storage..."
+#mv /data/migrating/* /data/storage/
+#if [[ $? != 0 ]]; then echo "Failed to move migrated segments :("; exit 1; fi
+
+
+
@@ -0,0 +1,5 @@
++ /indexes
++ /indexes/*
++ /indexes/*/segments/
++ /indexes/*/segments/*___unsorted_*
+- *

0 comments on commit 6906289

Please sign in to comment.