Merged script-dev changes

Also updated dependencies link to Loggly

Linux Script/configure-linux.sh

@@ -15,7 +15,7 @@ function ctrl_c()  {
 #name of the current script. This will get overwritten by the child script which calls this
 SCRIPT_NAME=configure-linux.sh
 #version of the current script. This will get overwritten by the child script which calls this
-SCRIPT_VERSION=1.5
+SCRIPT_VERSION=1.7
 
 #application tag. This will get overwritten by the child script which calls this
 APP_TAG=
@@ -145,10 +145,12 @@ installLogglyConf()
 	if [ "$LINUX_DO_VERIFICATION" = "true" ]; then
 		#check if the logs are going to loggly fro linux system now
 		checkIfLogsMadeToLoggly
+	fi
 
-		#log success message
+	if [ "$IS_INVOKED" = "" ]; then
 		logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly."
 	fi
+
 }
 
 #remove loggly configuration from Linux system
@@ -269,15 +271,15 @@ checkIfLogglyServersAccessible()
 	if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then
 		echo "INFO: $LOGS_01_HOST is reachable."
 	else
-		logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system."
+		logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings."
 		exit 1
 	fi
 
 	echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time."
 	if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then
 		echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port."
 	else
-		logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system."
+		logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings."
 		exit 1
 	fi
 
@@ -401,22 +403,8 @@ write22LogglyConfFile()
 {
 	echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist."
 	if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then
-		logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist."
-		while true; do
-			read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn
-			case $yn in
-				[Yy]* )
-				logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP";
-				sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP;
-				checkAuthTokenAndWriteContents;
-				break;;
-				[Nn]* )
-				LINUX_DO_VERIFICATION="false"
-				logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification."
-				break;;
-				* ) echo "Please answer yes or no.";;
-			esac
-		done
+		logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist."
+		checkIfConfigurationChanged
 	else
 		logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE"
 		checkAuthTokenAndWriteContents
@@ -435,6 +423,48 @@ checkAuthTokenAndWriteContents()
 	fi
 }
 
+#matches if the content of 22-loggly.conf content is changed
+checkIfConfigurationChanged()
+{
+	ASK_FOR_VERIFICATION="false"
+
+	#strings to be checked which should be present in the existing 22-loggly.conf. 
+	#If these strings are not same then a warning message will be shown to user to update the 22-loggly.conf file
+	STR_TO_BE_CHECKED[0]="\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID] %msg%\""
+	STR_TO_BE_CHECKED[1]="*.*             @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat"
+
+	for i in "${STR_TO_BE_CHECKED[@]}"
+	do
+		if ! sudo grep -Fxq "$i" $LOGGLY_RSYSLOG_CONFFILE; then
+			ASK_FOR_VERIFICATION="true"
+			break;
+		fi
+	done
+
+	if [ "$ASK_FOR_VERIFICATION" == "true" ]; then
+		logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE content is incorrect."
+		while true; 
+		do
+			read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn
+			case $yn in
+				[Yy]* )
+				logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP";
+				sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP;
+				checkAuthTokenAndWriteContents;
+				break;;
+				[Nn]* )
+				LINUX_DO_VERIFICATION="false"
+				logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification."
+				break;;
+				* ) echo "Please answer yes or no.";;
+			esac
+		done
+	else
+		LINUX_DO_VERIFICATION="false"
+	fi
+
+}
+
 #write the contents to 22-loggly.conf file
 writeContents()
 {
@@ -506,7 +536,7 @@ checkIfLogsMadeToLoggly()
 		let counter=$counter+1
 		if [ "$counter" -gt "$maxCounter" ]; then
 			MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/"
-			logMsgToConfigSysLog "ERROR" "ERROR: Verification logs did not make it to Loggly in time. Please check your token & network/firewall settings and retry."
+			logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry."
 			exit 1
 		fi
 	done
@@ -601,9 +631,9 @@ logMsgToConfigSysLog()
 sendPayloadToConfigSysLog()
 {
 	if [ "$APP_TAG" = "" ]; then
-		var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}"
+		var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\",\"rsyslog-version\":\"$RSYSLOG_VERSION\"}"
 	else
-		var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}"
+		var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\",\"rsyslog-version\":\"$RSYSLOG_VERSION\"}"
 	fi
 	curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1
 }

Modular Scripts/Apache2/configure-apache.sh

@@ -297,7 +297,7 @@ checkIfApacheLogsMadeToLoggly()
 		searchAndFetch apacheLatestLogCount "$queryUrl"
 		let counter=$counter+1
 		if [ "$counter" -gt "$maxCounter" ]; then
-			logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check your token & network/firewall settings and retry."
+			logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry."
 			exit 1
 		fi
 	done

Modular Scripts/File Monitoring/configure-file-monitoring.sh

@@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked"
 #name of the current script
 SCRIPT_NAME=configure-file-monitoring.sh
 #version of the current script
-SCRIPT_VERSION=1.3
+SCRIPT_VERSION=1.4
 
 #file to monitor (contains complete path and file name) provided by user
 LOGGLY_FILE_TO_MONITOR=
@@ -57,6 +57,9 @@ installLogglyConfForFile()
 	#check if file to monitor exists
 	checkIfFileExist
 
+	#checks if the file has proper read permission
+	checkFileReadPermission
+
 	#check if the alias is already taken
 	checkIfFileAliasExist
 
@@ -189,6 +192,18 @@ checkLogFileSize()
 	fi
 }
 
+
+#checks the input file has proper read permissions 
+checkFileReadPermission()
+{
+	FILE_PERMISSIONS=$(ls -l $LOGGLY_FILE_TO_MONITOR)
+	#checking if the file has read permission for others
+	PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1}
+	if [ $PERMISSION_READ_OTHERS != r ]; then 
+		logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail."
+	fi
+}
+
 #function to write the contents of syslog config file
 write21ConfFileContents()
 {
@@ -259,7 +274,7 @@ checkIfFileLogsMadeToLoggly()
 		searchAndFetch fileLatestLogCount "$queryUrl"
 		let counter=$counter+1
 		if [ "$counter" -gt "$maxCounter" ]; then
-			logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry."
+			logMsgToConfigSysLog "ERROR" "ERROR: File logs did not make to Loggly in time. Please check network and firewall settings and retry."
 			exit 1
 		fi
 	done

Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh

@@ -8,7 +8,7 @@ source configure-file-monitoring.sh "being-invoked"
 #name of the current script
 SCRIPT_NAME=configure-s3-file-monitoring.sh
 #version of the current script
-SCRIPT_VERSION=1.1
+SCRIPT_VERSION=1.3
 
 #s3 bucket name to configure
 LOGGLY_S3_BUCKET_NAME=
@@ -67,15 +67,16 @@ installLogglyConfForS3()
 	#invoke file monitoring on each file after checking if it is a text file or not
 	invokeS3FileMonitoring
 
+	#install a cron job to sync the downloaded files after each 5 minutes 
+	installCronToSyncS3BucketPeriodically
+
 	if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then
 		#check if s3 logs made it to loggly
 		checkIfS3LogsMadeToLoggly
 	else
 		logMsgToConfigSysLog "WARN" "WARN: Did not find any files to configure. Nothing to do."
 	fi
-
-	#delete temporary directory
-	#deleteTempDir
+
 }
 
 
@@ -95,9 +96,12 @@ removeLogglyConfForS3()
 
 	#remove file monitoring
 	removeS3FileMonitoring
+
+	#delete the s3 sync cron from the crontab
+	deleteS3CronFromCrontab
 
 	#delete temporary directory if exists
-	TEMP_DIR=/tmp/$LOGGLY_S3_ALIAS
+	TEMP_DIR=/tmp/s3monitoring/$LOGGLY_S3_ALIAS
 	deleteTempDir
 
 	#log success message
@@ -165,8 +169,8 @@ downloadS3cmd()
 #configure s3cmd
 configureS3cmd()
 {
-	s3cmd --configure
-	IS_S3CMD_CONFIGURED_BY_SCRIPT="true"
+	sudo s3cmd --configure
+	IS_S3CMD_CONFIGURED_BY_SCRIPT="true"	
 	#check if s3cmd configured successfully now
 	checkIfS3cmdConfigured
 }
@@ -203,7 +207,7 @@ checkIfValidS3Bucket()
 
 createTempDir()
 {
-	TEMP_DIR=/tmp/$LOGGLY_S3_ALIAS
+	TEMP_DIR=/tmp/s3monitoring/$LOGGLY_S3_ALIAS
 	if [ -d "$TEMP_DIR" ]; then
 		if [ "$(ls -A $TEMP_DIR)" ]; then
 			logMsgToConfigSysLog "WARN" "WARN: There are some files/folders already present in $TEMP_DIR. If you continue, the files currently inside the $TEMP_DIR will also be configured to send logs to loggly."
@@ -221,7 +225,12 @@ createTempDir()
 			done
 		fi		
 	else
-		mkdir /tmp/$LOGGLY_S3_ALIAS
+		if [ -d "/tmp/s3monitoring" ]; then
+			mkdir /tmp/s3monitoring/$LOGGLY_S3_ALIAS
+		else
+			mkdir /tmp/s3monitoring
+			mkdir /tmp/s3monitoring/$LOGGLY_S3_ALIAS
+		fi
 	fi
 }
 
@@ -231,7 +240,7 @@ downloadS3Bucket()
 		#Files are downloaded in nested directory
 		cd $TEMP_DIR
 		echo "Downloading files, may take some time..."
-		s3cmd get -r -f $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1
+		sudo s3cmd get -r -f $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1
 		if [ $? -ne 0 ]; then
 			logMsgToConfigSysLog "ERROR" "ERROR: Error downloading files recursively from $LOGGLY_S3_BUCKET_NAME"
 			exit 1
@@ -242,7 +251,7 @@ downloadS3Bucket()
 
 invokeS3FileMonitoring()
 {
-	dir=/tmp/$LOGGLY_S3_ALIAS
+	dir=/tmp/s3monitoring/$LOGGLY_S3_ALIAS
 	#TODO: Not supporting multiple files with same name in different directories
 	#only supporting file with naming convention *.*
 	for f in $(find $dir -name '*')
@@ -260,6 +269,7 @@ invokeS3FileMonitoring()
 			LOGGLY_FILE_TAG="s3file"
 			CONF_FILE_FORMAT_NAME="LogglyFormatS3"
 			constructFileVariables
+			checkFileReadPermission
 			checkLogFileSize $LOGGLY_FILE_TO_MONITOR
 			write21ConfFileContents
 			IS_ANY_FILE_CONFIGURED="true"
@@ -274,6 +284,68 @@ invokeS3FileMonitoring()
 	fi
 }
 
+installCronToSyncS3BucketPeriodically()
+{
+	while true; do
+		read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn
+		case $yn in
+			[Yy]* )
+
+				#copying .s3cfg file to /root so that it can be used by crontab for sync
+				if ! sudo test -f "/root/.s3cfg" ; then
+					sudo cp $HOME/.s3cfg /root
+				fi
+
+				CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS"
+				CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS"
+
+				#checking if the provided s3 path if of directory or file
+				IS_DIR="true"
+				BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/})
+
+				if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then
+					for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE)
+					do
+						if [ -f $fle ]; then
+							IS_DIR="false"
+							break
+						fi
+					done
+				fi
+
+				#adding file name to the sync folder as the bucket path is
+				#provided upto a file
+				if [ "$IS_DIR" == "false" ]; then
+					CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE"
+				fi
+
+				logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes."
+
+				#setting up cron job
+				CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH"
+
+				EXISTING_CRONS=$(sudo crontab -l 2>&1)
+				case $EXISTING_CRONS in
+					no*)
+						;;
+					*)
+						echo "$EXISTING_CRONS" >> $CRON_FILE
+						;;
+				esac
+
+				echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >> $CRON_FILE
+				sudo crontab $CRON_FILE
+				sudo rm -fr $CRON_FILE
+				break;;
+			[Nn]* ) 
+				logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation."
+				break;;
+			* ) echo "Please answer yes or no.";;
+		esac
+	done
+
+}
+
 deleteTempDir()
 {	
 	if [ -d "$TEMP_DIR" ]; then
@@ -307,7 +379,7 @@ checkIfS3LogsMadeToLoggly()
 		searchAndFetch fileLatestLogCount "$queryUrl"
 		let counter=$counter+1
 		if [ "$counter" -gt "$maxCounter" ]; then
-			logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry."
+			logMsgToConfigSysLog "ERROR" "ERROR: S3 logs did not make to Loggly in time. Please check network and firewall settings and retry."
 			exit 1
 		fi
 	done
@@ -347,6 +419,12 @@ removeS3FileMonitoring()
 	restartRsyslog
 }
 
+deleteS3CronFromCrontab()
+{
+	logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron."
+	sudo crontab -l | grep -v  "/tmp/s3monitoring/$LOGGLY_S3_ALIAS" | crontab -
+}
+
 #display usage syntax
 usage()
 {

Modular Scripts/Tomcat/configure-tomcat.sh

@@ -568,7 +568,7 @@ checkIfTomcatLogsMadeToLoggly()
 		searchAndFetch tomcatLatestLogCount "$queryUrl"
 		let counter=$counter+1
 		if [ "$counter" -gt "$maxCounter" ]; then
-			logMsgToConfigSysLog "ERROR" "ERROR: Tomcat logs did not make to Loggly in time. Please check your token & network/firewall settings and retry."
+			logMsgToConfigSysLog "ERROR" "ERROR: Tomcat logs did not make to Loggly in time. Please check network and firewall settings and retry."
 			exit 1
 		fi
 	done