Work in progress

contrib/inventory-generation/README.md

@@ -1,4 +1,24 @@
 # Inventory generation
 
-This folder contains script to generate data to be used to generate inventories: uuid, hostname, ip, mac, certificates, OS name, agent version 
-q
+This folder contains tooling to generate data for Rudder:
+* raw source data
+* inventories based on these data
+* reports (syslog based) based on policy applied to nodes
+* reports (HTTPS based) signed with raw data, based on policy applied to nodes
+
+
+# Folder Description
+
+* data: data containing the raw data for inventories, on per node uuid. It contains: uuid, hostname, ip, mac, public/private key, certificate, OS name, agent version 
+* inventories: the generated inventories
+* template: template used to generate inventories
+
+# Scripts
+
+inventory-generation: generate the inventories
+
+
+## Tips
+
+On the Rudder server, you'll wan't to increase hard ansd soft limit for nofile in /etc/secutiry/limits.conf, and remove rate limiting in journald in /etc/systemd/journald.conf
+

contrib/inventory-generation/inventory-generation

@@ -16,7 +16,7 @@
 # @man +
 # @man *-p --path*: use the given path to for the data generated
 
-set -x
+set -e
 
 DATA=false
 NUMBER=0
@@ -25,9 +25,22 @@ WIPE=false
 TEMPLATE=tml.ocs
 DEST_PATH=data
 
+DEST_INVENTORIES=inventories
+DEST_INVENTORIES_TMP=inventories_tmp
+
+# Private key passphrase
+PASSPHRASE="Cfengine passphrase"
+
 # detect if any option has been passed to the script
 ANY_OPTION_DEFINED=false
 
+HASH=sha512
+
+#Definining list of inventory (short & long)
+DEF_LONG_OS_NAME=("Microsoft Windows Server 2019 Standard" "Debian GNU/Linux 9.1 (stretch)" "Debian GNU/Linux 9.2 (stretch)" "Debian GNU/Linux 8.6 (Jessie)" "CentOS Linux release 7.6.1810 (Core)" "CentOS Linux release 7.4.1810 (Core)" "CentOS Linux release 6.10 (Final)")
+DEF_SHORT_OS_NAME=(Windows Debian Debian Debian Centos Centos Centos)
+
+MAX_SOFT_VERSION=100
 
 # Output usage
 function usage()
@@ -73,17 +86,22 @@ function generate_data {
   do
     UUID=$(uuidgen)
 
-    mkdir ${DEST_PATH}/${UUID}
+    mkdir ${DEST_PATH}/${UUID} || true
 
-    openssl genrsa -des3 -out ${DEST_PATH}/${UUID}/localhost.priv -passout "pass:Cfengine passphrase" 4096
-    openssl rsa -in ${DEST_PATH}/${UUID}/localhost.priv -passin "pass:Cfengine passphrase" -RSAPublicKey_out -out ${DEST_PATH}/${UUID}/localhost.pub
+    openssl genrsa -des3 -out ${DEST_PATH}/${UUID}/localhost.priv -passout "pass:${PASSPHRASE}" 4096
+    openssl rsa -in ${DEST_PATH}/${UUID}/localhost.priv -passin "pass:${PASSPHRASE}" -RSAPublicKey_out -out ${DEST_PATH}/${UUID}/localhost.pub
+
+
+    openssl req -new -sha256 -key ${DEST_PATH}/${UUID}/localhost.priv -out ${DEST_PATH}/${UUID}/agent.cert -passin "pass:${PASSPHRASE}" -x509 -days 3650 -extensions agent_cert -config openssl-agent.cnf -subj "/UID=${UUID}"
 
     echo "RUDDER_TEST_${i}" > ${DEST_PATH}/${UUID}/hostname
 
 
     IP4=$((IP4+1))
     validate_ip
 
+    echo "${UUID}" > ${DEST_PATH}/${UUID}/uuid
+
     echo "${IP1}.${IP2}.${IP3}.${IP4}" > ${DEST_PATH}/${UUID}/ip
 
     echo "d0:ab:d5:e5:90:e8" > ${DEST_PATH}/${UUID}/mac
@@ -98,7 +116,81 @@ function generate_data {
 
 }
 
+function create_inventory {
+  mkdir ${DEST_INVENTORIES} || true
+  mkdir ${DEST_INVENTORIES_TMP} || true
+
+  echo "Creating inventories based on the folders in ${DEST_PATH}"
+
+  # compute numer of OSs
+  os_list_size=${#DEF_LONG_OS_NAME[@]}
+  echo $os_list_size
+
+  # List all data 
+  for d in ${DEST_PATH}/*; do
+    if [ -d "$d" ]; then
+
+      # File path in the rudder utilities format (for better maintenance)
+      CERT=$d/agent.cert
+      PRIVKEY=$d/localhost.priv
+      PUBKEY=$d/localhost.pub
+
+      # read variables    
+      UUID=$(<$d/uuid)
+      IP=$(<$d/ip)
+      MAC=$(<$d/mac)
+      OS_NAME=$(<$d/osname)
+      FULL_OS_NAME=$(<$d/fullosname)
+      AGENT_VERSION=$(<$d/agent_version)
+      RUDDER_HOSTNAME=$(<$d/hostname)
+
+      PUBLIC_KEY=$(<$d/localhost.pub)
+      AGENT_CERT=$(<$CERT)
+      #Random value
+      RAND_OS_VAL=$((RANDOM%$os_list_size))
+      RAND_OS_NAME=${DEF_SHORT_OS_NAME[$RAND_OS_VAL]}
+      RAND_FULL_OS_NAME=${DEF_LONG_OS_NAME[$RAND_OS_VAL]}     
+
+      RAND_SOFT_VERSION=$(($RANDOM%$MAX_SOFT_VERSION))
+
+      FILENAME=${RUDDER_HOSTNAME}-${UUID}.ocs
+
+      FILE=${DEST_INVENTORIES}/${FILENAME}
+      READY_FILE=${DEST_INVENTORIES}/${FILENAME}.gz
+
+      # replace values in template and create temp file
+      eval "cat <<EOF
+$(<template/RUDDER-Template.ocs) 
+EOF" > ${FILE}
+
+      # sign it
+      SIGNATURE=`openssl dgst -passin "pass:${PASSPHRASE}" -${HASH} -hex -sign "${PRIVKEY}" < "${FILE}" | sed -e 's/.*= //'`
+
+      # Private key modification date
+      KEYDATE=`stat -c %y ${PRIVKEY}`
+
+      # Public key identifier (last 4 bytes of the modulus)
+      KEYID=`openssl rsa -passin "pass:${PASSPHRASE}" -in "${PRIVKEY}" -noout -modulus | sed 's/.*\(........\)$/\1/'`
+
+      cat > "${FILE}.sign" <<EOF
+header=rudder-signature-v1
+algorithm=${HASH}
+digest=${SIGNATURE}
+hostname=${HOSTNAME}
+keydate=${KEYDATE}
+keyid=${KEYID}
+EOF
+
+     # Compress inventories
+     gzip -fq ${FILE}
+    fi
+  done
+
+  # Finally, cleaning all temporary data
+  #rm -rf ${DEST_INVENTORY_TMP}
+  echo "Done!"
 
+}
 function wipe {
   echo "Wipping folder ${DEST_PATH}"
   echo "Type ctrl-c to abort now and return to continue."
@@ -109,9 +201,8 @@ function wipe {
 
 
 # Defines available options
-OPTIONS=d:g:wt:p:
-#LONGOPTS=data:,generate,wipe,template:,DEST_PATH:
-LONGOPTS=data:
+OPTIONS=d:gwt:p:
+LONGOPTS=data:,generate,wipe,template:,path:
 
 # Use ! to avoid failing with set -e
 # Use $PIPESTATUS to get the right return code even with !
@@ -190,3 +281,9 @@ then
   generate_data
 fi
 
+
+# Generate the data
+if [ "${GENERATE}" = "true" ]
+then
+  create_inventory
+fi