Run pg_receivexlog until postgres will start streaming from master (#154

)

On heavily loaded clusters wal-e can't keep up with the rate of generated wal segments.

postgres-appliance/basebackup.sh

@@ -27,6 +27,19 @@ done
 
 [[ -z $DATA_DIR || -z $CONNSTR || ! $RETRIES =~ ^[1-9]$ ]] && exit 1
 
+function receivexlog() {
+    pg_receivexlog --no-loop --directory="${XLOG_FAST}" --dbname="${CONNSTR}" &
+    receivexlog_pid=$!
+
+    # run pg_receivexlog until postgres will not start streaming
+    while ! ps ax | grep -qE '[w]al receiver process\s+streaming'; do
+        # exit if pg_receivexlog is not running
+        kill -0 $receivexlog_pid && sleep 1 || exit
+    done
+
+    kill $receivexlog_pid
+}
+
 ATTEMPT=0
 while [[ $((ATTEMPT++)) -le $RETRIES ]]; do
     rm -fr "${DATA_DIR}"
@@ -38,6 +51,8 @@ while [[ $((ATTEMPT++)) -le $RETRIES ]]; do
         mv ${DATA_DIR}/pg_xlog $XLOG_FAST
         rm -fr $XLOG_FAST/archive_status
         mkdir ${DATA_DIR}/pg_xlog
+
+        receivexlog &
         break
     elif [[ $ATTEMPT -le $RETRIES ]]; then
         sleep $((ATTEMPT*10))

postgres-appliance/configure_spilo.py

@@ -188,15 +188,10 @@ def deep_update(a, b):
     on_restart: {{CALLBACK_SCRIPT}}
     on_role_change: {{CALLBACK_SCRIPT}}
  {{/CALLBACK_SCRIPT}}
+{{#USE_WALE}}
   create_replica_method:
-    {{#USE_WALE}}
     - wal_e
     - basebackup_fast_xlog
-    {{/USE_WALE}}
-    {{^USE_WALE}}
-    - basebackup
-    {{/USE_WALE}}
- {{#USE_WALE}}
   wal_e:
     command: patroni_wale_restore
     envdir: {{WALE_ENV_DIR}}