Version new

.github/workflows/ci.yml

@@ -26,6 +26,7 @@ jobs:
         ports:
           - 5432:5432
         options: >-
+          --name pgtools-postgres-ci
           --health-cmd "pg_isready -U postgres"
           --health-interval 10s
           --health-timeout 5s
@@ -50,15 +51,45 @@ jobs:
           echo "PostgreSQL did not become ready in time" >&2
           exit 1
 
-      - name: ShellCheck automation scripts
-        run: shellcheck automation/*.sh
+      - name: Enable pg_stat_statements
+        run: |
+          psql -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"
+          psql -c "SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements';"
+
+      - name: Configure preload and restart PostgreSQL
+        run: |
+          psql -c "ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements';"
+          docker restart pgtools-postgres-ci
+
+          for i in {1..30}; do
+            if pg_isready -h "$PGHOST" -p "$PGPORT" -U "$PGUSER"; then
+              break
+            fi
+            sleep 2
+          done
+
+          if ! pg_isready -h "$PGHOST" -p "$PGPORT" -U "$PGUSER"; then
+            echo "PostgreSQL did not become ready after restart" >&2
+            exit 1
+          fi
+
+          psql -tA -c "SHOW shared_preload_libraries;" | grep -q "pg_stat_statements"
+          psql -c "SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements';"
+
+      - name: ShellCheck all shell scripts
+        run: |
+          shellcheck \
+            automation/*.sh \
+            integration/*.sh \
+            configuration/*.sh \
+            maintenance/*.sh \
+            scripts/*.sh
 
-      # Todo: enable full test suite when stable
-      # - name: Fast automation test suite
-      #   run: ./automation/test_pgtools.sh --fast
+      - name: Fast automation test suite
+        run: ./automation/test_pgtools.sh --fast
 
-      # - name: HOT checklist JSON validation
-      #   run: ./automation/run_hot_update_report.sh --format json --database "$PGDATABASE" --stdout
+      - name: HOT checklist JSON validation
+        run: ./automation/run_hot_update_report.sh --format json --database "$PGDATABASE" --stdout
 
-      # - name: HOT checklist text validation
-      #   run: ./automation/run_hot_update_report.sh --format text --database "$PGDATABASE" --stdout
+      - name: HOT checklist text validation
+        run: ./automation/run_hot_update_report.sh --format text --database "$PGDATABASE" --stdout

administration/partition_management.sql

@@ -138,7 +138,26 @@ FROM parent_stats ps
 ORDER BY ps.total_size DESC;
 
 -- Individual partition details with performance metrics
-SELECT 
+WITH partition_sizes AS (
+    SELECT
+        pi.inhparent,
+        pn_parent.nspname||'.'||pc_parent.relname AS parent_table,
+        pn_child.nspname||'.'||pc_child.relname AS partition_name,
+        pg_total_relation_size(pc_child.oid) AS partition_size,
+        pg_relation_size(pc_child.oid) AS table_size,
+        COALESCE(st.n_live_tup, 0) AS row_count,
+        pg_get_expr(pc_child.relpartbound, pc_child.oid) AS partition_bounds
+    FROM pg_inherits pi
+    JOIN pg_class pc_parent ON pi.inhparent = pc_parent.oid
+    JOIN pg_namespace pn_parent ON pc_parent.relnamespace = pn_parent.oid
+    JOIN pg_class pc_child ON pi.inhrelid = pc_child.oid
+    JOIN pg_namespace pn_child ON pc_child.relnamespace = pn_child.oid
+    LEFT JOIN pg_stat_user_tables st ON st.schemaname = pn_child.nspname AND st.relname = pc_child.relname
+    WHERE EXISTS (
+        SELECT 1 FROM pg_partitioned_table ppt WHERE ppt.partrelid = pc_parent.oid
+    )
+)
+SELECT
     ps.parent_table,
     ps.partition_name,
     ps.partition_bounds,
@@ -158,8 +177,8 @@ SELECT
         THEN 'SMALL: Consider consolidating with adjacent partitions'
         ELSE 'OK: Normal partition usage'
     END AS partition_health,
-    COALESCE(st.last_vacuum, 'Never'::timestamp) AS last_vacuum,
-    COALESCE(st.last_analyze, 'Never'::timestamp) AS last_analyze,
+    st.last_vacuum,
+    st.last_analyze,
     CASE 
         WHEN st.last_vacuum < now() - interval '7 days' AND ps.row_count > 1000 
         THEN 'NEEDS VACUUM: No recent vacuum activity'
@@ -213,8 +232,27 @@ FROM partition_queries
 GROUP BY constraint_exclusion_setting;
 
 -- Partition maintenance recommendations and automation
-WITH maintenance_analysis AS (
-    SELECT 
+WITH partition_sizes AS (
+    SELECT
+        pi.inhparent,
+        pn_parent.nspname||'.'||pc_parent.relname AS parent_table,
+        pn_child.nspname||'.'||pc_child.relname AS partition_name,
+        pg_total_relation_size(pc_child.oid) AS partition_size,
+        pg_relation_size(pc_child.oid) AS table_size,
+        COALESCE(st.n_live_tup, 0) AS row_count,
+        pg_get_expr(pc_child.relpartbound, pc_child.oid) AS partition_bounds
+    FROM pg_inherits pi
+    JOIN pg_class pc_parent ON pi.inhparent = pc_parent.oid
+    JOIN pg_namespace pn_parent ON pc_parent.relnamespace = pn_parent.oid
+    JOIN pg_class pc_child ON pi.inhrelid = pc_child.oid
+    JOIN pg_namespace pn_child ON pc_child.relnamespace = pn_child.oid
+    LEFT JOIN pg_stat_user_tables st ON st.schemaname = pn_child.nspname AND st.relname = pc_child.relname
+    WHERE EXISTS (
+        SELECT 1 FROM pg_partitioned_table ppt WHERE ppt.partrelid = pc_parent.oid
+    )
+),
+maintenance_analysis AS (
+    SELECT
         ps.parent_table,
         COUNT(*) AS partition_count,
         COUNT(*) FILTER (WHERE ps.row_count = 0) AS empty_partitions,

automation/cleanup_reports.sh

@@ -180,7 +180,7 @@ clean_directory() {
     local total_size=0
 
     while IFS= read -r -d '' file; do
-        ((file_count++))
+        file_count=$((file_count + 1))
         if command -v stat > /dev/null 2>&1; then
             local size
             size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null || echo "0")
@@ -225,7 +225,7 @@ clean_temp_files() {
         if [[ -e "$file" ]] && [[ -f "$file" ]]; then
             # Check if file is older than 1 day
             if [[ $(find "$file" -mtime +1 2>/dev/null) ]]; then
-                ((file_count++))
+                file_count=$((file_count + 1))
                 if [[ "$DRY_RUN" == "true" ]]; then
                     echo "Would delete temp file: $file"
                 else

automation/pgtools_health_check.sh

@@ -274,13 +274,13 @@ run_health_checks() {
 
         if [[ -f "$script_path" ]]; then
             if run_script "$script_path" "$script_name" "$output_file"; then
-                ((completed_scripts++))
+                completed_scripts=$((completed_scripts + 1))
             else
-                ((failed_scripts++))
+                failed_scripts=$((failed_scripts + 1))
             fi
         else
             error "Script not found: $script_path"
-            ((failed_scripts++))
+            failed_scripts=$((failed_scripts + 1))
         fi
     done
 

automation/test_pgtools.sh

@@ -111,6 +111,43 @@ if [[ -f "$CONFIG_FILE" ]]; then
     source "$CONFIG_FILE"
 fi
 
+# Environment-driven skip guards (populated in main() before syntax tests run).
+# Scripts that require superuser or specific extensions are skipped when the
+# CI/test database cannot support them, rather than reporting a false failure.
+SKIP_SUPERUSER_SCRIPTS="false"
+SKIP_EXTENSION_SCRIPTS="false"
+HAS_PG_STAT_STATEMENTS="false"
+HAS_PG_BUFFERCACHE="false"
+
+detect_environment_capabilities() {
+    # Superuser gate — permission_audit.sql and switch_pg_wal_file.sql need it.
+    local is_super
+    if is_super="$(psql -tA -c "SELECT current_setting('is_superuser', true);" 2>/dev/null)"; then
+        if [[ "$(echo "$is_super" | tr -d '[:space:]')" != "on" ]]; then
+            SKIP_SUPERUSER_SCRIPTS="true"
+        fi
+    else
+        SKIP_SUPERUSER_SCRIPTS="true"
+    fi
+
+    # Detect extension availability once so per-file gating is deterministic.
+    if psql -tA -c "SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements' LIMIT 1;" 2>/dev/null | grep -q 1; then
+        HAS_PG_STAT_STATEMENTS="true"
+    fi
+
+    if psql -tA -c "SELECT 1 FROM pg_extension WHERE extname = 'pg_buffercache' LIMIT 1;" 2>/dev/null | grep -q 1; then
+        HAS_PG_BUFFERCACHE="true"
+    fi
+
+    if [[ "$HAS_PG_STAT_STATEMENTS" == "false" && "$HAS_PG_BUFFERCACHE" == "false" ]]; then
+        SKIP_EXTENSION_SCRIPTS="true"
+    fi
+
+    if [[ "$VERBOSE" == "true" ]]; then
+        log "Environment detection: SKIP_SUPERUSER_SCRIPTS=$SKIP_SUPERUSER_SCRIPTS SKIP_EXTENSION_SCRIPTS=$SKIP_EXTENSION_SCRIPTS HAS_PG_STAT_STATEMENTS=$HAS_PG_STAT_STATEMENTS HAS_PG_BUFFERCACHE=$HAS_PG_BUFFERCACHE"
+    fi
+}
+
 # Test execution framework
 run_test() {
     local test_name="$1"
@@ -120,17 +157,17 @@ run_test() {
         return 0
     fi
 
-    ((TESTS_RUN++))
-    
+    TESTS_RUN=$((TESTS_RUN + 1))
+
     if [[ "$VERBOSE" == "true" ]]; then
         log "Running test: $test_name"
     fi
-    
+
     if $test_function; then
-        ((TESTS_PASSED++))
+        TESTS_PASSED=$((TESTS_PASSED + 1))
         success "✓ $test_name"
     else
-        ((TESTS_FAILED++))
+        TESTS_FAILED=$((TESTS_FAILED + 1))
         error "✗ $test_name"
     fi
 }
@@ -159,30 +196,112 @@ test_extensions_available() {
     return 0  # Don't fail if extensions missing
 }
 
-# Syntax validation tests
+# Directories that contribute .sql files to the syntax test corpus. Kept as an
+# allowlist rather than a recursive find so we can intentionally exclude folders
+# (e.g. timescaledb/, which only runs under a TimescaleDB-enabled cluster).
+SQL_TEST_DIRS=(
+    "$PGTOOLS_ROOT/administration"
+    "$PGTOOLS_ROOT/backup"
+    "$PGTOOLS_ROOT/configuration"
+    "$PGTOOLS_ROOT/maintenance"
+    "$PGTOOLS_ROOT/monitoring"
+    "$PGTOOLS_ROOT/optimization"
+    "$PGTOOLS_ROOT/performance"
+    "$PGTOOLS_ROOT/security"
+    "$PGTOOLS_ROOT/troubleshooting"
+)
+
+# Files that require a superuser connection. Skipped when SKIP_SUPERUSER_SCRIPTS
+# is true so limited-role CI runs don't report false failures.
+SQL_REQUIRES_SUPERUSER=(
+    "security/permission_audit.sql"
+    "maintenance/switch_pg_wal_file.sql"
+)
+
+# Files that require pg_stat_statements.
+SQL_REQUIRES_PG_STAT_STATEMENTS=(
+    "optimization/missing_indexes.sql"
+    "performance/query_performance_profiler.sql"
+    "troubleshooting/postgres_troubleshooting_queries.sql"
+    "troubleshooting/postgres_troubleshooting_query_pack_01.sql"
+    "troubleshooting/postgres_troubleshooting_query_pack_02.sql"
+    "troubleshooting/postgres_troubleshooting_query_pack_03.sql"
+)
+
+# Files that require pg_buffercache.
+SQL_REQUIRES_PG_BUFFERCACHE=(
+    "__none__"
+)
+
+# TimescaleDB-only files. Always skipped by the syntax test; covered separately
+# when the test runs against a TimescaleDB-enabled instance.
+SQL_REQUIRES_TIMESCALEDB=(
+    "administration/NonHypertables.sql"
+)
+
+_sql_list_contains() {
+    # $1 = relative path (e.g. "security/permission_audit.sql")
+    # $2..N = entries to check against
+    local needle="$1"; shift
+    local entry
+    for entry in "$@"; do
+        if [[ "$entry" == "$needle" ]]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Syntax validation tests — iterates every .sql file in SQL_TEST_DIRS and runs
+# it with ON_ERROR_STOP=1 so any parse/bind error becomes a non-zero exit.
 test_sql_syntax() {
-    local sql_files=(
-        "$PGTOOLS_ROOT/backup/backup_validation.sql"
-        "$PGTOOLS_ROOT/security/permission_audit.sql"
-        "$PGTOOLS_ROOT/monitoring/connection_pools.sql"
-        "$PGTOOLS_ROOT/optimization/missing_indexes.sql"
-        "$PGTOOLS_ROOT/administration/partition_management.sql"
-    )
-
-    for sql_file in "${sql_files[@]}"; do
-        if [[ -f "$sql_file" ]]; then
-            if ! psql -f "$sql_file" --dry-run > /dev/null 2>&1; then
-                # Dry run not supported, try syntax check
-                if ! psql -c "\\i $sql_file" > /dev/null 2>&1; then
-                    error "SQL syntax error in: $sql_file"
-                    return 1
-                fi
-            fi
-        else
-            warn "SQL file not found: $sql_file"
+    local failed=0
+    local dir
+    local sql_file
+    local rel_path
+
+    for dir in "${SQL_TEST_DIRS[@]}"; do
+        if [[ ! -d "$dir" ]]; then
+            warn "SQL directory not found: $dir"
+            continue
         fi
+
+        while IFS= read -r -d '' sql_file; do
+            rel_path="${sql_file#"$PGTOOLS_ROOT"/}"
+
+            if _sql_list_contains "$rel_path" "${SQL_REQUIRES_TIMESCALEDB[@]}"; then
+                [[ "$VERBOSE" == "true" ]] && log "Skipping TimescaleDB-only script: $rel_path"
+                continue
+            fi
+
+            if [[ "$SKIP_SUPERUSER_SCRIPTS" == "true" ]] \
+                && _sql_list_contains "$rel_path" "${SQL_REQUIRES_SUPERUSER[@]}"; then
+                [[ "$VERBOSE" == "true" ]] && log "Skipping superuser-only script (not superuser): $rel_path"
+                continue
+            fi
+
+            if [[ "$HAS_PG_STAT_STATEMENTS" == "false" ]] \
+                && _sql_list_contains "$rel_path" "${SQL_REQUIRES_PG_STAT_STATEMENTS[@]}"; then
+                [[ "$VERBOSE" == "true" ]] && log "Skipping pg_stat_statements-dependent script: $rel_path"
+                continue
+            fi
+
+            if [[ "$HAS_PG_BUFFERCACHE" == "false" ]] \
+                && _sql_list_contains "$rel_path" "${SQL_REQUIRES_PG_BUFFERCACHE[@]}"; then
+                [[ "$VERBOSE" == "true" ]] && log "Skipping pg_buffercache-dependent script: $rel_path"
+                continue
+            fi
+
+            if ! psql -v ON_ERROR_STOP=1 -f "$sql_file" > /dev/null 2>&1; then
+                error "SQL execution error in: $rel_path"
+                failed=1
+            elif [[ "$VERBOSE" == "true" ]]; then
+                log "OK: $rel_path"
+            fi
+        done < <(find "$dir" -maxdepth 2 -type f -name '*.sql' -print0)
     done
-    return 0
+
+    return "$failed"
 }
 
 test_automation_scripts() {
@@ -305,7 +424,11 @@ generate_test_report() {
     echo "Tests run: $TESTS_RUN"
     echo "Passed: $TESTS_PASSED"
     echo "Failed: $TESTS_FAILED"
-    echo "Success rate: $(( TESTS_PASSED * 100 / TESTS_RUN ))%"
+    if [[ "$TESTS_RUN" -gt 0 ]]; then
+        echo "Success rate: $(( TESTS_PASSED * 100 / TESTS_RUN ))%"
+    else
+        echo "Success rate: n/a (no tests matched the current pattern/filters)"
+    fi
     echo "==============================================="
 
     if [[ "$TESTS_FAILED" -gt 0 ]]; then
@@ -333,6 +456,10 @@ main() {
     run_test "connection_basic" test_database_connection
     run_test "connection_permissions" test_database_permissions
     run_test "connection_extensions" test_extensions_available
+
+    # Detect environment once; cheaper than doing it per-file, and the skip
+    # flags need to be populated before the syntax sweep starts.
+    detect_environment_capabilities
 
     # Syntax tests
     run_test "syntax_sql_files" test_sql_syntax