Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 21 additions & 21 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ jobs:
psql -h localhost -p 5434 -U postgres -d target_db <<SQL
CREATE EXTENSION IF NOT EXISTS pgtap;
CREATE EXTENSION IF NOT EXISTS pgclone;
SELECT pgclone_version();
SELECT pgclone.version();
SQL

psql -h localhost -p 5434 -U postgres -d target_db \
Expand All @@ -129,7 +129,7 @@ jobs:
}

# Clone roles
RESULT=$(pg "SELECT pgclone_clone_roles('${SOURCE_CONNINFO}');" || echo "ERROR")
RESULT=$(pg "SELECT pgclone.clone_roles('${SOURCE_CONNINFO}');" || echo "ERROR")
run_test "clone_roles runs" "[ '$RESULT' != 'ERROR' ]"
R1=$(pg "SELECT 1 FROM pg_roles WHERE rolname = 'test_reader';" || echo "0")
run_test "test_reader exists" "[ '$R1' = '1' ]"
Expand All @@ -139,27 +139,27 @@ jobs:
run_test "test_admin exists" "[ '$R3' = '1' ]"

# Verify
VC=$(pg "SELECT count(*) FROM pgclone_verify('${SOURCE_CONNINFO}', 'test_schema');" || echo "0")
VC=$(pg "SELECT count(*) FROM pgclone.verify('${SOURCE_CONNINFO}', 'test_schema');" || echo "0")
run_test "verify returns rows" "[ '$VC' -ge 1 ]"

# Masking report
RC=$(pg "SELECT count(*) FROM pgclone_masking_report('test_schema') WHERE table_name = 'employees' AND column_name = 'email';" || echo "0")
RC=$(pg "SELECT count(*) FROM pgclone.masking_report('test_schema') WHERE table_name = 'employees' AND column_name = 'email';" || echo "0")
run_test "report detects email" "[ '$RC' = '1' ]"

# DDM
pg "DROP TABLE IF EXISTS test_schema.employees_ddm CASCADE;" || true
pg "SELECT pgclone_table('${SOURCE_CONNINFO}', 'test_schema', 'employees', true, 'employees_ddm');" || true
RESULT=$(pg "SELECT pgclone_create_masking_policy('test_schema', 'employees_ddm', '{\"email\": \"email\", \"full_name\": \"name\", \"ssn\": \"null\"}', 'postgres');" || echo "ERROR")
pg "SELECT pgclone.table('${SOURCE_CONNINFO}', 'test_schema', 'employees', true, 'employees_ddm');" || true
RESULT=$(pg "SELECT pgclone.create_masking_policy('test_schema', 'employees_ddm', '{\"email\": \"email\", \"full_name\": \"name\", \"ssn\": \"null\"}', 'postgres');" || echo "ERROR")
run_test "create_masking_policy" "[ '$RESULT' != 'ERROR' ]"
MASKED=$(pg "SELECT count(*) FROM test_schema.employees_ddm_masked WHERE full_name = 'XXXX';" || echo "0")
run_test "masked view works" "[ '$MASKED' = '5' ]"
pg "SELECT pgclone_drop_masking_policy('test_schema', 'employees_ddm');" || true
pg "SELECT pgclone.drop_masking_policy('test_schema', 'employees_ddm');" || true
pg "DROP TABLE IF EXISTS test_schema.employees_ddm CASCADE;" || true

echo "LOOPBACK: $PASS passed, $FAIL failed"
[ $FAIL -eq 0 ] || exit 1

- name: "Test: pgclone_database_create"
- name: "Test: pgclone.database_create"
run: |
SOURCE_CONNINFO="host=localhost port=5433 dbname=source_db user=postgres password=testpass"

Expand All @@ -170,7 +170,7 @@ jobs:

# Create and clone
psql -h localhost -p 5434 -U postgres -d postgres -v ON_ERROR_STOP=1 \
-c "SELECT pgclone_database_create('${SOURCE_CONNINFO}', 'clone_test_db', true);"
-c "SELECT pgclone.database_create('${SOURCE_CONNINFO}', 'clone_test_db', true);"

# Verify
DB_EXISTS=$(psql -h localhost -p 5434 -U postgres -d postgres -tAc \
Expand All @@ -183,7 +183,7 @@ jobs:

# Idempotent re-clone
psql -h localhost -p 5434 -U postgres -d postgres -v ON_ERROR_STOP=1 \
-c "SELECT pgclone_database_create('${SOURCE_CONNINFO}', 'clone_test_db', false);"
-c "SELECT pgclone.database_create('${SOURCE_CONNINFO}', 'clone_test_db', false);"

# Cleanup
psql -h localhost -p 5434 -U postgres -d postgres \
Expand All @@ -206,7 +206,7 @@ jobs:
}

# Clean slate — drop leftover tables from sync tests
$PG -c "SELECT pgclone_clear_jobs();" 2>/dev/null || true
$PG -c "SELECT pgclone.clear_jobs();" 2>/dev/null || true
$PG <<SQL 2>/dev/null
DROP TABLE IF EXISTS public.simple_test CASCADE;
DROP TABLE IF EXISTS public.simple_test_copy CASCADE;
Expand All @@ -218,10 +218,10 @@ jobs:

# 1: table_async basic
echo "---- Async: table_async basic ----"
JOB=$($PG -tAc "SELECT pgclone_table_async('${SOURCE_CONNINFO}', 'public', 'simple_test', true);")
JOB=$($PG -tAc "SELECT pgclone.table_async('${SOURCE_CONNINFO}', 'public', 'simple_test', true);")
run_test "returns job_id" "[ -n '$JOB' ] && [ '$JOB' -gt 0 ]"
for i in $(seq 1 30); do
S=$($PG -tAc "SELECT status FROM pgclone_jobs_view WHERE job_id=$JOB;" 2>/dev/null | tr -d '[:space:]')
S=$($PG -tAc "SELECT status FROM pgclone.jobs_view WHERE job_id=$JOB;" 2>/dev/null | tr -d '[:space:]')
[ "$S" = "completed" ] || [ "$S" = "failed" ] && break; sleep 1
done
run_test "job completed" "[ '$S' = 'completed' ]"
Expand All @@ -230,9 +230,9 @@ jobs:

# 2: table_async with target name
echo "---- Async: table_async target name ----"
JOB2=$($PG -tAc "SELECT pgclone_table_async('${SOURCE_CONNINFO}', 'public', 'simple_test', true, 'async_renamed');")
JOB2=$($PG -tAc "SELECT pgclone.table_async('${SOURCE_CONNINFO}', 'public', 'simple_test', true, 'async_renamed');")
for i in $(seq 1 30); do
S2=$($PG -tAc "SELECT status FROM pgclone_jobs_view WHERE job_id=$JOB2;" 2>/dev/null | tr -d '[:space:]')
S2=$($PG -tAc "SELECT status FROM pgclone.jobs_view WHERE job_id=$JOB2;" 2>/dev/null | tr -d '[:space:]')
[ "$S2" = "completed" ] || [ "$S2" = "failed" ] && break; sleep 1
done
run_test "renamed job completed" "[ '$S2' = 'completed' ]"
Expand All @@ -242,9 +242,9 @@ jobs:
# 3: schema_async
echo "---- Async: schema_async ----"
$PG -c "DROP SCHEMA IF EXISTS test_schema CASCADE;" 2>/dev/null || true
JOB3=$($PG -tAc "SELECT pgclone_schema_async('${SOURCE_CONNINFO}', 'test_schema', true);")
JOB3=$($PG -tAc "SELECT pgclone.schema_async('${SOURCE_CONNINFO}', 'test_schema', true);")
for i in $(seq 1 60); do
S3=$($PG -tAc "SELECT status FROM pgclone_jobs_view WHERE job_id=$JOB3;" 2>/dev/null | tr -d '[:space:]')
S3=$($PG -tAc "SELECT status FROM pgclone.jobs_view WHERE job_id=$JOB3;" 2>/dev/null | tr -d '[:space:]')
[ "$S3" = "completed" ] || [ "$S3" = "failed" ] && break; sleep 1
done
run_test "schema_async completed" "[ '$S3' = 'completed' ]"
Expand All @@ -253,16 +253,16 @@ jobs:

# 4: progress/jobs/view
echo "---- Async: progress & jobs ----"
PR=$($PG -tAc "SELECT pgclone_progress($JOB);" 2>/dev/null)
PR=$($PG -tAc "SELECT pgclone.progress($JOB);" 2>/dev/null)
run_test "progress returns JSON" "echo '$PR' | grep -q 'job_id'"
JJ=$($PG -tAc "SELECT pgclone_jobs();" 2>/dev/null)
JJ=$($PG -tAc "SELECT pgclone.jobs();" 2>/dev/null)
run_test "jobs returns JSON" "echo '$JJ' | grep -q 'job_id'"
VC=$($PG -tAc "SELECT count(*) FROM pgclone_jobs_view;" 2>/dev/null | tr -d '[:space:]')
VC=$($PG -tAc "SELECT count(*) FROM pgclone.jobs_view;" 2>/dev/null | tr -d '[:space:]')
run_test "jobs_view has rows" "[ '$VC' -ge 1 ]"

# 5: clear_jobs
echo "---- Async: clear_jobs ----"
CL=$($PG -tAc "SELECT pgclone_clear_jobs();" 2>/dev/null | tr -d '[:space:]')
CL=$($PG -tAc "SELECT pgclone.clear_jobs();" 2>/dev/null | tr -d '[:space:]')
run_test "clear_jobs works" "[ '$CL' -ge 1 ]"

echo ""
Expand Down
31 changes: 31 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,37 @@

All notable changes to pgclone are documented in this file.

## [4.0.0] — BREAKING

### Changed
- **Schema namespace**: All pgclone functions now live under the `pgclone` schema, created automatically by the extension
- `pgclone_table(...)` → `pgclone.table(...)`
- `pgclone_schema(...)` → `pgclone.schema(...)`
- `pgclone_database(...)` → `pgclone.database(...)`
- `pgclone_database_create(...)` → `pgclone.database_create(...)`
- `pgclone_table_async(...)` → `pgclone.table_async(...)`
- `pgclone_schema_async(...)` → `pgclone.schema_async(...)`
- `pgclone_progress(...)` → `pgclone.progress(...)`
- `pgclone_cancel(...)` → `pgclone.cancel(...)`
- `pgclone_resume(...)` → `pgclone.resume(...)`
- `pgclone_jobs()` → `pgclone.jobs()`
- `pgclone_clear_jobs()` → `pgclone.clear_jobs()`
- `pgclone_progress_detail()` → `pgclone.progress_detail()`
- `pgclone_jobs_view` → `pgclone.jobs_view`
- `pgclone_discover_sensitive(...)` → `pgclone.discover_sensitive(...)`
- `pgclone_mask_in_place(...)` → `pgclone.mask_in_place(...)`
- `pgclone_create_masking_policy(...)` → `pgclone.create_masking_policy(...)`
- `pgclone_drop_masking_policy(...)` → `pgclone.drop_masking_policy(...)`
- `pgclone_clone_roles(...)` → `pgclone.clone_roles(...)`
- `pgclone_verify(...)` → `pgclone.verify(...)`
- `pgclone_masking_report(...)` → `pgclone.masking_report(...)`
- `pgclone_version()` → `pgclone.version()`
- `pgclone_table_ex(...)` → `pgclone.table_ex(...)`
- `pgclone_schema_ex(...)` → `pgclone.schema_ex(...)`
- `pgclone_functions(...)` → `pgclone.functions(...)`
- Extension control file now specifies `schema = pgclone`
- **Upgrade path**: This is a breaking change. Users must `DROP EXTENSION pgclone; CREATE EXTENSION pgclone;` to upgrade from v3.x. All application queries must be updated to use the new `pgclone.` prefix.

## [3.6.0]

### Added
Expand Down
6 changes: 3 additions & 3 deletions META.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
"name": "pgclone",
"abstract": "Clone PostgreSQL databases, schemas, tables between staging, test, dev and prod environments",
"description": "PostgreSQL extension for easily cloning your DB, Schemas, Tables and more between environments",
"version": "3.6.0",
"version": "4.0.0",
"maintainer": "Valeh Agayev <valeh.agayev@gmail.com>",
"license": "postgresql",
"provides": {
"pgclone": {
"abstract": "Clone PostgreSQL databases, schemas, and tables across environments",
"file": "sql/pgclone--3.6.0.sql",
"version": "3.6.0"
"file": "sql/pgclone--4.0.0.sql",
"version": "4.0.0"
}
},
"prereqs": {
Expand Down
17 changes: 9 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
[![CI](https://github.com/valehdba/pgclone/actions/workflows/ci.yml/badge.svg)](https://github.com/valehdba/pgclone/actions/workflows/ci.yml)
[![Postgres 14–18](https://img.shields.io/badge/Postgres-14%E2%80%9318-336791?logo=postgresql&logoColor=white)](https://github.com/valehdba/pgclone)
[![License](https://img.shields.io/badge/License-PostgreSQL-blue.svg)](https://github.com/valehdba/pgclone/blob/main/LICENSE)
[![Version](https://img.shields.io/badge/version-3.6.0-orange)](https://github.com/valehdba/pgclone/releases/tag/v3.6.0)
[![Version](https://img.shields.io/badge/version-4.0.0-orange)](https://github.com/valehdba/pgclone/releases/tag/v4.0.0)

A PostgreSQL extension that clones databases, schemas, tables, and functions between PostgreSQL instances — directly from SQL. No `pg_dump`, no `pg_restore`, no shell scripts.

Expand Down Expand Up @@ -36,19 +36,19 @@ sudo make install
CREATE EXTENSION pgclone;

-- Clone a table
SELECT pgclone_table(
SELECT pgclone.table(
'host=source-server dbname=mydb user=postgres password=secret',
'public', 'customers', true
);

-- Clone an entire schema
SELECT pgclone_schema(
SELECT pgclone.schema(
'host=source-server dbname=mydb user=postgres password=secret',
'sales', true
);

-- Clone a full database
SELECT pgclone_database(
SELECT pgclone.database(
'host=source-server dbname=mydb user=postgres password=secret',
true
);
Expand Down Expand Up @@ -96,7 +96,7 @@ sudo make install PG_CONFIG=/usr/lib/postgresql/18/bin/pg_config

```sql
CREATE EXTENSION pgclone;
SELECT pgclone_version();
SELECT pgclone.version();
```

For async operations, add to `postgresql.conf` and restart:
Expand Down Expand Up @@ -127,8 +127,8 @@ pgclone uses Unix domain sockets for local loopback connections, so the default
- [x] v1.1.0: Selective column cloning and data filtering
- [x] v1.2.0: Materialized views and exclusion constraints
- [x] v2.0.0: True multi-worker parallel cloning
- [x] v2.0.1: `pgclone_database_create()` — create + clone database
- [x] v2.1.0: Progress tracking view (`pgclone_jobs_view`)
- [x] v2.0.1: `pgclone.database_create()` — create + clone database
- [x] v2.1.0: Progress tracking view (`pgclone.jobs_view`)
- [x] v2.1.1: Visual progress bar
- [x] v2.1.3: Elapsed time tracking
- [x] v2.1.4: Unix domain socket auth (no more pg_hba.conf trust requirement)
Expand All @@ -141,7 +141,8 @@ pgclone uses Unix domain sockets for local loopback connections, so the default
- [x] v3.4.0: Clone roles with permissions and passwords
- [x] v3.5.0: Clone verification — compare row counts across source and target
- [x] v3.6.0: GDPR/Compliance masking report
- [ ] v4.0.0: Copy-on-Write (CoW) mode for local cloning
- [x] v4.0.0: Schema namespace — all functions under `pgclone` schema (`pgclone.table()`, `pgclone.schema()`, etc.)
- [ ] v5.0.0: Copy-on-Write (CoW) mode for local cloning

## License

Expand Down
6 changes: 3 additions & 3 deletions docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ typedef struct PgcloneSharedState {

- Allocated once during `_PG_init()` via shared memory hooks
- Protected by a lightweight lock (`LWLock`) for concurrent access
- Read by `pgclone_progress()`, `pgclone_jobs()`, and `pgclone_jobs_view`
- Read by `pgclone.progress()`, `pgclone.jobs()`, and `pgclone.jobs_view`
- Written by background workers as they progress

---
Expand Down Expand Up @@ -189,7 +189,7 @@ PostgreSQL 17 removed the `die` signal handler, replacing it with `SignalHandler

## Background Worker Lifecycle

1. **Registration:** `pgclone_table_async()` or `pgclone_schema_async()` allocates a job slot in shared memory, populates connection info and parameters, then calls `RegisterDynamicBackgroundWorker()`.
1. **Registration:** `pgclone.table_async()` or `pgclone.schema_async()` allocates a job slot in shared memory, populates connection info and parameters, then calls `RegisterDynamicBackgroundWorker()`.

2. **Startup:** The worker process starts via `pgclone_bgw_main()`, which:
- Sets up signal handlers
Expand All @@ -198,7 +198,7 @@ PostgreSQL 17 removed the `die` signal handler, replacing it with `SignalHandler

3. **Execution:** The worker calls the same core clone functions used by sync operations, with periodic updates to shared memory (rows copied, current table, elapsed time).

4. **Worker Pool mode (v2.2.0):** For `pgclone_schema_async` with `"parallel": N`, the parent process:
4. **Worker Pool mode (v2.2.0):** For `pgclone.schema_async` with `"parallel": N`, the parent process:
- Queries the source for the list of tables
- Populates a shared-memory task queue (`PgclonePoolQueue`)
- Launches exactly N pool workers via `pgclone_pool_worker_main()`
Expand Down
Loading
Loading