Added PG TDE tests

Author: Mohit Joshi

postgresql/bugs/PG-1727/PG-1727.sh

@@ -6,11 +6,12 @@ PRIMARY_DATA=$INSTALL_DIR/primary_data
 REPLICA_DATA=$INSTALL_DIR/replica_data
 PRIMARY_LOGFILE=$PRIMARY_DATA/server.log
 REPLICA_LOGFILE=$REPLICA_DATA/server.log
-TABLES=1000
+TABLES=100
+export PGCTLTIMEOUT=600
 
 # initate the database
 initialize_server() {
-    PG_PIDS=$(lsof -ti :5433 -ti :5434 2>/dev/null) || true
+    PG_PIDS=$(lsof -ti :5433 -ti :5434 2>/dev/null)
     if [[ -n "$PG_PIDS" ]]; then
         echo "Killing PostgreSQL processes: $PG_PIDS"
         kill -9 $PG_PIDS
@@ -59,9 +60,9 @@ logging_collector = on
 log_directory = '$REPLICA_DATA'
 log_filename = 'server.log'
 log_statement = 'all'
+unix_socket_directories = '/tmp'
 SQL
     $INSTALL_DIR/bin/pg_ctl -D $REPLICA_DATA -l $REPLICA_LOGFILE start
-    
 }
 
 enable_tde_and_create_load() {
@@ -85,7 +86,7 @@ rotate_server_key() {
     echo "Rotating server keys for 30 seconds..."
 
     local start_time=$(date +%s)
-    local duration=30
+    local duration=8
 
     while [[ $(($(date +%s) - start_time)) -lt $duration ]]; do
         RAND=$RANDOM
@@ -97,11 +98,11 @@ rotate_server_key() {
 
 verify_streaming_replication() {
     echo "Creating verification table on primary..."
-    $INSTALL_DIR/bin/psql -d postgres -p 5433 -c "CREATE TABLE verify_replication(id INT PRIMARY KEY, val TEXT);" || exit 1
-    $INSTALL_DIR/bin/psql -d postgres -p 5433 -c "INSERT INTO verify_replication VALUES (1, 'streaming_test');" || exit 1
+    $INSTALL_DIR/bin/psql -d postgres -p 5433 -c "CREATE TABLE verify_replication(id INT PRIMARY KEY, val TEXT);"
+    $INSTALL_DIR/bin/psql -d postgres -p 5433 -c "INSERT INTO verify_replication VALUES (1, 'streaming_test');"
 
     echo "Waiting for replication to apply..."
-    sleep 5
+    sleep 20
 
     echo "Checking data on replica..."
     result=$($INSTALL_DIR/bin/psql -d postgres -p 5434 -Atc "SELECT val FROM verify_replication WHERE id=1;" 2>/dev/null)
@@ -121,7 +122,7 @@ verify_logical_replication() {
     local delay=5
     local all_tables_replicated=true
 
-    for ((i=1; i<=TABLES; i++)); do
+    for ((i=1; i<=$TABLES; i++)); do
         table_name="sbtest$i"
         echo "Checking table: $table_name"
 
@@ -167,13 +168,13 @@ run_workload_during_conversion() {
         --table-size=1000 \
         --time=30 \
         run > /tmp/workload.log 2>&1 &
-    WORKLOAD_PID=$!
 }
 
 run_pg_createsubscriber() {
     echo "Running pg_createsubscriber..."
 
     stop_server $REPLICA_DATA
+    rm -f $REPLICA_DATA/postmaster.pid
 
     $INSTALL_DIR/bin/pg_createsubscriber \
         -d postgres \
@@ -183,6 +184,7 @@ run_pg_createsubscriber() {
 	--publisher-server="host=127.0.0.1 port=5433 dbname=postgres user=$(whoami)" \
         --publication=mypub \
         --subscription=mysub \
+	--socketdir=/tmp \
         --verbose
 
     start_server $REPLICA_DATA
@@ -206,11 +208,13 @@ echo "5=>Verifying Streaming Replication"
 verify_streaming_replication
 
 echo "6=>Run workload in parallel on Primary Server"
-rotate_server_key &
+#rotate_server_key &
 run_workload_during_conversion
 
 echo "7=>Convert physical replica into logical replica"
-run_pg_createsubscriber
+#run_pg_createsubscriber
+stop_server $REPLICA_DATA
+start_server $REPLICA_DATA
 
 #echo "8=>Verifying Logical Replication"
 #verify_logical_replication

postgresql/tests/PG-1813.sh renamed to postgresql/bugs/PG-1809/PG-1809.sh

@@ -12,8 +12,8 @@ PG_BASEBACKUP="$INSTALL_DIR/bin/pg_basebackup"
 PSQL="$INSTALL_DIR/bin/psql"
 SYSBENCH="/usr/bin/sysbench"
 
-PORT_PRIMARY=5432
-PORT_REPLICA=5433
+PORT_PRIMARY=5433
+PORT_REPLICA=5434
 REPL_USER=repl
 REPL_PASS=replica
 DB_NAME=postgres
@@ -22,22 +22,18 @@ DB_USER=$(whoami)
 # Clean slate
 pkill -9 postgres
 rm -rf "$PRIMARY_DATA" "$REPLICA_DATA"
-rm -rf /tmp/primary_keyfile
 
 # Step 1: Init primary
 $INSTALL_DIR/bin/initdb -D "$PRIMARY_DATA"
 
 # Configure primary
 cat >> "$PRIMARY_DATA/postgresql.conf" <<EOF
 port = $PORT_PRIMARY
-shared_preload_libraries = 'pg_tde'
-default_table_access_method = 'tde_heap'
 wal_level = replica
-wal_compression = on
 wal_log_hints = on
-wal_keep_size = 512MB
+wal_keep_size = 1024MB
 max_replication_slots = 2
-max_wal_senders = 2
+max_wal_senders = 10
 listen_addresses = 'localhost'
 logging_collector = on
 log_directory = 'log'
@@ -51,92 +47,87 @@ echo "=> Step 1: Start primary"
 echo "#########################"
 $PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" start
 sleep 3
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE EXTENSION pg_tde;"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_add_global_key_provider_file('file_provider','/tmp/primary_keyfile');"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_create_key_using_global_key_provider('key1','file_provider');"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_set_default_key_using_global_key_provider('key1','file_provider');"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
-# Restart primary
-$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" restart
-sleep 3
+$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE TABLE t1(a int, b varchar(20));"
 
 # Create replication user
 $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE ROLE $REPL_USER WITH LOGIN REPLICATION SUPERUSER PASSWORD '$REPL_PASS';"
 
-# Create tables
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE TABLE t1(id INT, name TEXT);"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(101,'First Record Before BaseBackup');"
-
 echo "=> Step 2: Take base backup for replica"
 echo "#######################################"
 $PG_BASEBACKUP -D "$REPLICA_DATA" -X stream -R -h localhost -p $PORT_PRIMARY -U $REPL_USER
 
 # Configure replica
 cat > "$REPLICA_DATA/postgresql.conf" <<EOF
 port = $PORT_REPLICA
-shared_preload_libraries = 'pg_tde'
-default_table_access_method = 'tde_heap'
 hot_standby = on
 logging_collector = on
 log_directory = 'log'
 log_filename = 'replica.log'
-wal_level = replica
-wal_compression = on
-wal_keep_size= 512MB
-max_wal_senders = 2
+wal_keep_size=1024MB
+max_wal_senders = 10
 EOF
 
 echo "=>Step 3: Start replica"
 echo "########################"
 $PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" start
-sleep 5
-$PSQL -p $PORT_REPLICA -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
-# Restart replica
-$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
-sleep 5
+sleep 2
+
+echo "=> Check if replication is working"
+$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(1,'mohit');"
+sleep 2
 
-echo "=>Step 4: Generate some WAL on primary"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES (102, 'Second Record After Setting Streaming Replication');"
+echo "=> Verify if rows get replicated"
+$PSQL -p $PORT_REPLICA -d $DB_NAME -c "SELECT * FROM t1;"
 
+echo "=>Step 4: Stop primary (simulate crash)"
+echo "#######################################"
+echo "Simulating primary crash..."
 $PG_CTL -D "$PRIMARY_DATA" -m immediate stop
-sleep 3
 
 echo "=> Step 5: Promote replica"
 echo "##########################"
-# Remove auto conf file which was created by pg_basebackup
-rm -f $REPLICA_DATA/postgresql.auto.conf
 $PG_CTL -D "$REPLICA_DATA" promote
+sleep 5
 
-echo "=>Step 6: Use sysbench to generate WAL on promoted replica"
+echo "=>Step 6: Create table to generate WAL on promoted replica"
 echo "##########################################################"
-echo "Generating WAL with sysbench..."
-$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t1 VALUES(103,'Third Record Replicated Via PgRewind');"
+echo "Generating WAL..."
+$PSQL -p $PORT_REPLICA -d $DB_NAME -c "CREATE TABLE t2(a int, b varchar(20));"
 
 # Backing up the Original config file
 cp $PRIMARY_DATA/postgresql.conf /tmp/postgresql_bk.conf
-echo "=> Step 8: Rewind old primary"
+echo "=> Step 7: Rewind old primary"
 echo "############################"
 echo "Rewinding old primary..."
 $PG_REWIND --target-pgdata="$PRIMARY_DATA" \
   --source-server="host=localhost port=$PORT_REPLICA user=$REPL_USER dbname=$DB_NAME"
 
 # Restoring the Original config file
-cp /tmp/postgresql_bk.conf $PRIMARY_DATA/postgresql.conf
+mv /tmp/postgresql_bk.conf $PRIMARY_DATA/postgresql.conf
 
-echo "=> Step 9: Configure old primary as standby"
+echo "=> Step 8: Configure old primary as standby"
 echo "###########################################"
 touch $PRIMARY_DATA/standby.signal
 cat >> "$PRIMARY_DATA/postgresql.conf" <<EOF
 primary_conninfo = 'host=localhost port=$PORT_REPLICA user=$REPL_USER password=$REPL_PASS'
 EOF
 
-echo "=> Step 10: Start rewound primary"
+echo "=> Step 9: Start rewound primary"
 echo "################################"
 $PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" start
 sleep 5
 
-# Validate data on both servers
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM t1"
-$PSQL -p $PORT_REPLICA -d $DB_NAME -c "SELECT * FROM t1"
+echo "=> Check if replication is working"
+$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t2 VALUES(2,'mohit');"
+sleep 2
+
+echo "=> Verify if rows get replicated"
+$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM t2;"
+
+
+# Done
+echo -e "\n✅ Check status of Old Primary"
+$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM pg_stat_wal_receiver;"
 
-tail -10f $PRIMARY_DATA/log/primary.log
+# Tail logs
+tail -n 10 "$PRIMARY_DATA/log/primary.log"

postgresql/bugs/PG-1811/PG-1811.sh

@@ -11,6 +11,10 @@ PG_REWIND="$INSTALL_DIR/bin/pg_rewind"
 PG_BASEBACKUP="$INSTALL_DIR/bin/pg_basebackup"
 PSQL="$INSTALL_DIR/bin/psql"
 SYSBENCH="/usr/bin/sysbench"
+SYSBENCH_TABLES=1000
+SYSBENCH_RECORDS=1000
+THREADS=10
+PGCTLTIMEOUT=600
 
 PORT_PRIMARY=5433
 PORT_REPLICA=5434
@@ -34,7 +38,7 @@ default_table_access_method = 'tde_heap'
 port = $PORT_PRIMARY
 wal_level = replica
 wal_log_hints = on
-wal_keep_size = 1024MB
+wal_keep_size = 2048MB
 max_replication_slots = 2
 max_wal_senders = 10
 listen_addresses = 'localhost'
@@ -55,10 +59,10 @@ $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_add_global_key_provider_fil
 $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_create_key_using_global_key_provider('key1','file_provider');"
 $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_set_server_key_using_global_key_provider('key1','file_provider');"
 $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_set_key_using_global_key_provider('key1','file_provider');"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
+#$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
 # Restart primary
-$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" restart
-sleep 3
+#$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" restart
+#sleep 3
 
 # Create replication user
 $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE ROLE $REPL_USER WITH LOGIN REPLICATION SUPERUSER PASSWORD '$REPL_PASS';"
@@ -84,10 +88,10 @@ echo "=>Step 3: Start replica"
 echo "########################"
 $PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" start
 sleep 5
-$PSQL -p $PORT_REPLICA -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
+#$PSQL -p $PORT_REPLICA -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
 # Restart replica
-$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
-sleep 5
+#$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
+#sleep 5
 
 echo "=>Step 3: Stop primary (simulate crash)"
 echo "#######################################"
@@ -105,12 +109,12 @@ echo "##########################################################"
 echo "Generating WAL with sysbench..."
 $SYSBENCH --db-driver=pgsql --pgsql-host=127.0.0.1 --pgsql-port=$PORT_REPLICA \
   --pgsql-user=$DB_USER --pgsql-db=$DB_NAME \
-  --threads=10 --tables=1000 --table-size=1000 \
+  --threads=$THREADS --tables=$SYSBENCH_TABLES --table-size=$SYSBENCH_RECORDS \
   /usr/share/sysbench/oltp_write_only.lua prepare
 
 $SYSBENCH --db-driver=pgsql --pgsql-host=127.0.0.1 --pgsql-port=$PORT_REPLICA \
   --pgsql-user=$DB_USER --pgsql-db=$DB_NAME \
-  --threads=10 --tables=1000 --table-size=1000 --time=300 --report-interval=5 \
+  --threads=$THREADS --tables=$SYSBENCH_TABLES --table-size=$SYSBENCH_RECORDS --time=200 --report-interval=5 \
   /usr/share/sysbench/oltp_write_only.lua run
 
 # Backing up the Original config file

postgresql/bugs/PG-1813/PG-1813.sh

@@ -65,7 +65,7 @@ $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE ROLE $REPL_USER WITH LOGIN REPLICA
 
 # Create tables
 $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE TABLE t1(id INT, name TEXT);"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(101,'First Record');"
+$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(101,'First Record Before BaseBackup');"
 
 echo "=> Step 2: Take base backup for replica"
 echo "#######################################"
@@ -96,7 +96,7 @@ $PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
 sleep 5
 
 echo "=>Step 4: Generate some WAL on primary"
-$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES (102, 'Second Record');"
+$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES (102, 'Second Record After Setting Streaming Replication');"
 
 $PG_CTL -D "$PRIMARY_DATA" -m immediate stop
 sleep 3
@@ -110,7 +110,7 @@ $PG_CTL -D "$REPLICA_DATA" promote
 echo "=>Step 6: Use sysbench to generate WAL on promoted replica"
 echo "##########################################################"
 echo "Generating WAL with sysbench..."
-$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t1 VALUES(103,'Third Record');"
+$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t1 VALUES(103,'Third Record Replicated Via PgRewind');"
 
 # Backing up the Original config file
 cp $PRIMARY_DATA/postgresql.conf /tmp/postgresql_bk.conf
@@ -140,4 +140,3 @@ $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM t1"
 $PSQL -p $PORT_REPLICA -d $DB_NAME -c "SELECT * FROM t1"
 
 tail -10f $PRIMARY_DATA/log/primary.log
-