Merge branch '152-rename-rds-iam-type' into 'master'

fix Database Lab configuration files (#152):
* remove excess sections in global config
* rename RDS IAM source type
* add `--no-owner` option

See merge request postgres-ai/database-lab!158

Author: agneum

configs/config.example.logical_generic.yml

@@ -29,16 +29,6 @@ global:
   # Database engine. Currently, the only supported option: "postgres".
   engine: postgres
 
-  # Docker image to perform various tasks, such as:
-  #   - initial data retrieval,
-  #   - continuous data synchronization,
-  #   - pre-processing.
-  # Default: "official" PostgreSQL image from Docker.
-  # Alternative: an extended image from the Postgres.ai team,
-  # containing a number of popular extensions, including those
-  # RDS provides. See https://postgres.ai/docs/database-lab/supported_databases
-  dockerImage: "postgres:12-alpine"
-
   # Full path to data directory. This directory must already exist
   # before launching Database Lab instance. It may be empty if
   # data initialization is configured (see below).

configs/config.example.logical_rds.yml renamed to configs/config.example.logical_rds_iam.yml

@@ -29,16 +29,6 @@ global:
   # Database engine. Currently, the only supported option: "postgres".
   engine: postgres
 
-  # Docker image to perform various tasks, such as:
-  #   - initial data retrieval,
-  #   - continuous data synchronization,
-  #   - pre-processing.
-  # Default: "official" PostgreSQL image from Docker.
-  # Alternative: an extended image from the Postgres.ai team,
-  # containing a number of popular extensions, including those
-  # RDS provides. See https://postgres.ai/docs/database-lab/supported_databases
-  dockerImage: "postgres:12-alpine"
-
   # Full path to data directory. This directory must already exist
   # before launching Database Lab instance. It may be empty if
   # data initialization is configured (see below).
@@ -146,16 +136,17 @@ retrieval:
 
         # Source of data.
         source:
-          # Source types: "local", "remote", "rds"
-          type: rds
+          # Source types: "local", "remote", "rdsIam"
+          type: rdsIam
+
           # RDS database details for pg_dump
           connection:
             dbname: test
             username: test_user
 
           # Optional definition of RDS data source.
-          rds:
-            # RDS policy name.
+          rdsIam:
+            # RDS IAM policy name.
             iamPolicyName: rds-dblab-retrieval
 
             # AWS Region.

configs/config.example.physical_generic.yml

@@ -29,16 +29,6 @@ global:
   # Database engine. Currently, the only supported option: "postgres".
   engine: postgres
 
-  # Docker image to perform various tasks, such as:
-  #   - initial data retrieval,
-  #   - continuous data synchronization,
-  #   - pre-processing.
-  # Default: "official" PostgreSQL image from Docker.
-  # Alternative: an extended image from the Postgres.ai team,
-  # containing a number of popular extensions, including those
-  # RDS provides. See https://postgres.ai/docs/database-lab/supported_databases
-  dockerImage: "postgres:12-alpine"
-
   # Full path to data directory. This directory must already exist
   # before launching Database Lab instance. It may be empty if
   # data initialization is configured (see below).

configs/config.example.physical_walg.yml

@@ -29,16 +29,6 @@ global:
   # Database engine. Currently, the only supported option: "postgres".
   engine: postgres
 
-  # Docker image to perform various tasks, such as:
-  #   - initial data retrieval,
-  #   - continuous data synchronization,
-  #   - pre-processing.
-  # Default: "official" PostgreSQL image from Docker.
-  # Alternative: an extended image from the Postgres.ai team,
-  # containing a number of popular extensions, including those
-  # RDS provides. See https://postgres.ai/docs/database-lab/supported_databases
-  dockerImage: "postgres:12-alpine"
-
   # Full path to data directory. This directory must already exist
   # before launching Database Lab instance. It may be empty if
   # data initialization is configured (see below).

pkg/retrieval/engine/postgres/logical/dump.go

@@ -42,7 +42,7 @@ const (
 	// Defines dump source types.
 	sourceTypeLocal  = "local"
 	sourceTypeRemote = "remote"
-	sourceTypeRDS    = "rds"
+	sourceTypeRDS    = "rdsIam"
 
 	// reservePort defines reserve port in case of a local dump.
 	reservePort = 9999
@@ -79,7 +79,7 @@ type DumpOptions struct {
 type Source struct {
 	Type       string     `yaml:"type"`
 	Connection Connection `yaml:"connection"`
-	RDS        *RDSConfig `yaml:"rds"`
+	RDS        *RDSConfig `yaml:"rdsIam"`
 }
 
 type dumpJobConfig struct {
@@ -174,12 +174,12 @@ func (d *DumpJob) setupDumper() error {
 
 	case sourceTypeRDS:
 		if d.Source.RDS == nil {
-			return errors.New("the RDS configuration section must not be empty when using the RDS source type")
+			return errors.New("the RDS IAM configuration section must not be empty when using the RDS IAM source type")
 		}
 
 		dumper, err := newRDSDumper(d.Source.RDS)
 		if err != nil {
-			return errors.Wrap(err, "failed to create an RDS dumper")
+			return errors.Wrap(err, "failed to create an RDS IAM dumper")
 		}
 
 		d.dumper = dumper
@@ -446,7 +446,8 @@ func (d *DumpJob) buildLogicalDumpCommand() []string {
 }
 
 func (d *DumpJob) buildLogicalRestoreCommand() []string {
-	restoreCmd := []string{"|", "pg_restore", "--username", defaults.Username, "--create", "--dbname", defaults.DBName, "--no-privileges"}
+	restoreCmd := []string{"|", "pg_restore", "--username", defaults.Username, "--create", "--dbname", defaults.DBName,
+		"--no-privileges", "--no-owner"}
 
 	if d.Restore.ForceInit {
 		restoreCmd = append(restoreCmd, "--clean", "--if-exists")

pkg/retrieval/engine/postgres/logical/restore.go

@@ -282,14 +282,12 @@ func (r *RestoreJob) retrieveDataStateAt(ctx context.Context, contID string) (st
 }
 
 func (r *RestoreJob) buildLogicalRestoreCommand() []string {
-	restoreCmd := []string{"pg_restore", "--username", defaults.Username, "--dbname", defaults.DBName, "--create", "--no-privileges"}
+	restoreCmd := []string{"pg_restore", "--username", defaults.Username, "--dbname", defaults.DBName, "--create",
+		"--no-privileges", "--no-owner"}
 
 	if r.ForceInit {
 		restoreCmd = append(restoreCmd, "--clean", "--if-exists")
 	}
-	//else {
-	//	restoreCmd = append(restoreCmd)
-	//}
 
 	restoreCmd = append(restoreCmd, "--jobs", strconv.Itoa(r.ParallelJobs))
 

pkg/retrieval/engine/postgres/logical/restore_test.go

@@ -24,14 +24,14 @@ func TestRestoreCommandBuilding(t *testing.T) {
 				ForceInit:    false,
 				DumpFile:     "/tmp/db.dump",
 			},
-			Command: []string{"pg_restore", "--username", "postgres", "--dbname", "postgres", "--create", "--no-privileges", "--jobs", "1", "/tmp/db.dump"},
+			Command: []string{"pg_restore", "--username", "postgres", "--dbname", "postgres", "--create", "--no-privileges", "--no-owner", "--jobs", "1", "/tmp/db.dump"},
 		},
 		{
 			CopyOptions: RestoreOptions{
 				ParallelJobs: 4,
 				ForceInit:    true,
 			},
-			Command: []string{"pg_restore", "--username", "postgres", "--dbname", "postgres", "--create", "--no-privileges", "--clean", "--if-exists", "--jobs", "4", ""},
+			Command: []string{"pg_restore", "--username", "postgres", "--dbname", "postgres", "--create", "--no-privileges", "--no-owner", "--clean", "--if-exists", "--jobs", "4", ""},
 		},
 		{
 			CopyOptions: RestoreOptions{
@@ -40,7 +40,7 @@ func TestRestoreCommandBuilding(t *testing.T) {
 				Partial:      Partial{Tables: []string{"test", "users"}},
 				DumpFile:     "/tmp/db.dump",
 			},
-			Command: []string{"pg_restore", "--username", "postgres", "--dbname", "postgres", "--create", "--no-privileges", "--jobs", "1", "--table", "test", "--table", "users", "/tmp/db.dump"},
+			Command: []string{"pg_restore", "--username", "postgres", "--dbname", "postgres", "--create", "--no-privileges", "--no-owner", "--jobs", "1", "--table", "test", "--table", "users", "/tmp/db.dump"},
 		},
 	}