Merge branch '152-snapshot-customization' into 'master'

fix: allow to use custom Docker images for data promotion, add preprocessing runner (#152)

See merge request postgres-ai/database-lab!167

Author: agneum

configs/config.example.physical_generic.yml

@@ -153,6 +153,9 @@ retrieval:
         # Promote PGDATA after data fetching.
         promote: true
 
+        # Custom Docker image for data promotion.
+        dockerImage: "postgresai/sync-instance:12"
+
         # It is possible to define a pre-precessing script. For example, "/tmp/scripts/custom.sh".
         # Default: empty string (no pre-processing defined).
         # This can be used for scrubbing eliminating PII data, to define data masking, etc.

configs/config.example.physical_walg.yml

@@ -142,6 +142,9 @@ retrieval:
         # Promote PGDATA after data fetching.
         promote: true
 
+        # Custom Docker image for data promotion.
+        dockerImage: "postgresai/sync-instance:12"
+
         # It is possible to define a pre-precessing script. For example, "/tmp/scripts/custom.sh".
         # Default: empty string (no pre-processing defined).
         # This can be used for scrubbing eliminating PII data, to define data masking, etc.

pkg/retrieval/engine/postgres/snapshot/physical.go

@@ -70,6 +70,7 @@ type PhysicalInitial struct {
 // PhysicalOptions describes options for a physical initialization job.
 type PhysicalOptions struct {
 	Promote             bool              `yaml:"promote"`
+	DockerImage         string            `yaml:"dockerImage"`
 	PreprocessingScript string            `yaml:"preprocessingScript"`
 	Configs             map[string]string `yaml:"configs"`
 	Scheduler           *Scheduler        `yaml:"scheduler"`
@@ -329,7 +330,10 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string)
 		return errors.Wrap(err, "failed to build container host config")
 	}
 
-	promoteImage := fmt.Sprintf("postgresai/sync-instance:%s", pgVersion)
+	promoteImage := p.options.DockerImage
+	if promoteImage == "" {
+		promoteImage = fmt.Sprintf("postgresai/sync-instance:%s", pgVersion)
+	}
 
 	if err := tools.PullImage(ctx, p.dockerClient, promoteImage); err != nil {
 		return errors.Wrap(err, "failed to scan image pulling response")

pkg/retrieval/engine/postgres/snapshot/snapshot.go

@@ -8,13 +8,13 @@ package snapshot
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"strings"
 
 	"github.com/pkg/errors"
 
 	"gitlab.com/postgres-ai/database-lab/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
+	"gitlab.com/postgres-ai/database-lab/pkg/services/provision/runners"
 )
 
 func extractDataStateAt(dbMarker *dbmarker.Marker) string {
@@ -31,12 +31,12 @@ func extractDataStateAt(dbMarker *dbmarker.Marker) string {
 }
 
 func runPreprocessingScript(preprocessingScript string) error {
-	commandOutput, err := exec.Command("bash", preprocessingScript).Output()
+	commandOutput, err := runners.NewLocalRunner(false).Run(preprocessingScript)
 	if err != nil {
 		return errors.Wrap(err, "failed to run custom script")
 	}
 
-	log.Msg(string(commandOutput))
+	log.Msg(commandOutput)
 
 	return nil
 }