fix: mounting volumes in Postgres containers depends on Database Lab process environment (#152)

Author: agneum

go.mod

@@ -6,12 +6,14 @@ require (
 	github.com/AlekSi/pointer v1.1.0
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/aws/aws-sdk-go v1.33.8
 	github.com/containerd/containerd v1.3.4 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/gorilla/mux v1.7.3
 	github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4
@@ -23,6 +25,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/xid v1.2.1
 	github.com/sergi/go-diff v1.1.0
+	github.com/shirou/gopsutil v2.20.7+incompatible
 	github.com/sirupsen/logrus v1.4.2
 	github.com/stretchr/testify v1.5.1
 	github.com/urfave/cli/v2 v2.1.1

go.sum

@@ -7,6 +7,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/aws/aws-sdk-go v1.33.8 h1:2/sOfb9oPHTRZ0lxinoaTPDcYwNa1H/SpKP4nVRBwmg=
 github.com/aws/aws-sdk-go v1.33.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -30,6 +32,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -80,6 +84,8 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shirou/gopsutil v2.20.7+incompatible h1:Ymv4OD12d6zm+2yONe39VSmp2XooJe8za7ngOLW/o/w=
+github.com/shirou/gopsutil v2.20.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=

pkg/retrieval/engine/postgres/initialize/logical/dump.go

@@ -219,17 +219,12 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
 		return errors.Wrap(err, "failed to scan pulling image response")
 	}
 
-	cont, err := d.dockerClient.ContainerCreate(ctx,
-		&container.Config{
-			Env:         d.getEnvironmentVariables(),
-			Image:       d.DockerImage,
-			Healthcheck: health.GetConfig(),
-		},
-		&container.HostConfig{
-			Mounts:      d.getMountVolumes(),
-			NetworkMode: d.getContainerNetworkMode(),
-		},
-		&network.NetworkingConfig{},
+	hostConfig, err := d.buildHostConfig()
+	if err != nil {
+		return errors.Wrap(err, "failed to build container host config")
+	}
+
+	cont, err := d.dockerClient.ContainerCreate(ctx, d.buildContainerConfig(), hostConfig, &network.NetworkingConfig{},
 		d.dumpContainerName(),
 	)
 	if err != nil {
@@ -368,15 +363,30 @@ func (d *DumpJob) getEnvironmentVariables() []string {
 	return envs
 }
 
+func (d *DumpJob) buildContainerConfig() *container.Config {
+	return &container.Config{
+		Env:         d.getEnvironmentVariables(),
+		Image:       d.DockerImage,
+		Healthcheck: health.GetConfig(),
+	}
+}
+
+func (d *DumpJob) buildHostConfig() (*container.HostConfig, error) {
+	hostConfig := &container.HostConfig{
+		Mounts:      d.getMountVolumes(),
+		NetworkMode: d.getContainerNetworkMode(),
+	}
+
+	if err := tools.AddVolumesToHostConfig(hostConfig, d.globalCfg.DataDir); err != nil {
+		return nil, err
+	}
+
+	return hostConfig, nil
+}
+
 // getMountVolumes returns a list of mount volumes.
 func (d *DumpJob) getMountVolumes() []mount.Mount {
-	mounts := []mount.Mount{
-		{
-			Type:   mount.TypeBind,
-			Source: d.globalCfg.DataDir,
-			Target: d.globalCfg.DataDir,
-		},
-	}
+	mounts := d.dumper.GetMounts()
 
 	if d.DumpOptions.DumpFile != "" {
 		mounts = append(mounts, mount.Mount{
@@ -386,9 +396,6 @@ func (d *DumpJob) getMountVolumes() []mount.Mount {
 		})
 	}
 
-	// Add dump specific mounts.
-	mounts = append(mounts, d.dumper.GetMounts()...)
-
 	return mounts
 }
 

pkg/retrieval/engine/postgres/initialize/logical/logical.go

@@ -22,7 +22,6 @@ func recalculateStats(ctx context.Context, dockerClient *client.Client, contID s
 	exec, err := dockerClient.ContainerExecCreate(ctx, contID, types.ExecConfig{
 		AttachStdout: true,
 		AttachStderr: true,
-		Tty:          true,
 		Cmd:          analyzeCmd,
 	})
 

pkg/retrieval/engine/postgres/initialize/logical/restore.go

@@ -119,28 +119,14 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
 		return errors.Wrap(err, "failed to scan image pulling response")
 	}
 
+	hostConfig, err := r.buildHostConfig()
+	if err != nil {
+		return errors.Wrap(err, "failed to build container host config")
+	}
+
 	cont, err := r.dockerClient.ContainerCreate(ctx,
-		&container.Config{
-			Env: []string{
-				"PGDATA=" + r.globalCfg.DataDir,
-			},
-			Image:       r.RestoreOptions.DockerImage,
-			Healthcheck: health.GetConfig(),
-		},
-		&container.HostConfig{
-			Mounts: []mount.Mount{
-				{
-					Type:   mount.TypeBind,
-					Source: r.RestoreOptions.DumpFile,
-					Target: r.RestoreOptions.DumpFile,
-				},
-				{
-					Type:   mount.TypeBind,
-					Source: r.globalCfg.DataDir,
-					Target: r.globalCfg.DataDir,
-				},
-			},
-		},
+		r.buildContainerConfig(),
+		hostConfig,
 		&network.NetworkingConfig{},
 		r.restoreContainerName(),
 	)
@@ -208,6 +194,34 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
 	return nil
 }
 
+func (r *RestoreJob) buildContainerConfig() *container.Config {
+	return &container.Config{
+		Env: []string{
+			"PGDATA=" + r.globalCfg.DataDir,
+		},
+		Image:       r.RestoreOptions.DockerImage,
+		Healthcheck: health.GetConfig(),
+	}
+}
+
+func (r *RestoreJob) buildHostConfig() (*container.HostConfig, error) {
+	hostConfig := &container.HostConfig{
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeBind,
+				Source: r.RestoreOptions.DumpFile,
+				Target: r.RestoreOptions.DumpFile,
+			},
+		},
+	}
+
+	if err := tools.AddVolumesToHostConfig(hostConfig, r.globalCfg.DataDir); err != nil {
+		return nil, err
+	}
+
+	return hostConfig, nil
+}
+
 func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
 	dataStateAt, err := r.retrieveDataStateAt(ctx, contID)
 	if err != nil {

pkg/retrieval/engine/postgres/initialize/physical/physical.go

@@ -236,14 +236,14 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
 }
 
 func (r *RestoreJob) startContainer(ctx context.Context, containerName string) (string, error) {
+	hostConfig, err := r.buildHostConfig()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to build container host config")
+	}
+
 	syncInstance, err := r.dockerClient.ContainerCreate(ctx,
-		&container.Config{
-			Env:   r.getEnvironmentVariables(),
-			Image: r.DockerImage,
-		},
-		&container.HostConfig{
-			Mounts: r.getMountVolumes(),
-		},
+		r.buildContainerConfig(),
+		hostConfig,
 		&network.NetworkingConfig{},
 		containerName,
 	)
@@ -355,21 +355,23 @@ func (r *RestoreJob) getEnvironmentVariables() []string {
 	return envVariables
 }
 
-func (r *RestoreJob) getMountVolumes() []mount.Mount {
-	mounts := append(
-		[]mount.Mount{
-			{
-				Type:   mount.TypeBind,
-				Source: r.globalCfg.DataDir,
-				Target: r.globalCfg.DataDir,
-				BindOptions: &mount.BindOptions{
-					Propagation: mount.PropagationRShared,
-				},
-			},
-		},
-		r.restorer.GetMounts()...)
-
-	return mounts
+func (r *RestoreJob) buildContainerConfig() *container.Config {
+	return &container.Config{
+		Env:   r.getEnvironmentVariables(),
+		Image: r.DockerImage,
+	}
+}
+
+func (r *RestoreJob) buildHostConfig() (*container.HostConfig, error) {
+	hostConfig := &container.HostConfig{
+		Mounts: r.restorer.GetMounts(),
+	}
+
+	if err := tools.AddVolumesToHostConfig(hostConfig, r.globalCfg.DataDir); err != nil {
+		return nil, err
+	}
+
+	return hostConfig, nil
 }
 
 func (r *RestoreJob) markDatabaseData() error {

pkg/retrieval/engine/postgres/initialize/snapshot/physical.go

@@ -21,7 +21,6 @@ import (
 	"github.com/AlekSi/pointer"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
@@ -306,6 +305,11 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string)
 		return errors.Wrap(err, "failed to adjust recovery configuration")
 	}
 
+	hostConfig, err := p.buildHostConfig(clonePath)
+	if err != nil {
+		return errors.Wrap(err, "failed to build container host config")
+	}
+
 	promoteImage := fmt.Sprintf("postgresai/sync-instance:%s", pgVersion)
 
 	if err := tools.PullImage(ctx, p.dockerClient, promoteImage); err != nil {
@@ -314,30 +318,8 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string)
 
 	// Run promotion container.
 	cont, err := p.dockerClient.ContainerCreate(ctx,
-		&container.Config{
-			Labels: map[string]string{"label": "dblab_control"},
-			Env: []string{
-				"PGDATA=" + clonePath,
-				"POSTGRES_HOST_AUTH_METHOD=trust",
-			},
-			Image: promoteImage,
-			Healthcheck: health.GetConfig(
-				health.OptionInterval(hcPromoteInterval),
-				health.OptionRetries(hcPromoteRetries),
-			),
-		},
-		&container.HostConfig{
-			Mounts: []mount.Mount{
-				{
-					Type:   mount.TypeBind,
-					Source: clonePath,
-					Target: clonePath,
-					BindOptions: &mount.BindOptions{
-						Propagation: mount.PropagationRShared,
-					},
-				},
-			},
-		},
+		p.buildContainerConfig(clonePath, promoteImage),
+		hostConfig,
 		&network.NetworkingConfig{},
 		p.promoteContainerName(),
 	)
@@ -470,6 +452,31 @@ func (p *PhysicalInitial) adjustRecoveryConfiguration(pgVersion, clonePGDataDir
 	return nil
 }
 
+func (p *PhysicalInitial) buildContainerConfig(clonePath, promoteImage string) *container.Config {
+	return &container.Config{
+		Labels: map[string]string{"label": "dblab_control"},
+		Env: []string{
+			"PGDATA=" + clonePath,
+			"POSTGRES_HOST_AUTH_METHOD=trust",
+		},
+		Image: promoteImage,
+		Healthcheck: health.GetConfig(
+			health.OptionInterval(hcPromoteInterval),
+			health.OptionRetries(hcPromoteRetries),
+		),
+	}
+}
+
+func (p *PhysicalInitial) buildHostConfig(clonePath string) (*container.HostConfig, error) {
+	hostConfig := &container.HostConfig{}
+
+	if err := tools.AddVolumesToHostConfig(hostConfig, clonePath); err != nil {
+		return nil, err
+	}
+
+	return hostConfig, nil
+}
+
 func (p *PhysicalInitial) checkRecovery(ctx context.Context, containerID string) (string, error) {
 	checkRecoveryCmd := []string{"psql", "-U", defaults.Username, "-XAtc", "select pg_is_in_recovery()"}
 	log.Msg("Check recovery command", checkRecoveryCmd)

pkg/retrieval/engine/postgres/initialize/tools/tools.go

@@ -19,9 +19,12 @@ import (
 
 	"github.com/AlekSi/pointer"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/pkg/errors"
+	"github.com/shirou/gopsutil/host"
 
 	"gitlab.com/postgres-ai/database-lab/pkg/log"
 )
@@ -74,6 +77,28 @@ func DetectPGVersion(dataDir string) (string, error) {
 	return string(bytes.TrimSpace(version)), nil
 }
 
+// AddVolumesToHostConfig adds volumes to container host configuration depends on process environment.
+func AddVolumesToHostConfig(hostConfig *container.HostConfig, dataDir string) error {
+	hostInfo, err := host.Info()
+	if err != nil {
+		return errors.Wrap(err, "failed to get host info")
+	}
+
+	log.Dbg("Virtualization system: ", hostInfo.VirtualizationSystem)
+
+	if hostInfo.VirtualizationRole == "guest" {
+		hostConfig.VolumesFrom = []string{hostInfo.Hostname}
+	} else {
+		hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
+			Type:   mount.TypeBind,
+			Source: dataDir,
+			Target: dataDir,
+		})
+	}
+
+	return nil
+}
+
 // InspectCommandResponse inspects success of command execution.
 func InspectCommandResponse(ctx context.Context, dockerClient *client.Client, containerID, commandID string) error {
 	inspect, err := dockerClient.ContainerExecInspect(ctx, commandID)