Merge branch '134-reload-configuration' into 'master'

feat: reload configuration without downtime (#134):
* reload Database Lab configuration without downtime
* remove legacy options
* highlight configuration errors
* reload snapshot and retention schedulers

Closes #134

See merge request postgres-ai/database-lab!200

Author: agneum

cmd/database-lab/main.go

@@ -11,11 +11,12 @@
 package main
 
 import (
-	"bytes"
 	"context"
+	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/docker/docker/client"
-	"github.com/jessevdk/go-flags"
 	"github.com/pkg/errors"
 	"github.com/rs/xid"
 
@@ -31,57 +32,18 @@ import (
 	"gitlab.com/postgres-ai/database-lab/version"
 )
 
-var opts struct {
-	VerificationToken string `short:"t" long:"token" description:"API verification token" env:"VERIFICATION_TOKEN"`
-
-	MountDir      string `long:"mount-dir" description:"clones data mount directory" env:"MOUNT_DIR"`
-	UnixSocketDir string `long:"sockets-dir" description:"unix sockets directory for secure connection to clones" env:"UNIX_SOCKET_DIR"`
-	DockerImage   string `long:"docker-image" description:"clones Docker image" env:"DOCKER_IMAGE"`
-
-	ShowHelp func() error `long:"help" description:"Show this help message"`
-}
-
 func main() {
 	log.Msg("Database Lab version: ", version.GetVersion())
 
-	// Load CLI options.
-	if _, err := parseArgs(); err != nil {
-		if flags.WroteHelp(err) {
-			return
-		}
+	instanceID := xid.New().String()
 
-		log.Fatal("Args parse error:", err)
-	}
+	log.Msg("Database Lab Instance ID:", instanceID)
 
-	cfg, err := config.LoadConfig("config.yml")
+	cfg, err := loadConfiguration(instanceID)
 	if err != nil {
 		log.Fatalf(errors.WithMessage(err, "failed to parse config"))
 	}
 
-	log.DEBUG = cfg.Global.Debug
-	log.Dbg("Config loaded", cfg)
-
-	// TODO(anatoly): Annotate envs in configs. Use different lib for flags/configs?
-	if len(opts.MountDir) > 0 {
-		cfg.Provision.Options.ClonesMountDir = opts.MountDir
-	}
-
-	if len(opts.UnixSocketDir) > 0 {
-		cfg.Provision.Options.UnixSocketDir = opts.UnixSocketDir
-	}
-
-	if len(opts.DockerImage) > 0 {
-		cfg.Provision.Options.DockerImage = opts.DockerImage
-	}
-
-	if cfg.Provision.Options.ClonesMountDir != "" {
-		cfg.Global.ClonesMountDir = cfg.Provision.Options.ClonesMountDir
-	}
-
-	cfg.Global.InstanceID = xid.New().String()
-
-	log.Msg("Database Lab Instance ID", cfg.Global.InstanceID)
-
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -121,40 +83,73 @@ func main() {
 		log.Fatalf(errors.WithMessage(err, "failed to create a new platform service"))
 	}
 
-	if len(opts.VerificationToken) > 0 {
-		cfg.Server.VerificationToken = opts.VerificationToken
-	}
-
 	observerCfg := &observer.Config{
 		CloneDir:   cfg.Provision.Options.ClonesMountDir,
 		DataSubDir: cfg.Global.DataSubDir,
 		SocketDir:  cfg.Provision.Options.UnixSocketDir,
 	}
 
-	// Start the Database Lab.
 	server := srv.NewServer(&cfg.Server, observerCfg, cloningSvc, platformSvc, dockerCLI)
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP)
+
+	go func() {
+		for range c {
+			log.Msg("Reloading configuration")
+
+			if err := reloadConfig(instanceID, provisionSvc, retrievalSvc, cloningSvc, platformSvc, server); err != nil {
+				log.Err("Failed to reload configuration", err)
+			}
+
+			log.Msg("Configuration has been reloaded")
+		}
+	}()
+
+	// Start the Database Lab.
 	if err = server.Run(); err != nil {
 		log.Fatalf(err)
 	}
 }
 
-func parseArgs() ([]string, error) {
-	var parser = flags.NewParser(&opts, flags.Default & ^flags.HelpFlag)
+func loadConfiguration(instanceID string) (*config.Config, error) {
+	cfg, err := config.LoadConfig("config.yml")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse config")
+	}
 
-	// jessevdk/go-flags lib doesn't allow to use short flag -h because
-	// it's binded to usage help. We need to hack it a bit to use -h
-	// for as a hostname option.
-	// See https://github.com/jessevdk/go-flags/issues/240
-	opts.ShowHelp = func() error {
-		var b bytes.Buffer
+	log.DEBUG = cfg.Global.Debug
+	log.Dbg("Config loaded", cfg)
 
-		parser.WriteHelp(&b)
+	if cfg.Provision.Options.ClonesMountDir != "" {
+		cfg.Global.ClonesMountDir = cfg.Provision.Options.ClonesMountDir
+	}
 
-		return &flags.Error{
-			Type:    flags.ErrHelp,
-			Message: b.String(),
-		}
+	cfg.Global.InstanceID = instanceID
+
+	return cfg, nil
+}
+
+func reloadConfig(instanceID string, provisionSvc provision.Provision, retrievalSvc *retrieval.Retrieval, cloningSvc cloning.Cloning,
+	platformSvc *platform.Service, server *srv.Server) error {
+	cfg, err := loadConfiguration(instanceID)
+	if err != nil {
+		return err
+	}
+
+	if err := provision.IsValidConfig(cfg.Provision); err != nil {
+		return err
 	}
 
-	return parser.Parse()
+	if err := retrieval.IsValidConfig(cfg); err != nil {
+		return err
+	}
+
+	provisionSvc.Reload(cfg.Provision)
+	retrievalSvc.Reload(cfg)
+	cloningSvc.Reload(cfg.Cloning)
+	platformSvc.Reload(cfg.Platform)
+	server.Reload(cfg.Server)
+
+	return nil
 }

pkg/config/config.go

@@ -6,7 +6,6 @@
 package config
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os/user"
 	"path"
@@ -87,7 +86,7 @@ func LoadConfig(name string) (*Config, error) {
 
 	cfg := &Config{}
 	if err := yaml.Unmarshal(b, cfg); err != nil {
-		return nil, fmt.Errorf("error parsing %s config", name)
+		return nil, errors.WithMessagef(err, "error parsing %s config", name)
 	}
 
 	if err := cfg.setUpProvisionParams(); err != nil {

pkg/retrieval/components/components.go

@@ -22,6 +22,9 @@ type JobRunner interface {
 	// Name returns a job name.
 	Name() string
 
+	// Reload reloads job configuration.
+	Reload(cfg map[string]interface{}) error
+
 	// Run starts a job.
 	Run(ctx context.Context) error
 }

pkg/retrieval/engine/postgres/logical/dump.go

@@ -120,16 +120,10 @@ func NewDumpJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Gl
 		},
 	}
 
-	if err := options.Unmarshal(cfg.Options, &dumpJob.DumpOptions); err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal configuration options")
+	if err := dumpJob.Reload(cfg.Options); err != nil {
+		return nil, errors.Wrap(err, "failed to load job config")
 	}
 
-	if err := dumpJob.validate(); err != nil {
-		return nil, errors.Wrap(err, "invalid logical dump job")
-	}
-
-	dumpJob.setDefaults()
-
 	if err := dumpJob.setupDumper(); err != nil {
 		return nil, errors.Wrap(err, "failed to set up a dump helper")
 	}
@@ -195,6 +189,21 @@ func (d *DumpJob) Name() string {
 	return d.name
 }
 
+// Reload reloads job configuration.
+func (d *DumpJob) Reload(cfg map[string]interface{}) (err error) {
+	if err := options.Unmarshal(cfg, &d.DumpOptions); err != nil {
+		return errors.Wrap(err, "failed to unmarshal configuration options")
+	}
+
+	if err := d.validate(); err != nil {
+		return errors.Wrap(err, "invalid logical dump job")
+	}
+
+	d.setDefaults()
+
+	return nil
+}
+
 // Run starts the job.
 func (d *DumpJob) Run(ctx context.Context) (err error) {
 	log.Msg(fmt.Sprintf("Run job: %s. Options: %v", d.Name(), d.DumpOptions))

pkg/retrieval/engine/postgres/logical/restore.go

@@ -73,7 +73,7 @@ func NewJob(cfg config.JobConfig, docker *client.Client, globalCfg *dblabCfg.Glo
 		dbMark:       &dbmarker.Config{DataType: dbmarker.LogicalDataType},
 	}
 
-	if err := options.Unmarshal(cfg.Options, &restoreJob.RestoreOptions); err != nil {
+	if err := restoreJob.Reload(cfg.Options); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal configuration options")
 	}
 
@@ -98,6 +98,17 @@ func (r *RestoreJob) Name() string {
 	return r.name
 }
 
+// Reload reloads job configuration.
+func (r *RestoreJob) Reload(cfg map[string]interface{}) (err error) {
+	if err := options.Unmarshal(cfg, &r.RestoreOptions); err != nil {
+		return errors.Wrap(err, "failed to unmarshal configuration options")
+	}
+
+	r.setDefaults()
+
+	return nil
+}
+
 // Run starts the job.
 func (r *RestoreJob) Run(ctx context.Context) (err error) {
 	log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.RestoreOptions))

pkg/retrieval/engine/postgres/physical/physical.go

@@ -95,7 +95,7 @@ func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global
 		dbMarker:     marker,
 	}
 
-	if err := options.Unmarshal(cfg.Options, &physicalJob.CopyOptions); err != nil {
+	if err := physicalJob.Reload(cfg.Options); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal configuration options")
 	}
 
@@ -131,6 +131,11 @@ func (r *RestoreJob) Name() string {
 	return r.name
 }
 
+// Reload reloads job configuration.
+func (r *RestoreJob) Reload(cfg map[string]interface{}) (err error) {
+	return options.Unmarshal(cfg, &r.CopyOptions)
+}
+
 // Run starts the job.
 func (r *RestoreJob) Run(ctx context.Context) (err error) {
 	log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.CopyOptions))

pkg/retrieval/engine/postgres/snapshot/logical.go

@@ -51,7 +51,7 @@ func NewLogicalInitialJob(cfg config.JobConfig, cloneManager thinclones.Manager,
 		dbMarker:     marker,
 	}
 
-	if err := options.Unmarshal(cfg.Options, &li.options); err != nil {
+	if err := li.Reload(cfg.Options); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal configuration options")
 	}
 
@@ -63,6 +63,11 @@ func (s *LogicalInitial) Name() string {
 	return s.name
 }
 
+// Reload reloads job configuration.
+func (s *LogicalInitial) Reload(cfg map[string]interface{}) (err error) {
+	return options.Unmarshal(cfg, &s.options)
+}
+
 // Run starts the job.
 func (s *LogicalInitial) Run(_ context.Context) error {
 	if s.options.PreprocessingScript != "" {