Refactor sync into separate action functions (#268)

* sync-actions

* dns rewrites and filters

* continue on filter error

* servides

* client settings

* dns

* dhcp

* remove deprecated env var

* fix client tests

* tests

* copy replica config

* map continue on error

* map env var with underscore

Makefile

@@ -104,6 +104,9 @@ start-replica:
 	docker run --pull always --name adguardhome-replica -p 9091:3000 --rm adguard/adguardhome:latest
 #	docker run --pull always --name adguardhome-replica -p 9090:80 -p 9091:3000 --rm adguard/adguardhome:v0.107.13
 
+copy-replica-config:
+	docker cp adguardhome-replica:/opt/adguardhome/conf/AdGuardHome.yaml tmp/AdGuardHome.yaml
+
 start-replica2:
 	docker run --pull always --name adguardhome-replica2 -p 9093:3000 --rm adguard/adguardhome:latest
 #	docker run --pull always --name adguardhome-replica -p 9090:80 -p 9091:3000 --rm adguard/adguardhome:v0.107.13

README.md

@@ -163,6 +163,7 @@ services:
       # REPLICA1_DHCPSERVERENABLED: true/false (optional) enables/disables the dhcp server on the replica
       CRON: "*/10 * * * *" # run every 10 minutes
       RUNONSTART: true
+      # CONTINUEONERROR: false # If enabled, the synchronisation task will not fail on single errors, but will log the errors and continue
 
       # Configure the sync API server, disabled if api port is 0
       API_PORT: 8080
@@ -195,6 +196,9 @@ cron: "*/10 * * * *"
 # runs the synchronisation on startup
 runOnStart: true
 
+# If enabled, the synchronisation task will not fail on single errors, but will log the errors and continue
+continueOnError: false
+
 origin:
   # url of the origin instance
   url: https://192.168.1.2:3000

cmd/root.go

@@ -12,13 +12,14 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"go.uber.org/zap"
 )
 
 const (
 	configCron            = "cron"
 	configRunOnStart      = "runOnStart"
-	configPrintConfigOnly = "printConfigOnly"
+	configPrintConfigOnly = "PRINT_CONFIG_ONLY"
+	configContinueOnError = "CONTINUE_ON_ERROR"
+	configLogLevel        = "LOG_LEVEL"
 
 	configAPIPort     = "api.port"
 	configAPIUsername = "api.username"
@@ -62,10 +63,8 @@ const (
 	envReplicasInsecureSkipVerifyFormat = "REPLICA%s_INSECURESKIPVERIFY"
 	envReplicasAutoSetup                = "REPLICA%s_AUTOSETUP"
 	envReplicasInterfaceName            = "REPLICA%s_INTERFACENAME"
-	// Deprecated: use envReplicasInterfaceName instead
-	envReplicasInterfaceNameDeprecated = "REPLICA%s_INTERFACWENAME"
-	envDHCPServerEnabled               = "REPLICA%s_DHCPSERVERENABLED"
-	envWebURL                          = "REPLICA%s_WEBURL"
+	envDHCPServerEnabled                = "REPLICA%s_DHCPSERVERENABLED"
+	envWebURL                           = "REPLICA%s_WEBURL"
 )
 
 var (
@@ -133,7 +132,7 @@ func initConfig() {
 	}
 }
 
-func getConfig(logger *zap.SugaredLogger) (*types.Config, error) {
+func getConfig() (*types.Config, error) {
 	cfg := &types.Config{}
 	if err := viper.Unmarshal(cfg); err != nil {
 		return nil, err
@@ -145,14 +144,14 @@ func getConfig(logger *zap.SugaredLogger) (*types.Config, error) {
 	}
 
 	if len(cfg.Replicas) == 0 {
-		cfg.Replicas = append(cfg.Replicas, collectEnvReplicas(logger)...)
+		cfg.Replicas = append(cfg.Replicas, collectEnvReplicas()...)
 	}
 
 	return cfg, nil
 }
 
 // Manually collect replicas from env.
-func collectEnvReplicas(logger *zap.SugaredLogger) []types.AdGuardInstance {
+func collectEnvReplicas() []types.AdGuardInstance {
 	var replicas []types.AdGuardInstance
 	for _, v := range os.Environ() {
 		if envReplicasURLPattern.MatchString(v) {
@@ -169,14 +168,6 @@ func collectEnvReplicas(logger *zap.SugaredLogger) []types.AdGuardInstance {
 				InterfaceName:      os.Getenv(fmt.Sprintf(envReplicasInterfaceName, sm[1])),
 			}
 
-			if re.InterfaceName == "" {
-				if in, ok := os.LookupEnv(fmt.Sprintf(envReplicasInterfaceNameDeprecated, sm[1])); ok {
-					logger.
-						With("correct", envReplicasInterfaceName, "deprecated", envReplicasInterfaceNameDeprecated).
-						Warn("Deprecated env variable is used, please use the correct one")
-					re.InterfaceName = in
-				}
-			}
 			if dhcpEnabled, ok := os.LookupEnv(fmt.Sprintf(envDHCPServerEnabled, sm[1])); ok {
 				if strings.EqualFold(dhcpEnabled, "true") {
 					re.DHCPServerEnabled = boolPtr(true)

cmd/root_test.go

@@ -4,11 +4,9 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/bakito/adguardhome-sync/pkg/log"
 	"github.com/bakito/adguardhome-sync/pkg/types"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"go.uber.org/zap"
 )
 
 var envVars = []string{
@@ -24,14 +22,11 @@ var envVars = []string{
 	"FEATURES_DNS_ACCESSLISTS",
 	"FEATURES_DNS_REWRITES",
 	"REPLICA1_INTERFACENAME",
-	"REPLICA1_INTERFACWENAME",
 	"REPLICA1_DHCPSERVERENABLED",
 }
 
 var _ = Describe("Run", func() {
-	var logger *zap.SugaredLogger
 	BeforeEach(func() {
-		logger = log.GetLogger("root")
 		for _, envVar := range envVars {
 			Ω(os.Unsetenv(envVar)).ShouldNot(HaveOccurred())
 		}
@@ -44,38 +39,23 @@ var _ = Describe("Run", func() {
 	})
 	Context("getConfig", func() {
 		It("features should be true by default", func() {
-			cfg, err := getConfig(logger)
+			cfg, err := getConfig()
 			Ω(err).ShouldNot(HaveOccurred())
 			verifyFeatures(cfg, true)
 		})
 		It("features should be false", func() {
 			for _, envVar := range envVars {
 				Ω(os.Setenv(envVar, "false")).ShouldNot(HaveOccurred())
 			}
-			cfg, err := getConfig(logger)
+			cfg, err := getConfig()
 			Ω(err).ShouldNot(HaveOccurred())
 			verifyFeatures(cfg, false)
 		})
 		Context("interface name", func() {
 			It("should set interface name of replica 1", func() {
 				Ω(os.Setenv("REPLICA1_URL", "https://foo.bar")).ShouldNot(HaveOccurred())
 				Ω(os.Setenv(fmt.Sprintf(envReplicasInterfaceName, "1"), "eth0")).ShouldNot(HaveOccurred())
-				cfg, err := getConfig(logger)
-				Ω(err).ShouldNot(HaveOccurred())
-				Ω(cfg.Replicas[0].InterfaceName).Should(Equal("eth0"))
-			})
-			It("should set interface name of replica 1 from deprecated env", func() {
-				Ω(os.Setenv("REPLICA1_URL", "https://foo.bar")).ShouldNot(HaveOccurred())
-				Ω(os.Setenv(fmt.Sprintf(envReplicasInterfaceNameDeprecated, "1"), "eth0")).ShouldNot(HaveOccurred())
-				cfg, err := getConfig(logger)
-				Ω(err).ShouldNot(HaveOccurred())
-				Ω(cfg.Replicas[0].InterfaceName).Should(Equal("eth0"))
-			})
-			It("deprecated should not overwrite the correct", func() {
-				Ω(os.Setenv("REPLICA1_URL", "https://foo.bar")).ShouldNot(HaveOccurred())
-				Ω(os.Setenv(fmt.Sprintf(envReplicasInterfaceNameDeprecated, "1"), "eth1")).ShouldNot(HaveOccurred())
-				Ω(os.Setenv(fmt.Sprintf(envReplicasInterfaceName, "1"), "eth0")).ShouldNot(HaveOccurred())
-				cfg, err := getConfig(logger)
+				cfg, err := getConfig()
 				Ω(err).ShouldNot(HaveOccurred())
 				Ω(cfg.Replicas[0].InterfaceName).Should(Equal("eth0"))
 			})
@@ -84,15 +64,15 @@ var _ = Describe("Run", func() {
 			It("should enable the dhcp server of replica 1", func() {
 				Ω(os.Setenv("REPLICA1_URL", "https://foo.bar")).ShouldNot(HaveOccurred())
 				Ω(os.Setenv(fmt.Sprintf(envDHCPServerEnabled, "1"), "true")).ShouldNot(HaveOccurred())
-				cfg, err := getConfig(logger)
+				cfg, err := getConfig()
 				Ω(err).ShouldNot(HaveOccurred())
 				Ω(cfg.Replicas[0].DHCPServerEnabled).ShouldNot(BeNil())
 				Ω(*cfg.Replicas[0].DHCPServerEnabled).Should(BeTrue())
 			})
 			It("should disable the dhcp server of replica 1", func() {
 				Ω(os.Setenv("REPLICA1_URL", "https://foo.bar")).ShouldNot(HaveOccurred())
 				Ω(os.Setenv(fmt.Sprintf(envDHCPServerEnabled, "1"), "false")).ShouldNot(HaveOccurred())
-				cfg, err := getConfig(logger)
+				cfg, err := getConfig()
 				Ω(err).ShouldNot(HaveOccurred())
 				Ω(cfg.Replicas[0].DHCPServerEnabled).ShouldNot(BeNil())
 				Ω(*cfg.Replicas[0].DHCPServerEnabled).Should(BeFalse())

cmd/run.go

@@ -15,7 +15,7 @@ var doCmd = &cobra.Command{
 	Long:  `Synchronizes the configuration form an origin instance to a replica`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		logger = log.GetLogger("run")
-		cfg, err := getConfig(logger)
+		cfg, err := getConfig()
 		if err != nil {
 			logger.Error(err)
 			return err
@@ -50,6 +50,13 @@ func init() {
 	doCmd.PersistentFlags().Bool("printConfigOnly", false, "Prints the configuration only and exists. "+
 		"Can be used to debug the config E.g: when having authentication issues.")
 	_ = viper.BindPFlag(configPrintConfigOnly, doCmd.PersistentFlags().Lookup("printConfigOnly"))
+	doCmd.PersistentFlags().Bool("continueOnError", false, "If enabled, the synchronisation task "+
+		"will not fail on single errors, but will log the errors and continue.")
+	_ = viper.BindPFlag(configContinueOnError, doCmd.PersistentFlags().Lookup("continueOnError"))
+
+	doCmd.PersistentFlags().String("logLevel", "info", "The log level to set. One of (debug, info, warn  and error)")
+	_ = viper.BindPFlag(configLogLevel, doCmd.PersistentFlags().Lookup("logLevel"))
+
 	doCmd.PersistentFlags().Int("api-port", 8080, "Sync API Port, the API endpoint will be started to enable remote triggering; if 0 port API is disabled.")
 	_ = viper.BindPFlag(configAPIPort, doCmd.PersistentFlags().Lookup("api-port"))
 	doCmd.PersistentFlags().String("api-username", "", "Sync API username")