Update synapse import documentation

The script appears to work, just needed some touchups.

Fixes #93

README.md

@@ -113,21 +113,27 @@ release though if you want to avoid building it yourself.
 2. Edit/setup `media-repo.yaml` per the install instructions above
 3. Run `bin/import_synapse`. The usage is below. 
     ```
-    Usage of ./bin/import_synapse:
+    Usage of import_synapse.exe:
       -baseUrl string
             The base URL to access your homeserver with (default "http://localhost:8008")
+      -config string
+            The path to the media repo configuration (with the database section completed) (default "media-repo.yaml")
       -dbHost string
-            The IP or hostname of the postgresql server with the synapse database (default "localhost")
+            The PostgresSQL hostname for your Synapse database (default "localhost")
       -dbName string
-            The name of the synapse database (default "synapse")
+            The name of your Synapse database (default "synapse")
       -dbPassword string
-            The password to authorize the postgres user. Can be omitted to be prompted when run
+            The password for your Synapse's PostgreSQL database. Can be omitted to be prompted when run
       -dbPort int
-            The port to access postgres on (default 5432)
+            The port for your Synapse's PostgreSQL database (default 5432)
       -dbUsername string
-            The username to access postgres with (default "synapse")
+            The username for your Synapse's PostgreSQL database (default "synapse")
+      -migrations string
+            The absolute path the media repo's migrations folder (default "./migrations")
       -serverName string
             The name of your homeserver (eg: matrix.org) (default "localhost")
+      -workers int
+            The number of workers to use when downloading media. Using multiple workers risks deduplication not working as efficiently. (default 1)
     ```
     Assuming the media repository, postgres database, and synapse are all on the same host, the command to run would look something like: `bin/import_synapse -serverName myserver.com -dbUsername my_database_user -dbName synapse`
 4. Wait for the import to complete. The script will automatically deduplicate media.

cmd/import_synapse/main.go

@@ -17,10 +17,9 @@ import (
 	"github.com/turt2live/matrix-media-repo/common/config"
 	"github.com/turt2live/matrix-media-repo/common/logging"
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/common/runtime"
 	"github.com/turt2live/matrix-media-repo/controllers/upload_controller"
 	"github.com/turt2live/matrix-media-repo/storage"
-	"github.com/turt2live/matrix-media-repo/storage/datastore"
-	"github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3"
 	"github.com/turt2live/matrix-media-repo/synapse"
 )
 
@@ -31,15 +30,15 @@ type fetchRequest struct {
 }
 
 func main() {
-	postgresHost := flag.String("dbHost", "localhost", "The IP or hostname of the postgresql server with the synapse database")
-	postgresPort := flag.Int("dbPort", 5432, "The port to access postgres on")
-	postgresUsername := flag.String("dbUsername", "synapse", "The username to access postgres with")
-	postgresPassword := flag.String("dbPassword", "", "The password to authorize the postgres user. Can be omitted to be prompted when run")
-	postgresDatabase := flag.String("dbName", "synapse", "The name of the synapse database")
+	postgresHost := flag.String("dbHost", "localhost", "The PostgresSQL hostname for your Synapse database")
+	postgresPort := flag.Int("dbPort", 5432, "The port for your Synapse's PostgreSQL database")
+	postgresUsername := flag.String("dbUsername", "synapse", "The username for your Synapse's PostgreSQL database")
+	postgresPassword := flag.String("dbPassword", "", "The password for your Synapse's PostgreSQL database. Can be omitted to be prompted when run")
+	postgresDatabase := flag.String("dbName", "synapse", "The name of your Synapse database")
 	baseUrl := flag.String("baseUrl", "http://localhost:8008", "The base URL to access your homeserver with")
 	serverName := flag.String("serverName", "localhost", "The name of your homeserver (eg: matrix.org)")
-	configPath := flag.String("config", "media-repo.yaml", "The path to the configuration")
-	migrationsPath := flag.String("migrations", "./migrations", "The absolute path the migrations folder")
+	configPath := flag.String("config", "media-repo.yaml", "The path to the media repo configuration (with the database section completed)")
+	migrationsPath := flag.String("migrations", "./migrations", "The absolute path the media repo's migrations folder")
 	numWorkers := flag.Int("workers", 1, "The number of workers to use when downloading media. Using multiple workers risks deduplication not working as efficiently.")
 	flag.Parse()
 
@@ -63,60 +62,8 @@ func main() {
 		panic(err)
 	}
 
-	logrus.Info("Preparing database...")
-	mediaStore := storage.GetDatabase().GetMediaStore(rcontext.Initial())
-
-	logrus.Info("Initializing datastores...")
-	enabledDatastores := 0
-	for _, ds := range config.Get().DataStores {
-		if !ds.Enabled {
-			continue
-		}
-
-		enabledDatastores++
-		uri := datastore.GetUriForDatastore(ds)
-
-		_, err := storage.GetOrCreateDatastoreOfType(rcontext.Initial(), ds.Type, uri)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	// Print all the known datastores at startup. Doubles as a way to initialize the database.
-	datastores, err := mediaStore.GetAllDatastores()
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	logrus.Info("Datastores:")
-	for _, ds := range datastores {
-		logrus.Info(fmt.Sprintf("\t%s (%s): %s", ds.Type, ds.DatastoreId, ds.Uri))
-
-		if ds.Type == "s3" {
-			conf, err := datastore.GetDatastoreConfig(ds)
-			if err != nil {
-				continue
-			}
-
-			s3, err := ds_s3.GetOrCreateS3Datastore(ds.DatastoreId, conf)
-			if err != nil {
-				continue
-			}
-
-			err = s3.EnsureBucketExists()
-			if err != nil {
-				logrus.Warn("\t\tBucket does not exist!")
-			}
-
-			err = s3.EnsureTempPathExists()
-			if err != nil {
-				logrus.Warn("\t\tTemporary path does not exist!")
-			}
-		}
-	}
-
-	if len(config.Get().Uploads.StoragePaths) > 0 {
-		logrus.Warn("You are using `storagePaths` in your configuration - in a future update, this will be removed. Please use datastores instead (see sample config).")
-	}
+	logrus.Info("Starting up...")
+	runtime.RunStartupSequence()
 
 	logrus.Info("Setting up for importing...")
 
@@ -189,14 +136,14 @@ func fetchMedia(req interface{}) interface{} {
 		logrus.Error(err.Error())
 		return nil
 	}
+	defer body.Close()
 
 	_, err = upload_controller.StoreDirect(body, -1, record.ContentType, record.UploadName, record.UserId, payload.serverName, record.MediaId, common.KindLocalMedia, ctx)
 	if err != nil {
 		logrus.Error(err.Error())
 		return nil
 	}
 
-	body.Close()
 	return nil
 }
 

cmd/media_repo/inits.go

@@ -4,13 +4,10 @@ import (
 	"fmt"
 
 	"github.com/sirupsen/logrus"
-	"github.com/turt2live/matrix-media-repo/common"
-	"github.com/turt2live/matrix-media-repo/common/config"
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/controllers/maintenance_controller"
 	"github.com/turt2live/matrix-media-repo/storage"
 	"github.com/turt2live/matrix-media-repo/storage/datastore"
-	"github.com/turt2live/matrix-media-repo/storage/datastore/ds_s3"
 )
 
 func scanAndStartUnfinishedTasks() error {
@@ -63,80 +60,3 @@ func scanAndStartUnfinishedTasks() error {
 	return nil
 }
 
-func loadDatabase() {
-	logrus.Info("Preparing database...")
-	storage.GetDatabase()
-}
-
-func loadDatastores() {
-	if len(config.Get().Uploads.StoragePaths) > 0 {
-		logrus.Warn("storagePaths usage is deprecated - please use datastores instead")
-		for _, p := range config.Get().Uploads.StoragePaths {
-			ctx := rcontext.Initial().LogWithFields(logrus.Fields{"path": p})
-			ds, err := storage.GetOrCreateDatastoreOfType(ctx, "file", p)
-			if err != nil {
-				logrus.Fatal(err)
-			}
-
-			fakeConfig := config.DatastoreConfig{
-				Type:       "file",
-				Enabled:    true,
-				MediaKinds: common.AllKinds,
-				Options:    map[string]string{"path": ds.Uri},
-			}
-			config.Get().DataStores = append(config.Get().DataStores, fakeConfig)
-		}
-	}
-
-	mediaStore := storage.GetDatabase().GetMediaStore(rcontext.Initial())
-
-	logrus.Info("Initializing datastores...")
-	for _, ds := range config.UniqueDatastores() {
-		if !ds.Enabled {
-			continue
-		}
-
-		uri := datastore.GetUriForDatastore(ds)
-
-		_, err := storage.GetOrCreateDatastoreOfType(rcontext.Initial(), ds.Type, uri)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-	}
-
-	// Print all the known datastores at startup. Doubles as a way to initialize the database.
-	datastores, err := mediaStore.GetAllDatastores()
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	logrus.Info("Datastores:")
-	for _, ds := range datastores {
-		logrus.Info(fmt.Sprintf("\t%s (%s): %s", ds.Type, ds.DatastoreId, ds.Uri))
-
-		if ds.Type == "s3" {
-			conf, err := datastore.GetDatastoreConfig(ds)
-			if err != nil {
-				continue
-			}
-
-			s3, err := ds_s3.GetOrCreateS3Datastore(ds.DatastoreId, conf)
-			if err != nil {
-				continue
-			}
-
-			err = s3.EnsureBucketExists()
-			if err != nil {
-				logrus.Warn("\t\tBucket does not exist!")
-			}
-
-			err = s3.EnsureTempPathExists()
-			if err != nil {
-				logrus.Warn("\t\tTemporary path does not exist!")
-			}
-		}
-	}
-
-	if len(config.Get().Uploads.StoragePaths) > 0 {
-		logrus.Warn("You are using `storagePaths` in your configuration - in a future update, this will be removed. Please use datastores instead (see sample config).")
-	}
-}

cmd/media_repo/main.go

@@ -2,31 +2,19 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"os"
 	"os/signal"
 
 	"github.com/sirupsen/logrus"
 	"github.com/turt2live/matrix-media-repo/api/webserver"
 	"github.com/turt2live/matrix-media-repo/common/config"
 	"github.com/turt2live/matrix-media-repo/common/logging"
+	"github.com/turt2live/matrix-media-repo/common/runtime"
 	"github.com/turt2live/matrix-media-repo/common/version"
 	"github.com/turt2live/matrix-media-repo/metrics"
 	"github.com/turt2live/matrix-media-repo/tasks"
 )
 
-func printVersion(usingLogger bool) {
-	version.SetDefaults()
-
-	if usingLogger {
-		logrus.Info("Version: " + version.Version)
-		logrus.Info("Commit: " + version.GitCommit)
-	} else {
-		fmt.Println("Version: " + version.Version)
-		fmt.Println("Commit: " + version.GitCommit)
-	}
-}
-
 func main() {
 	configPath := flag.String("config", "media-repo.yaml", "The path to the configuration")
 	migrationsPath := flag.String("migrations", "./migrations", "The absolute path for the migrations folder")
@@ -35,7 +23,7 @@ func main() {
 	flag.Parse()
 
 	if *versionFlag {
-		printVersion(false)
+		version.Print(false)
 		return // exit 0
 	}
 
@@ -55,11 +43,7 @@ func main() {
 	}
 
 	logrus.Info("Starting up...")
-	printVersion(true)
-
-	config.PrintDomainInfo()
-	loadDatabase()
-	loadDatastores()
+	runtime.RunStartupSequence()
 
 	logrus.Info("Checking background tasks...")
 	err = scanAndStartUnfinishedTasks()

cmd/media_repo/reloads.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/turt2live/matrix-media-repo/api/webserver"
 	"github.com/turt2live/matrix-media-repo/common/globals"
+	"github.com/turt2live/matrix-media-repo/common/runtime"
 	"github.com/turt2live/matrix-media-repo/metrics"
 	"github.com/turt2live/matrix-media-repo/storage"
 	"github.com/turt2live/matrix-media-repo/tasks"
@@ -57,7 +58,7 @@ func reloadDatabaseOnChan(reloadChan chan bool) {
 			shouldReload := <-reloadChan
 			if shouldReload {
 				storage.ReloadDatabase()
-				loadDatabase()
+				runtime.LoadDatabase()
 				globals.DatastoresReloadChan <- true
 			} else {
 				return // received stop
@@ -71,7 +72,7 @@ func reloadDatastoresOnChan(reloadChan chan bool) {
 		for {
 			shouldReload := <-reloadChan
 			if shouldReload {
-				loadDatastores()
+				runtime.LoadDatastores()
 			} else {
 				return // received stop
 			}