Use real MongoDB connection for E2E tests

src/LfMerge.Core.Tests/E2E/E2ETestBase.cs

@@ -25,7 +25,7 @@ public class E2ETestBase
 		private readonly HashSet<Guid> ProjectIdsToDelete = [];
 		public SRTestEnvironment TestEnv { get; set; }
 
-		public MongoConnectionDouble _mongoConnection;
+		public IMongoConnection _mongoConnection;
 		public MongoProjectRecordFactory _recordFactory;
 
 		public E2ETestBase()
@@ -81,14 +81,14 @@ public async Task TestSetup()
 				Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox.");
 			}
 			await TestEnv.Login();
+			TestEnv.LaunchMongo();
 
 			MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion);
-			_mongoConnection = MainClass.Container.Resolve<IMongoConnection>() as MongoConnectionDouble;
-			if (_mongoConnection == null)
-				throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work.");
-			_recordFactory = MainClass.Container.Resolve<MongoProjectRecordFactory>() as MongoProjectRecordFactoryDouble;
-			if (_recordFactory == null)
-				throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work.");
+			_mongoConnection = MainClass.Container.Resolve<IMongoConnection>();
+			var _mongoConnectionDouble = _mongoConnection as MongoConnectionDouble;
+			if (_mongoConnectionDouble != null)
+				throw new AssertionException("E2E tests need a real MongoConnection, not a mock.");
+			_recordFactory = MainClass.Container.Resolve<MongoProjectRecordFactory>();
 		}
 
 		[TearDown]
@@ -97,7 +97,7 @@ public async Task TestTeardown()
 			var outcome = TestContext.CurrentContext.Result.Outcome;
 			var success = outcome == ResultState.Success || outcome == ResultState.Ignored;
 			// Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation
-			TestEnv.DeleteTempFolderDuringCleanup = success;
+			TestEnv.CleanUpTestData = success;
 			// On failure, also leave LexBox project(s) in place for post-test investigation, even though this might tend to clutter things up a little
 			if (success) {
 				foreach (var projId in ProjectIdsToDelete) {
@@ -220,7 +220,7 @@ public void SendReceiveToLexbox(LanguageForgeProject lfProject)
 
 		public (string, DateTime, DateTime) UpdateLfGloss(LanguageForgeProject lfProject, Guid entryId, string wsId, Func<string, string> textConverter)
 		{
-			var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			var lfEntry = _mongoConnection.GetLfLexEntryByGuid(lfProject, entryId);
 			Assert.That(lfEntry, Is.Not.Null);
 			var unchangedGloss = lfEntry.Senses[0].Gloss[wsId].Value;
 			lfEntry.Senses[0].Gloss["pt"].Value = textConverter(unchangedGloss);

src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs

@@ -50,7 +50,7 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins()
 			// Verify
 
 			// LF side should win conflict since its modified date was later
-			var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(lfProject, entryId);
 			Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF"));
 			// LF's modified dates should have been updated by the sync action
 			Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified));

src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs

@@ -830,7 +830,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldBeAbleToAddAndModifyParagraphsInCu
 			// Here we check two things:
 			// 1) Can we add paragraphs?
 			// 2) Can we change existing paragraphs?
-			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(lfProject, entryGuid);
 			// BsonDocument customFieldValues = GetCustomFieldValues(cache, lcmEntry, "entry");
 			BsonDocument customFieldsBson = lfEntry.CustomFields;
 			Assert.That(customFieldsBson.Contains("customField_entry_Cust_MultiPara"), Is.True,
@@ -923,7 +923,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldBeAbleToDeleteParagraphsInCustomMu
 
 			// Here we check just one thing:
 			// 1) Can we delete paragraphs?
-			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(lfProject, entryGuid);
 			// BsonDocument customFieldValues = GetCustomFieldValues(cache, lcmEntry, "entry");
 			BsonDocument customFieldsBson = lfEntry.CustomFields;
 			Assert.That(customFieldsBson.Contains("customField_entry_Cust_MultiPara"), Is.True,

src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs

@@ -234,7 +234,7 @@ public void Action_WithOneModifiedEntry_ShouldCountOneModified()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -263,7 +263,7 @@ public void Action_WithOneDeletedEntry_ShouldCountOneDeleted()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -325,7 +325,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -335,7 +335,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified()
 			_conn.UpdateMockLfLexEntry(entry);
 
 			Guid kenGuid = Guid.Parse(KenEntryGuidStr);
-			LfLexEntry kenEntry = _conn.GetLfLexEntryByGuid(kenGuid);
+			LfLexEntry kenEntry = _conn.GetLfLexEntryByGuid(lfProj, kenGuid);
 			string changedLexeme2 = "modified lexeme #2 for this test";
 			kenEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2);
 			kenEntry.AuthorInfo = new LfAuthorInfo();
@@ -362,11 +362,11 @@ public void Action_WithTwoDeletedEntries_ShouldCountTwoDeleted()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 			Guid kenGuid = Guid.Parse(KenEntryGuidStr);
-			entry = _conn.GetLfLexEntryByGuid(kenGuid);
+			entry = _conn.GetLfLexEntryByGuid(lfProj, kenGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -430,7 +430,7 @@ public void Action_WithOneModifiedEntry_ShouldNotCountThatModifiedEntryOnSecondR
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -470,7 +470,7 @@ public void Action_WithOneDeletedEntry_ShouldNotCountThatDeletedEntryOnSecondRun
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -557,7 +557,7 @@ public void Action_RunTwiceWithTheSameEntryModifiedEachTime_ShouldCountTwoModifi
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -606,7 +606,7 @@ public void Action_RunTwiceWithTheSameEntryDeletedEachTime_ShouldCountJustOneDel
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -621,7 +621,7 @@ public void Action_RunTwiceWithTheSameEntryDeletedEachTime_ShouldCountJustOneDel
 			Assert.That(LfMergeBridgeServices.FormatCommitMessageForLfMerge(_counts.Added, _counts.Modified, _counts.Deleted),
 				Is.EqualTo("Language Forge: 1 entry deleted"));
 
-			entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -689,7 +689,7 @@ public void Run_CustomMultiListRefTest(int whichSense, params string[] desiredKe
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LfSense sense = entry.Senses[whichSense];
 			SetCustomMultiOptionList(sense, "customField_senses_Cust_Multi_ListRef", desiredKeys);
 			entry.AuthorInfo = new LfAuthorInfo();

src/LfMerge.Core.Tests/SRTestEnvironment.cs

@@ -5,14 +5,17 @@
 using System.Net.Http;
 using System.Net.Http.Json;
 using System.Threading.Tasks;
+using Autofac;
 using BirdMessenger;
 using BirdMessenger.Collections;
 using GraphQL;
 using GraphQL.Client.Http;
 using GraphQL.Client.Serializer.SystemTextJson;
 using LfMerge.Core.FieldWorks;
 using LfMerge.Core.Logging;
+using LfMerge.Core.MongoConnector;
 using NUnit.Framework;
+using SIL.CommandLineProcessing;
 using SIL.TestUtilities;
 
 namespace LfMerge.Core.Tests
@@ -39,6 +42,7 @@ public class SRTestEnvironment : TestEnvironment
 		private bool AlreadyLoggedIn = false;
 		private TemporaryFolder TempFolder { get; init; }
 		private Lazy<GraphQLHttpClient> LazyGqlClient { get; init; }
+		private string MongoContainerId { get; set; }
 		public GraphQLHttpClient GqlClient => LazyGqlClient.Value;
 
 		public SRTestEnvironment(TemporaryFolder? tempFolder = null)
@@ -51,6 +55,59 @@ public SRTestEnvironment(TemporaryFolder? tempFolder = null)
 			Settings.CommitWhenDone = true; // For SR tests specifically, we *do* want changes to .fwdata files to be persisted
 		}
 
+		override protected void RegisterMongoConnection(ContainerBuilder builder)
+		{
+			// E2E tests want a real Mogno connection
+			builder.RegisterType<MongoConnection>().As<IMongoConnection>().SingleInstance();
+		}
+
+		public void LaunchMongo()
+		{
+			if (MongoContainerId is null)
+			{
+				var result = CommandLineRunner.Run("docker", "run -p 27017 -d mongo:6", ".", 30, NullProgress);
+				MongoContainerId = result.StandardOutput?.TrimEnd();
+				if (string.IsNullOrEmpty(MongoContainerId)) {
+					throw new InvalidOperationException("Mongo container failed to start, aborting test");
+				}
+				result = CommandLineRunner.Run("docker", $"port {MongoContainerId} 27017", ".", 30, NullProgress);
+				var hostAndPort = result.StandardOutput?.TrimEnd();
+				var parts = hostAndPort.Contains(':') ? hostAndPort.Split(':') : null;
+				if (parts is not null && parts.Length == 2) {
+					Settings.MongoHostname = parts[0].Replace("0.0.0.0", "localhost");
+					Settings.MongoPort = parts[1];
+				} else {
+					throw new InvalidOperationException($"Mongo container port {hostAndPort} could not be parsed, test will not be able to proceed");
+				}
+			}
+		}
+
+		public void StopMongo()
+		{
+			if (MongoContainerId is not null)
+			{
+				CommandLineRunner.Run("docker", $"stop {MongoContainerId}", ".", 30, NullProgress);
+				CommandLineRunner.Run("docker", $"rm {MongoContainerId}", ".", 30, NullProgress);
+				MongoContainerId = null;
+			}
+		}
+
+		protected override void Dispose(bool disposing)
+		{
+			if (_disposed) return;
+			if (disposing) {
+				// Set the LFMERGE_E2E_LEAVE_MONGO_CONTAINER_RUNNING_ON_FAILURE env var to a non-empty value if you want Mongo container to NOT be torn down on test failure
+				if (CleanUpTestData || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(MagicStrings.EnvVar_E2E_LeaveMongoContainerRunningOnFailure))) {
+					StopMongo();
+				} else {
+					Console.WriteLine($"Leaving Mongo container {MongoContainerId} around to examine data on failed test.");
+					Console.WriteLine($"It is listening on {Settings.MongoDbHostNameAndPort}");
+					Console.WriteLine($"To delete it, run `docker stop {MongoContainerId} ; docker rm {MongoContainerId}`.");
+				}
+			}
+			base.Dispose(disposing);
+		}
+
 		public async Task Login()
 		{
 			if (AlreadyLoggedIn) return;

src/LfMerge.Core.Tests/TestDoubles.cs

@@ -211,7 +211,7 @@ public IEnumerable<LfLexEntry> GetLfLexEntries()
 			return new List<LfLexEntry>(_storedLfLexEntries.Values.Select(entry => DeepCopy(entry)));
 		}
 
-		public LfLexEntry GetLfLexEntryByGuid(Guid key)
+		public LfLexEntry GetLfLexEntryByGuid(ILfProject _project, Guid key)
 		{
 			LfLexEntry result;
 			if (_storedLfLexEntries.TryGetValue(key, out result))

src/LfMerge.Core.Tests/TestEnvironment.cs

@@ -25,7 +25,8 @@ public class TestEnvironment : IDisposable
 		protected readonly TemporaryFolder     _languageForgeServerFolder;
 		private readonly bool                  _resetLfProjectsDuringCleanup;
 		private readonly bool                  _releaseSingletons;
-		public           bool                  DeleteTempFolderDuringCleanup { get; set; } = true;
+		protected        bool                  _disposed;
+		public           bool                  CleanUpTestData { get; set; } = true;
 		public           LfMergeSettings       Settings;
 		private readonly MongoConnectionDouble _mongoConnection;
 		public ILogger Logger => MainClass.Logger;
@@ -57,6 +58,8 @@ public TestEnvironment(bool registerSettingsModelDouble = true,
 			_releaseSingletons = !SingletonsContainer.Contains<CoreGlobalWritingSystemRepository>();
 		}
 
+		~TestEnvironment() => Dispose(false);
+
 		private string TestName
 		{
 			get
@@ -79,8 +82,7 @@ private ContainerBuilder RegisterTypes(bool registerSettingsModel,
 			containerBuilder.RegisterType<TestLogger>().SingleInstance().As<ILogger>()
 				.WithParameter(new TypedParameter(typeof(string), TestName));
 
-
-			containerBuilder.RegisterType<MongoConnectionDouble>().As<IMongoConnection>().SingleInstance();
+			RegisterMongoConnection(containerBuilder);
 
 			if (registerSettingsModel)
 			{
@@ -100,20 +102,33 @@ private ContainerBuilder RegisterTypes(bool registerSettingsModel,
 			return containerBuilder;
 		}
 
+		protected virtual void RegisterMongoConnection(ContainerBuilder builder)
+		{
+			builder.RegisterType<MongoConnectionDouble>().As<IMongoConnection>().SingleInstance();
+		}
+
 		public void Dispose()
 		{
-			_mongoConnection?.Reset();
-
-			MainClass.Container?.Dispose();
-			MainClass.Container = null;
-			if (_resetLfProjectsDuringCleanup)
-				LanguageForgeProjectAccessor.Reset();
-			if (DeleteTempFolderDuringCleanup)
-				_languageForgeServerFolder?.Dispose();
-			Settings = null;
-			if (_releaseSingletons)
-				SingletonsContainer.Release();
+			Dispose(true);
+			GC.SuppressFinalize(this);
+		}
 
+		protected virtual void Dispose(bool disposing)
+		{
+			if (_disposed) return;
+			if (disposing) {
+				_mongoConnection?.Reset();
+
+				MainClass.Container?.Dispose();
+				MainClass.Container = null;
+				if (_resetLfProjectsDuringCleanup)
+					LanguageForgeProjectAccessor.Reset();
+				if (CleanUpTestData)
+					_languageForgeServerFolder?.Dispose();
+				Settings = null;
+				if (_releaseSingletons)
+					SingletonsContainer.Release();
+			}
 			Environment.SetEnvironmentVariable("FW_CommonAppData", null);
 		}
 

src/LfMerge.Core/MagicStrings.cs

@@ -33,6 +33,7 @@ static MagicStrings()
 		public const string EnvVar_LanguageDepotUriPort = "LFMERGE_LANGUAGE_DEPOT_HG_PORT";
 		public const string EnvVar_TrustToken = "LANGUAGE_DEPOT_TRUST_TOKEN";
 		public const string EnvVar_HgUsername = "LANGUAGE_DEPOT_HG_USERNAME";
+		public const string EnvVar_E2E_LeaveMongoContainerRunningOnFailure = "LFMERGE_E2E_LEAVE_MONGO_CONTAINER_RUNNING_ON_FAILURE";
 
 		public static Dictionary<string, string> LcmOptionlistNames = new Dictionary<string, string>()
 		{

src/LfMerge.Core/MongoConnector/IMongoConnection.cs

@@ -19,6 +19,7 @@ public interface IMongoConnection
 		IEnumerable<TDocument> GetRecords<TDocument>(ILfProject project, string collectionName);
 		LfOptionList GetLfOptionListByCode(ILfProject project, string listCode);
 		long LexEntryCount(ILfProject project);
+		LfLexEntry GetLfLexEntryByGuid(ILfProject project, Guid key);
 		Dictionary<Guid, DateTime> GetAllModifiedDatesForEntries(ILfProject project);
 		bool UpdateRecord(ILfProject project, LfLexEntry data);
 		bool UpdateRecord(ILfProject project, LfOptionList data, string listCode);

src/LfMerge.Core/MongoConnector/MongoConnection.cs

@@ -88,7 +88,8 @@ private MongoClient GetNewConnection()
 			var clientSettings = MongoClientSettings.FromConnectionString(connectionString);
 			// clientSettings.WriteConcern = WriteConcern.WMajority; // If increasing the wait queue size still doesn't help, try this as well
 			clientSettings.WaitQueueSize = 50000;
-			clientSettings.Server = new MongoServerAddress(Settings.MongoHostname, Settings.MongoPort);
+			var mongoPort = int.TryParse(Settings.MongoPort, out var port) ? port : DefaultLfMergeSettings.MongoPort;
+			clientSettings.Server = new MongoServerAddress(Settings.MongoHostname, mongoPort);
 			return new MongoClient(clientSettings);
 		}
 
@@ -148,6 +149,13 @@ public long LexEntryCount(ILfProject project)
 			return lexicon.Count(allEntries);
 		}
 
+		public LfLexEntry GetLfLexEntryByGuid(ILfProject project, Guid key)
+		{
+			IMongoDatabase db = GetProjectDatabase(project);
+			IMongoCollection<LfLexEntry> collection = db.GetCollection<LfLexEntry>(MagicStrings.LfCollectionNameForLexicon);
+			return collection.Find<LfLexEntry>(entry => entry.Guid == key).First();
+		}
+
 		public Dictionary<Guid, DateTime> GetAllModifiedDatesForEntries(ILfProject project)
 		{
 			IMongoDatabase db = GetProjectDatabase(project);
@@ -554,7 +562,7 @@ public bool SetInputSystems(ILfProject project, Dictionary<string, LfInputSystem
 			MongoProjectRecord oldProjectRecord = collection.FindOneAndUpdate(filter, update, updateOptions);
 
 			// For initial clone, also update field writing systems accordingly
-			if (project.IsInitialClone)
+			if (oldProjectRecord != null && project.IsInitialClone)
 			{
 				// Currently only MultiText fields have input systems in the config. If MultiParagraph fields acquire input systems as well,
 				// we'll need to add those to the logic below.