Use real MongoDB connection for E2E tests (#348)

A real MongoConnection is used for E2E tests, so that they're truly
end-to-end, rather than the mock one that stores data in memory. The
docker image `mongo:6` is used to provide the MongoDB container.

* Skip updating project record if there is none

In E2E tests, the SetInputSystems call would fail because there is no
existing project record in the freshly-created, empty Mongo instance.
We don't want to change LfMerge to create the project record, because
the Language Forge code is responsible for doing that. So instead we
simply skip the update if the project record doesn't exist, allowing E2E
tests to pass while not changing the behavior of real code.

* Delete MongoDB container after test

Unless LFMERGE_E2E_LEAVE_MONGO_CONTAINER_RUNNING_ON_FAILURE is set (to
any non-empty value except "false" or "0"), we'll tear down the MongoDB
container after tests. Also, if the test succeeds, the MongoDB container
is torn down even if that env var is set. The only time the container is
retained is if the test fails *and* that env var is set.

src/LfMerge.Core.Tests/E2E/E2ETestBase.cs

@@ -25,7 +25,7 @@ public class E2ETestBase
 		private readonly HashSet<Guid> ProjectIdsToDelete = [];
 		public SRTestEnvironment TestEnv { get; set; }
 
-		public MongoConnectionDouble _mongoConnection;
+		public IMongoConnection _mongoConnection;
 		public MongoProjectRecordFactory _recordFactory;
 
 		public E2ETestBase()
@@ -81,14 +81,14 @@ public async Task TestSetup()
 				Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox.");
 			}
 			await TestEnv.Login();
+			TestEnv.LaunchMongo();
 
 			MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion);
-			_mongoConnection = MainClass.Container.Resolve<IMongoConnection>() as MongoConnectionDouble;
-			if (_mongoConnection == null)
-				throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work.");
-			_recordFactory = MainClass.Container.Resolve<MongoProjectRecordFactory>() as MongoProjectRecordFactoryDouble;
-			if (_recordFactory == null)
-				throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work.");
+			_mongoConnection = MainClass.Container.Resolve<IMongoConnection>();
+			var _mongoConnectionDouble = _mongoConnection as MongoConnectionDouble;
+			if (_mongoConnectionDouble != null)
+				throw new AssertionException("E2E tests need a real MongoConnection, not a mock.");
+			_recordFactory = MainClass.Container.Resolve<MongoProjectRecordFactory>();
 		}
 
 		[TearDown]
@@ -97,7 +97,7 @@ public async Task TestTeardown()
 			var outcome = TestContext.CurrentContext.Result.Outcome;
 			var success = outcome == ResultState.Success || outcome == ResultState.Ignored;
 			// Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation
-			TestEnv.DeleteTempFolderDuringCleanup = success;
+			TestEnv.CleanUpTestData = success;
 			// On failure, also leave LexBox project(s) in place for post-test investigation, even though this might tend to clutter things up a little
 			if (success) {
 				foreach (var projId in ProjectIdsToDelete) {
@@ -220,7 +220,7 @@ public void SendReceiveToLexbox(LanguageForgeProject lfProject)
 
 		public (string, DateTime, DateTime) UpdateLfGloss(LanguageForgeProject lfProject, Guid entryId, string wsId, Func<string, string> textConverter)
 		{
-			var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			var lfEntry = _mongoConnection.GetLfLexEntryByGuid(lfProject, entryId);
 			Assert.That(lfEntry, Is.Not.Null);
 			var unchangedGloss = lfEntry.Senses[0].Gloss[wsId].Value;
 			lfEntry.Senses[0].Gloss["pt"].Value = textConverter(unchangedGloss);

src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs

@@ -50,7 +50,7 @@ public async Task E2E_LFDataChangedLDDataChanged_LFWins()
 			// Verify
 
 			// LF side should win conflict since its modified date was later
-			var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(lfProject, entryId);
 			Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF"));
 			// LF's modified dates should have been updated by the sync action
 			Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified));

src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs

@@ -830,7 +830,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldBeAbleToAddAndModifyParagraphsInCu
 			// Here we check two things:
 			// 1) Can we add paragraphs?
 			// 2) Can we change existing paragraphs?
-			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(lfProject, entryGuid);
 			// BsonDocument customFieldValues = GetCustomFieldValues(cache, lcmEntry, "entry");
 			BsonDocument customFieldsBson = lfEntry.CustomFields;
 			Assert.That(customFieldsBson.Contains("customField_entry_Cust_MultiPara"), Is.True,
@@ -923,7 +923,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldBeAbleToDeleteParagraphsInCustomMu
 
 			// Here we check just one thing:
 			// 1) Can we delete paragraphs?
-			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry lfEntry = _conn.GetLfLexEntryByGuid(lfProject, entryGuid);
 			// BsonDocument customFieldValues = GetCustomFieldValues(cache, lcmEntry, "entry");
 			BsonDocument customFieldsBson = lfEntry.CustomFields;
 			Assert.That(customFieldsBson.Contains("customField_entry_Cust_MultiPara"), Is.True,

src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs

@@ -234,7 +234,7 @@ public void Action_WithOneModifiedEntry_ShouldCountOneModified()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -263,7 +263,7 @@ public void Action_WithOneDeletedEntry_ShouldCountOneDeleted()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -325,7 +325,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -335,7 +335,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified()
 			_conn.UpdateMockLfLexEntry(entry);
 
 			Guid kenGuid = Guid.Parse(KenEntryGuidStr);
-			LfLexEntry kenEntry = _conn.GetLfLexEntryByGuid(kenGuid);
+			LfLexEntry kenEntry = _conn.GetLfLexEntryByGuid(lfProj, kenGuid);
 			string changedLexeme2 = "modified lexeme #2 for this test";
 			kenEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2);
 			kenEntry.AuthorInfo = new LfAuthorInfo();
@@ -362,11 +362,11 @@ public void Action_WithTwoDeletedEntries_ShouldCountTwoDeleted()
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 			Guid kenGuid = Guid.Parse(KenEntryGuidStr);
-			entry = _conn.GetLfLexEntryByGuid(kenGuid);
+			entry = _conn.GetLfLexEntryByGuid(lfProj, kenGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -430,7 +430,7 @@ public void Action_WithOneModifiedEntry_ShouldNotCountThatModifiedEntryOnSecondR
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -470,7 +470,7 @@ public void Action_WithOneDeletedEntry_ShouldNotCountThatDeletedEntryOnSecondRun
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -557,7 +557,7 @@ public void Action_RunTwiceWithTheSameEntryModifiedEachTime_ShouldCountTwoModifi
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LcmCache cache = lfProj.FieldWorksProject.Cache;
 			string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id;
 			string changedLexeme = "modified lexeme for this test";
@@ -606,7 +606,7 @@ public void Action_RunTwiceWithTheSameEntryDeletedEachTime_ShouldCountJustOneDel
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -621,7 +621,7 @@ public void Action_RunTwiceWithTheSameEntryDeletedEachTime_ShouldCountJustOneDel
 			Assert.That(LfMergeBridgeServices.FormatCommitMessageForLfMerge(_counts.Added, _counts.Modified, _counts.Deleted),
 				Is.EqualTo("Language Forge: 1 entry deleted"));
 
-			entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			entry.IsDeleted = true;
 			_conn.UpdateMockLfLexEntry(entry);
 
@@ -689,7 +689,7 @@ public void Run_CustomMultiListRefTest(int whichSense, params string[] desiredKe
 			SutLcmToMongo.Run(lfProj);
 
 			Guid entryGuid = Guid.Parse(TestEntryGuidStr);
-			LfLexEntry entry = _conn.GetLfLexEntryByGuid(entryGuid);
+			LfLexEntry entry = _conn.GetLfLexEntryByGuid(lfProj, entryGuid);
 			LfSense sense = entry.Senses[whichSense];
 			SetCustomMultiOptionList(sense, "customField_senses_Cust_Multi_ListRef", desiredKeys);
 			entry.AuthorInfo = new LfAuthorInfo();

src/LfMerge.Core.Tests/SRTestEnvironment.cs

@@ -5,14 +5,17 @@
 using System.Net.Http;
 using System.Net.Http.Json;
 using System.Threading.Tasks;
+using Autofac;
 using BirdMessenger;
 using BirdMessenger.Collections;
 using GraphQL;
 using GraphQL.Client.Http;
 using GraphQL.Client.Serializer.SystemTextJson;
 using LfMerge.Core.FieldWorks;
 using LfMerge.Core.Logging;
+using LfMerge.Core.MongoConnector;
 using NUnit.Framework;
+using SIL.CommandLineProcessing;
 using SIL.TestUtilities;
 
 namespace LfMerge.Core.Tests
@@ -39,6 +42,7 @@ public class SRTestEnvironment : TestEnvironment
 		private bool AlreadyLoggedIn = false;
 		private TemporaryFolder TempFolder { get; init; }
 		private Lazy<GraphQLHttpClient> LazyGqlClient { get; init; }
+		private string MongoContainerId { get; set; }
 		public GraphQLHttpClient GqlClient => LazyGqlClient.Value;
 
 		public SRTestEnvironment(TemporaryFolder? tempFolder = null)
@@ -51,6 +55,66 @@ public SRTestEnvironment(TemporaryFolder? tempFolder = null)
 			Settings.CommitWhenDone = true; // For SR tests specifically, we *do* want changes to .fwdata files to be persisted
 		}
 
+		override protected void RegisterMongoConnection(ContainerBuilder builder)
+		{
+			// E2E tests want a real Mogno connection
+			builder.RegisterType<MongoConnection>().As<IMongoConnection>().SingleInstance();
+		}
+
+		public void LaunchMongo()
+		{
+			if (MongoContainerId is null)
+			{
+				var result = CommandLineRunner.Run("docker", "run -p 27017 -d mongo:6", ".", 30, NullProgress);
+				MongoContainerId = result.StandardOutput?.TrimEnd();
+				if (string.IsNullOrEmpty(MongoContainerId)) {
+					throw new InvalidOperationException("Mongo container failed to start, aborting test");
+				}
+				result = CommandLineRunner.Run("docker", $"port {MongoContainerId} 27017", ".", 30, NullProgress);
+				var hostAndPort = result.StandardOutput?.TrimEnd();
+				var parts = hostAndPort.Contains(':') ? hostAndPort.Split(':') : null;
+				if (parts is not null && parts.Length == 2) {
+					Settings.MongoHostname = parts[0].Replace("0.0.0.0", "localhost");
+					Settings.MongoPort = parts[1];
+				} else {
+					throw new InvalidOperationException($"Mongo container port {hostAndPort} could not be parsed, test will not be able to proceed");
+				}
+			}
+		}
+
+		public void StopMongo()
+		{
+			if (MongoContainerId is not null)
+			{
+				CommandLineRunner.Run("docker", $"stop {MongoContainerId}", ".", 30, NullProgress);
+				CommandLineRunner.Run("docker", $"rm {MongoContainerId}", ".", 30, NullProgress);
+				MongoContainerId = null;
+			}
+		}
+
+		private bool ShouldStopMongoOnFailure()
+		{
+			// Mongo container will be torn down on test failure unless LFMERGE_E2E_LEAVE_MONGO_CONTAINER_RUNNING_ON_FAILURE
+			// is set to a non-empty value (except "false" or "0", which mean the same as leaving it empty)
+			var envVar = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_E2E_LeaveMongoContainerRunningOnFailure)?.Trim();
+			return string.IsNullOrEmpty(envVar) || envVar == "false" || envVar == "0";
+		}
+
+		protected override void Dispose(bool disposing)
+		{
+			if (_disposed) return;
+			if (disposing) {
+				if (CleanUpTestData || ShouldStopMongoOnFailure()) {
+					StopMongo();
+				} else {
+					Console.WriteLine($"Leaving Mongo container {MongoContainerId} around to examine data on failed test.");
+					Console.WriteLine($"It is listening on {Settings.MongoDbHostNameAndPort}");
+					Console.WriteLine($"To delete it, run `docker stop {MongoContainerId} ; docker rm {MongoContainerId}`.");
+				}
+			}
+			base.Dispose(disposing);
+		}
+
 		public async Task Login()
 		{
 			if (AlreadyLoggedIn) return;

src/LfMerge.Core.Tests/TestDoubles.cs

@@ -211,7 +211,7 @@ public IEnumerable<LfLexEntry> GetLfLexEntries()
 			return new List<LfLexEntry>(_storedLfLexEntries.Values.Select(entry => DeepCopy(entry)));
 		}
 
-		public LfLexEntry GetLfLexEntryByGuid(Guid key)
+		public LfLexEntry GetLfLexEntryByGuid(ILfProject _project, Guid key)
 		{
 			LfLexEntry result;
 			if (_storedLfLexEntries.TryGetValue(key, out result))

src/LfMerge.Core.Tests/TestEnvironment.cs

@@ -25,7 +25,8 @@ public class TestEnvironment : IDisposable
 		protected readonly TemporaryFolder     _languageForgeServerFolder;
 		private readonly bool                  _resetLfProjectsDuringCleanup;
 		private readonly bool                  _releaseSingletons;
-		public           bool                  DeleteTempFolderDuringCleanup { get; set; } = true;
+		protected        bool                  _disposed;
+		public           bool                  CleanUpTestData { get; set; } = true;
 		public           LfMergeSettings       Settings;
 		private readonly MongoConnectionDouble _mongoConnection;
 		public ILogger Logger => MainClass.Logger;
@@ -57,6 +58,8 @@ public TestEnvironment(bool registerSettingsModelDouble = true,
 			_releaseSingletons = !SingletonsContainer.Contains<CoreGlobalWritingSystemRepository>();
 		}
 
+		~TestEnvironment() => Dispose(false);
+
 		private string TestName
 		{
 			get
@@ -79,8 +82,7 @@ private ContainerBuilder RegisterTypes(bool registerSettingsModel,
 			containerBuilder.RegisterType<TestLogger>().SingleInstance().As<ILogger>()
 				.WithParameter(new TypedParameter(typeof(string), TestName));
 
-
-			containerBuilder.RegisterType<MongoConnectionDouble>().As<IMongoConnection>().SingleInstance();
+			RegisterMongoConnection(containerBuilder);
 
 			if (registerSettingsModel)
 			{
@@ -100,20 +102,33 @@ private ContainerBuilder RegisterTypes(bool registerSettingsModel,
 			return containerBuilder;
 		}
 
+		protected virtual void RegisterMongoConnection(ContainerBuilder builder)
+		{
+			builder.RegisterType<MongoConnectionDouble>().As<IMongoConnection>().SingleInstance();
+		}
+
 		public void Dispose()
 		{
-			_mongoConnection?.Reset();
-
-			MainClass.Container?.Dispose();
-			MainClass.Container = null;
-			if (_resetLfProjectsDuringCleanup)
-				LanguageForgeProjectAccessor.Reset();
-			if (DeleteTempFolderDuringCleanup)
-				_languageForgeServerFolder?.Dispose();
-			Settings = null;
-			if (_releaseSingletons)
-				SingletonsContainer.Release();
+			Dispose(true);
+			GC.SuppressFinalize(this);
+		}
 
+		protected virtual void Dispose(bool disposing)
+		{
+			if (_disposed) return;
+			if (disposing) {
+				_mongoConnection?.Reset();
+
+				MainClass.Container?.Dispose();
+				MainClass.Container = null;
+				if (_resetLfProjectsDuringCleanup)
+					LanguageForgeProjectAccessor.Reset();
+				if (CleanUpTestData)
+					_languageForgeServerFolder?.Dispose();
+				Settings = null;
+				if (_releaseSingletons)
+					SingletonsContainer.Release();
+			}
 			Environment.SetEnvironmentVariable("FW_CommonAppData", null);
 		}
 

src/LfMerge.Core/MagicStrings.cs

@@ -33,6 +33,7 @@ static MagicStrings()
 		public const string EnvVar_LanguageDepotUriPort = "LFMERGE_LANGUAGE_DEPOT_HG_PORT";
 		public const string EnvVar_TrustToken = "LANGUAGE_DEPOT_TRUST_TOKEN";
 		public const string EnvVar_HgUsername = "LANGUAGE_DEPOT_HG_USERNAME";
+		public const string EnvVar_E2E_LeaveMongoContainerRunningOnFailure = "LFMERGE_E2E_LEAVE_MONGO_CONTAINER_RUNNING_ON_FAILURE";
 
 		public static Dictionary<string, string> LcmOptionlistNames = new Dictionary<string, string>()
 		{

src/LfMerge.Core/MongoConnector/IMongoConnection.cs

@@ -19,6 +19,7 @@ public interface IMongoConnection
 		IEnumerable<TDocument> GetRecords<TDocument>(ILfProject project, string collectionName);
 		LfOptionList GetLfOptionListByCode(ILfProject project, string listCode);
 		long LexEntryCount(ILfProject project);
+		LfLexEntry GetLfLexEntryByGuid(ILfProject project, Guid key);
 		Dictionary<Guid, DateTime> GetAllModifiedDatesForEntries(ILfProject project);
 		bool UpdateRecord(ILfProject project, LfLexEntry data);
 		bool UpdateRecord(ILfProject project, LfOptionList data, string listCode);