LT-20351: Fix LT-20351 and implement Novel Root Guesser

src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs

@@ -72,6 +72,11 @@ public IDictionary<string, object> Properties
             get { return _properties; }
         }
 
+        /// <summary>
+        /// Was this allomorph guessed by a lexical pattern?
+        /// </summary>
+        public bool Guessed { get; set; }
+
         public bool FreeFluctuatesWith(Allomorph other)
         {
             if (this == other)

src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs

@@ -90,6 +90,8 @@ public IEnumerable<Word> Apply(Word input)
                 output.Add(mruleOutWord);
                 if (_morpher.TraceManager.IsTracing)
                     _morpher.TraceManager.EndUnapplyStratum(_stratum, mruleOutWord);
+                if (_morpher.MaxUnapplications > 0 && output.Count >= _morpher.MaxUnapplications)
+                    break;
             }
             return output;
         }

src/SIL.Machine.Morphology.HermitCrab/CharacterDefinitionTable.cs

@@ -14,11 +14,13 @@ public class CharacterDefinitionTable : ICollection<CharacterDefinition>
     {
         private readonly Dictionary<string, CharacterDefinition> _charDefLookup;
         private readonly HashSet<CharacterDefinition> _charDefs;
+        private readonly Dictionary<string, NaturalClass> _naturalClassLookup;
 
         public CharacterDefinitionTable()
         {
             _charDefLookup = new Dictionary<string, CharacterDefinition>();
             _charDefs = new HashSet<CharacterDefinition>();
+            _naturalClassLookup = new Dictionary<string, NaturalClass>();
         }
 
         public string Name { get; set; }
@@ -43,6 +45,11 @@ public CharacterDefinition AddBoundary(IEnumerable<string> strRep)
             return Add(strRep, HCFeatureSystem.Boundary, null);
         }
 
+        public void AddNaturalClass(NaturalClass naturalClass)
+        {
+            _naturalClassLookup[naturalClass.Name] = naturalClass;
+        }
+
         /// <summary>
         /// Adds the character definition.
         /// </summary>
@@ -98,11 +105,14 @@ public IEnumerable<string> GetMatchingStrReps(ShapeNode node)
             }
         }
 
-        private bool GetShapeNodes(string str, out IEnumerable<ShapeNode> nodes, out int errorPos)
+        private bool GetShapeNodes(string str, bool allowPattern, out IEnumerable<ShapeNode> nodes, out int errorPos)
         {
             var nodesList = new List<ShapeNode>();
             int i = 0;
             string normalized = str.Normalize(NormalizationForm.FormD);
+            bool optional = false;
+            int optionalPos = 0;
+            int optionalCount = 0;
             while (i < normalized.Length)
             {
                 bool match = false;
@@ -120,26 +130,104 @@ private bool GetShapeNodes(string str, out IEnumerable<ShapeNode> nodes, out int
                         break;
                     }
                 }
-
-                if (!match)
+                if (match)
+                    continue;
+                if (allowPattern)
                 {
-                    nodes = null;
-                    errorPos = i;
-                    if (!str.IsNormalized(NormalizationForm.FormD))
-                        errorPos = normalized.Substring(0, errorPos).Normalize().Length;
-                    return false;
+                    // Check for pattern language.
+                    // NB: This only happens when the characters don't match.
+                    // I thought about implementing this using Pattern<Shape, ShapeNode>,
+                    // but the Matcher doesn't preserve the unifications of the nodes.
+                    if (normalized[i] == '[')
+                    {
+                        // Example: [Seg].
+                        // Look for a natural class.
+                        int closePos = normalized.IndexOf("]", i);
+                        if (closePos > 0)
+                        {
+                            string className = normalized.Substring(i + 1, closePos - i - 1);
+                            if (_naturalClassLookup.ContainsKey(className))
+                            {
+                                NaturalClass naturalClass = _naturalClassLookup[className];
+                                var node = new ShapeNode(naturalClass.FeatureStruct);
+                                nodesList.Add(node);
+                                i = closePos + 1;
+                                continue;
+                            }
+                        }
+                    }
+                    else if (normalized[i] == '(')
+                    {
+                        if (i + 1 < normalized.Length && normalized[i + 1] == '[')
+                        {
+                            // The natural class that follows is optional.
+                            // Wait for the close parenthesis to process.
+                            optional = true;
+                            optionalPos = i;
+                            optionalCount = nodesList.Count;
+                            i++;
+                            continue;
+                        }
+                    }
+                    else if (normalized[i] == ')')
+                    {
+                        if (optional && nodesList.Count == optionalCount + 1)
+                        {
+                            // Example: ([Seg]).
+                            // Ill-formed: ([C][V]).
+                            // Make the last node optional.
+                            nodesList[nodesList.Count - 1].Annotation.Optional = true;
+                            optional = false;
+                            i++;
+                            continue;
+                        }
+                    }
+                    else if (normalized[i] == '*')
+                    {
+                        if (i > 0 && normalized[i - 1] == ']')
+                        {
+                            // Example: [Seg]*.
+                            // Make the last node Kleene star.
+                            nodesList[nodesList.Count - 1].Annotation.Optional = true;
+                            nodesList[nodesList.Count - 1].SetIterative(true);
+                            i++;
+                            continue;
+                        }
+                    }
+                    // Kleene plus doesn't work because '+' is a boundary marker.
                 }
+
+                // Failure
+                nodes = null;
+                errorPos = i;
+                if (!str.IsNormalized(NormalizationForm.FormD))
+                    errorPos = normalized.Substring(0, errorPos).Normalize().Length;
+                return false;
+            }
+            if (optional)
+            {
+                // The open parenthesis didn't get closed.
+                nodes = null;
+                errorPos = optionalPos;
+                if (!str.IsNormalized(NormalizationForm.FormD))
+                    errorPos = normalized.Substring(0, errorPos).Normalize().Length;
+                return false;
             }
             nodes = nodesList;
             errorPos = -1;
             return true;
         }
 
         public Shape Segment(string str)
+        {
+            return Segment(str, false);
+        }
+
+        public Shape Segment(string str, bool allowPattern)
         {
             IEnumerable<ShapeNode> nodes;
             int errorPos;
-            if (GetShapeNodes(str, out nodes, out errorPos))
+            if (GetShapeNodes(str, allowPattern, out nodes, out errorPos))
             {
                 var shape = new Shape(begin => new ShapeNode(
                     begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor
@@ -162,7 +250,7 @@ public int TrySegment(string str, out Shape shape)
         {
             IEnumerable<ShapeNode> nodes;
             int errorPos;
-            if (GetShapeNodes(str, out nodes, out errorPos))
+            if (GetShapeNodes(str, true, out nodes, out errorPos))
             {
                 shape = new Shape(begin => new ShapeNode(
                     begin ? HCFeatureSystem.LeftSideAnchor : HCFeatureSystem.RightSideAnchor

src/SIL.Machine.Morphology.HermitCrab/HermitCrabExtensions.cs

@@ -88,6 +88,19 @@ internal static void SetDeleted(this ShapeNode node, bool deleted)
             );
         }
 
+        internal static bool IsIterative(this ShapeNode node)
+        {
+            return node.Annotation.Data != null;
+        }
+
+        internal static void SetIterative(this ShapeNode node, bool iterative)
+        {
+            if (iterative)
+                node.Annotation.Data = iterative;
+            else
+                node.Annotation.Data = null;
+        }
+
         private static readonly IEqualityComparer<ShapeNode> NodeComparer = new ProjectionEqualityComparer<
             ShapeNode,
             FeatureStruct