Merge branch 'develop' into dev-text-backend

Sapphire-Lilith · Aug 31, 2018 · 3010094 · 3010094
2 parents 79ce730 + 4848b14
commit 3010094
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/src/chardecoder/lineBreakDataMap.ml b/src/chardecoder/lineBreakDataMap.ml
@@ -142,11 +142,17 @@ type 'a rule = line_break_regexp * 'a * line_break_regexp
 (* -- the rules for normalizing texts about spaces, break letters, etc. -- *)
 let normalization_rule : (((Uchar.t * line_break_class) list) rule) list =
   [
-  (* -- ignore spaces or break letters *)
+  (* -- ignore spaces or breaks between a nonspaced character and spaced one -- *)
     ([nonspaced; set [SP; INBR]], [], [spaced]);
     ([spaced; set [SP; INBR]], [], [nonspaced]);
-  (* -- ignore break letters between nonspaced characters -- *)
+  (* -- ignore breaks between nonspaced characters -- *)
     ([nonspaced; exact INBR], [], [nonspaced]);
+  (* -- preserve spaces between nonspaced characters -- *)
+    ([nonspaced; exact SP], [bispace], [nonspaced]);
+  (* -- ignore spaces before and after a nonspaced characters -- *)
+    ([nonspaced; set [SP; INBR]], [], []);
+    ([set [SP; INBR]], [], [nonspaced]);
+  (* -- convert breaks into spaces -- *)
     ([exact INBR], [bispace], []);
   ]