Support ORU messages with multi-line reports spanning across OBX

TEMPLATING.md

@@ -257,6 +257,9 @@ The specification expression has the following format :
   Example: ``OBX.1 |OBX.2|OBX.3`` , if OBX.1 is null then only OBX.2 will be extracted.
 * Multiple value extraction - In HL7 several fields can have repeated values, so extract all repetition for that field the spec string should end with *.<br>
   Example: ``PID.3 *`` , ``OBX.1 |OBX.2 |OBX.3 *``
+* Preserving white space / empty fields - Blank fields may be used to represent new lines or white space in reports. The user may want to preserve this white space to keep the integrity of the original report. To preserve this white space, the spec string should end with an &. Note that this can be combined (and often will be combined) with the multiple value extraction, either &* or *& is supported.<br> 
+  Example: ``OBX.5 *&`` , ``OBX.5 &*``, ``OBX.5 & ``
+
 
 
 #### Variable
@@ -279,7 +282,7 @@ Engine supports the following condition types:
 Conditions can be used to choose between multiple sources of data when mapping to a FHIR type. For example, see how `coding` is set in [CodeableConcept.yml](src/main/resources/hl7/datatype/CodeableConcept.yml). `coding` is set by the either coding_1, coding_2, or coding_3 based on the conditions. The last condition that evaluates to true in the list will create the value.
 
 #### Different types of expressions
-* ResourceExpression : This type of expression is used when a field is a data type defined in one of the [data type templates](src/main/resources/hl7/datatype). These data type templates define different [FHIR data types](https://hl7.org/FHIR/datatypes.html).
+* ResourceExpression : This type of expression is used when a field is a data type defined in one of the [data type templates](../master/src/main/resources/hl7/datatype). These data type templates define different [FHIR data types](https://hl7.org/FHIR/datatypes.html).
   Example:
 
 ```yml

src/main/java/io/github/linuxforhealth/hl7/data/Hl7DataHandlerUtil.java

@@ -30,6 +30,10 @@ public static String getStringValue(Object obj) {
   }
 
   public static String getStringValue(Object obj, boolean allComponents) {
+	  return getStringValue(obj,allComponents,". ");
+  }
+
+  public static String getStringValue(Object obj, boolean allComponents, String separatorString) {
     if (obj == null) {
       return null;
     }
@@ -43,20 +47,17 @@ public static String getStringValue(Object obj, boolean allComponents) {
         returnValue = toStringValue(list.get(0), allComponents);
       } else if (!list.isEmpty()) {
         StringBuilder sb = new StringBuilder();
-        list.forEach(e -> sb.append(toStringValue(e, allComponents)).append(". "));
+        list.forEach(e -> sb.append(toStringValue(e, allComponents)).append(separatorString));
         returnValue = StringUtils.strip(sb.toString());
-
       } else {
         returnValue = null;
       }
 
-
     } else {
       returnValue = toStringValue(local, allComponents);
     }
 
     return returnValue;
-
   }
 
   public static String getTableNumber(Object obj) {
@@ -103,7 +104,6 @@ private static String toStringValue(Object local, boolean allComponents) {
   }
 
 
-
   private static String convertVariesDataTypeToString(Object obj, boolean allComponents) {
     if (obj instanceof Variable) {
       Variable v = (Variable) obj;
@@ -124,9 +124,7 @@ private static String getValueFromComposite(Composite com) {
       }
     return StringUtils.stripEnd(sb.toString(), ", ");
 
-
   }
 
 
-
 }

src/main/java/io/github/linuxforhealth/hl7/data/Hl7RelatedGeneralUtils.java

@@ -8,11 +8,13 @@
 import java.time.temporal.ChronoUnit;
 import java.time.temporal.Temporal;
 import java.time.temporal.UnsupportedTemporalTypeException;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.text.StringTokenizer;
 import org.hl7.fhir.r4.model.codesystems.EncounterStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import io.github.linuxforhealth.hl7.data.date.DateUtil;
 
 
@@ -127,5 +129,10 @@ public static String split(Object input, String delimitter, int index) {
     return null;
   }
 
+  public static String concatenateWithChar(Object input, String delimiterChar) {
+	  String result = Hl7DataHandlerUtil.getStringValue(input, true, delimiterChar);
+	  return result;
+  }
+
 
 }

src/main/java/io/github/linuxforhealth/hl7/data/SimpleDataTypeMapper.java

@@ -15,7 +15,8 @@ public enum SimpleDataTypeMapper {
   STRING(SimpleDataValueResolver.STRING), //
   STRING_ALL(SimpleDataValueResolver.STRING_ALL), //
   FLOAT(SimpleDataValueResolver.FLOAT), //
-
+  BASE64_BINARY(SimpleDataValueResolver.BASE64_BINARY),
+
   URI(SimpleDataValueResolver.URI_VAL), //
   URL(SimpleDataValueResolver.STRING), //
   INSTANT(SimpleDataValueResolver.INSTANT), //

src/main/java/io/github/linuxforhealth/hl7/data/SimpleDataValueResolver.java

@@ -8,6 +8,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -211,6 +212,12 @@ public class SimpleDataValueResolver {
 
   };
 
+  public static final ValueExtractor<Object, String> BASE64_BINARY = (Object value) -> {
+    String val = Hl7DataHandlerUtil.getStringValue(value);
+    return Base64.getEncoder().encodeToString(val.getBytes());
+
+  };
+
   public static final ValueExtractor<Object, Object> OBJECT = (Object value) -> {
     return value;
 

src/main/java/io/github/linuxforhealth/hl7/expression/ExpressionAttributes.java

@@ -35,7 +35,6 @@
 public class ExpressionAttributes {
   private static final String OBJECT_TYPE = Object.class.getSimpleName();
 
-
   // Basic properties of an expression
   private String name;
   private String type;
@@ -56,8 +55,6 @@ public class ExpressionAttributes {
   // if valueof attribute ends with * then list of values will be generated
   private boolean generateMultiple;
 
-
-
   // Property specific to ValueExtractionGeneralExpression
   private ImmutablePair<String, String> fetch;
 
@@ -77,22 +74,18 @@ private ExpressionAttributes(Builder exBuilder) {
       this.condition = ConditionUtil.createCondition(exBuilder.rawCondition);
     }
 
-
     this.constants = new HashMap<>();
     if (exBuilder.constants != null && !exBuilder.constants.isEmpty()) {
       this.constants.putAll(exBuilder.constants);
     }
 
-
     this.variables = new ArrayList<>();
     if (exBuilder.rawVariables != null) {
       for (Entry<String, String> e : exBuilder.rawVariables.entrySet()) {
         this.variables.add(VariableGenerator.parse(e.getKey(), e.getValue()));
       }
-
     }
 
-
     this.value = exBuilder.value;
     this.valueOf = exBuilder.valueOf;
     this.generateMultiple = exBuilder.generateList;
@@ -104,79 +97,56 @@ private ExpressionAttributes(Builder exBuilder) {
       this.expressionType = ExpressionType.HL7SPEC;
     }
 
-
   }
 
   public boolean isUseGroup() {
     return useGroup;
   }
 
-
   public String getType() {
     return type;
   }
 
-
-
   public String getDefaultValue() {
     return defaultValue;
   }
 
-
-
   public boolean isRequired() {
     return isRequired;
   }
 
-
-
   public List<Specification> getSpecs() {
     return ImmutableList.copyOf(specs);
   }
 
-
-
   public List<Variable> getVariables() {
     return ImmutableList.copyOf(variables);
   }
 
-
-
   public Condition getFilter() {
     return condition;
   }
 
-
-
   public Map<String, String> getConstants() {
     return ImmutableMap.copyOf(constants);
   }
 
-
-
   public boolean isGenerateMultiple() {
     return generateMultiple;
   }
 
-
-
   public String getValue() {
     return value;
   }
 
-
-
   public ImmutablePair<String, String> getFetch() {
     return fetch;
   }
 
-
-
   public ExpressionType getExpressionType() {
     return expressionType;
   }
 
-
   public String getValueOf() {
     return valueOf;
   }
@@ -190,23 +160,47 @@ public String getName() {
     return name;
   }
 
-  public static List<Specification> getSpecList(String inputString, boolean useGroup) {
-    final boolean extractMultiple;
-    String hl7SpecExpression = inputString;
-    if (StringUtils.endsWith(inputString, "*")) {
-      hl7SpecExpression = StringUtils.removeEnd(inputString, "*");
+  /**
+   * Extract special chars:
+   *      * indicates to extract fields from multiple entries
+   *      & indicates to retain empty (null) fields
+   * @param inputString
+   * @return ExpressionModifiers object with booleans indicating which modifiers were used and the expression after modifiers have been removed
+   */
+  public static final ExpressionModifiers extractExpressionModifiers(String inputString) {
+
+    boolean extractMultiple = false;
+    boolean retainEmpty = false;
+    String expression = inputString;
+
+    if (StringUtils.endsWith(expression, "*")) {
+      expression = StringUtils.removeEnd(expression, "*");
+      extractMultiple = true;
+    }
+    if (StringUtils.endsWith(expression, "&")) {
+      expression = StringUtils.removeEnd(expression, "&");
+      retainEmpty = true;
+    }
+    // Repeat check for asterisk to allow for different order of special chars
+    if (StringUtils.endsWith(expression, "*")) {
+      expression = StringUtils.removeEnd(expression, "*");
       extractMultiple = true;
-    } else {
-      extractMultiple = false;
     }
+    expression = StringUtils.strip(expression);
+
+    return new ExpressionModifiers(extractMultiple, retainEmpty, expression);
+  }
+
+  public static List<Specification> getSpecList(String inputString, boolean useGroup) {
+
+    ExpressionModifiers exp = extractExpressionModifiers(inputString);
 
-    hl7SpecExpression = StringUtils.strip(hl7SpecExpression);
     List<Specification> specs = new ArrayList<>();
-    if (StringUtils.isNotBlank(hl7SpecExpression)) {
-      StringTokenizer st = new StringTokenizer(hl7SpecExpression, "|").setIgnoreEmptyTokens(true)
+    if (StringUtils.isNotBlank(exp.expression)) {
+      StringTokenizer st = new StringTokenizer(exp.expression, "|").setIgnoreEmptyTokens(true)
           .setTrimmerMatcher(StringMatcherFactory.INSTANCE.spaceMatcher());
       st.getTokenList()
-          .forEach(s -> specs.add(SpecificationParser.parse(s, extractMultiple, useGroup)));
+          .forEach(s -> specs.add(SpecificationParser.parse(s, exp.extractMultiple, useGroup, exp.retainEmpty)));
     }
 
     return specs;
@@ -227,7 +221,6 @@ private static ImmutablePair<String, String> getPair(String tok) {
     } else {
       return null;
     }
-
   }
 
   @Override
@@ -236,7 +229,6 @@ public String toString() {
       this.toString = ReflectionToStringBuilder.toString(this, ToStringStyle.NO_CLASS_NAME_STYLE,
           false, false, true, null);
     }
-
     return this.toString;
   }
 
@@ -249,6 +241,7 @@ public String toString() {
   public static class Builder {
 
 
+
     private String name;
     private String type;
     private String defaultValue;
@@ -310,35 +303,29 @@ public Builder withCondition(String rawCondition) {
       return this;
     }
 
-
     public Builder withVars(Map<String, String> rawVariables) {
       this.rawVariables = rawVariables;
       return this;
     }
 
-
     public Builder withConstants(Map<String, String> constants) {
       this.constants = constants;
       return this;
     }
 
     public Builder withValueOf(String valueOf) {
-
       this.valueOf = StringUtils.trim(valueOf);
       if (this.expressionType == null) {
         this.expressionType = ExpressionType.SIMPLE;
       }
       return this;
-
     }
 
     public Builder withExpressionType(String expressionType) {
       this.expressionType = EnumUtils.getEnumIgnoreCase(ExpressionType.class, expressionType);
       return this;
     }
 
-
-
     public Builder withValue(String value) {
       this.value = value;
       this.expressionType = ExpressionType.SIMPLE;
@@ -350,15 +337,24 @@ public Builder withGenerateList(boolean generateList) {
       return this;
     }
 
-
-
     public ExpressionAttributes build() {
       return new ExpressionAttributes(this);
     }
 
   }
 
-
-
+  // Class used when extracting modifiers from the expression, contains the expression after modifiers have been removed and
+  // booleans indicating which modifiers were in the expression.
+  public static class ExpressionModifiers {
+    public boolean extractMultiple = false;  // true when * is used in the expression
+    public boolean retainEmpty = false;      // true when & is used in the expression
+    public String expression = "";           // resulting expression after the modifiers have been removed
+
+    ExpressionModifiers(boolean theExtractMultiple, boolean theRetainEmpty, String theExpression) {
+      extractMultiple = theExtractMultiple;
+      retainEmpty = theRetainEmpty;
+      expression = theExpression;
+    }
+  }
 }