Fix slow domain object ingestion

.github/workflows/importer.yml

@@ -38,7 +38,11 @@ jobs:
           path: ~/.m2
           restore-keys: ${{ runner.os }}-m2
 
-      - name: Set environment
+      - name: Set v1 environment
+        if: matrix.schema == 'v1'
+        run: echo 'groups=!v2' >> $GITHUB_ENV
+
+      - name: Set v2 environment
         if: matrix.schema == 'v2'
         run: echo 'groups=!v1' >> $GITHUB_ENV
 

hedera-mirror-importer/pom.xml

@@ -109,6 +109,10 @@
             <groupId>org.flywaydb</groupId>
             <artifactId>flyway-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.hibernate</groupId>
+            <artifactId>hibernate-jpamodelgen</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.hibernate.validator</groupId>
             <artifactId>hibernate-validator</artifactId>

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CacheConfiguration.java

@@ -38,7 +38,6 @@ public class CacheConfiguration {
 
     public static final String EXPIRE_AFTER_5M = "cacheManagerExpireAfter5m";
     public static final String EXPIRE_AFTER_30M = "cacheManagerExpireAfter30m";
-    public static final String NEVER_EXPIRE_LARGE = "cacheManagerNeverExpireLarge";
 
     @Bean(EXPIRE_AFTER_5M)
     @Primary
@@ -54,11 +53,4 @@ CacheManager cacheManager30m() {
         caffeineCacheManager.setCacheSpecification("maximumSize=10000,expireAfterWrite=30m");
         return new TransactionAwareCacheManagerProxy(caffeineCacheManager);
     }
-
-    @Bean(NEVER_EXPIRE_LARGE)
-    CacheManager cacheManagerNeverExpireLarge() {
-        CaffeineCacheManager caffeineCacheManager = new CaffeineCacheManager();
-        caffeineCacheManager.setCacheSpecification("maximumSize=2000000"); // 2 million 120MB
-        return new TransactionAwareCacheManagerProxy(caffeineCacheManager);
-    }
 }

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/converter/ByteArrayToHexSerializer.java

@@ -29,6 +29,8 @@
 
 @Named
 public class ByteArrayToHexSerializer extends JsonSerializer<byte[]> {
+    public static final ByteArrayToHexSerializer INSTANCE = new ByteArrayToHexSerializer();
+
     @Override
     public void serialize(byte[] value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
         if (value != null) {

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/converter/EntityIdSerializer.java

@@ -30,6 +30,8 @@
 
 @Named
 public class EntityIdSerializer extends JsonSerializer<EntityId> {
+    public static final EntityIdSerializer INSTANCE = new EntityIdSerializer();
+
     @Override
     public void serialize(EntityId value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
         if (!EntityId.isEmpty(value)) {

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/converter/NullableStringSerializer.java

@@ -0,0 +1,40 @@
+package com.hedera.mirror.importer.converter;
+
+/*-
+ * ‌
+ * Hedera Mirror Node
+ * ​
+ * Copyright (C) 2019 - 2021 Hedera Hashgraph, LLC
+ * ​
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ‍
+ */
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import java.io.IOException;
+import java.util.UUID;
+import javax.inject.Named;
+import org.apache.commons.lang3.StringUtils;
+
+@Named
+public class NullableStringSerializer extends JsonSerializer<String> {
+    public static final String NULLABLE_STRING_REPLACEMENT = UUID.randomUUID().toString();
+
+    @Override
+    public void serialize(String value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+        // empty strings are serialized as null, set to reserved space character and rely on db update sql to parse
+        gen.writeString(StringUtils.EMPTY.equals(value) ? NULLABLE_STRING_REPLACEMENT : value);
+    }
+}

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/AccountBalance.java

@@ -22,7 +22,6 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonUnwrapped;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
@@ -43,7 +42,6 @@
 import org.springframework.data.domain.Persistable;
 
 import com.hedera.mirror.importer.converter.AccountIdConverter;
-import com.hedera.mirror.importer.converter.EntityIdSerializer;
 import com.hedera.mirror.importer.parser.domain.StreamItem;
 
 @Data
@@ -85,7 +83,6 @@ public static class Id implements Serializable {
         private long consensusTimestamp;
 
         @Convert(converter = AccountIdConverter.class)
-        @JsonSerialize(using = EntityIdSerializer.class)
         private EntityId accountId;
     }
 }

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/AddressBookServiceEndpoint.java

@@ -22,7 +22,6 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonUnwrapped;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import java.io.Serializable;
 import javax.persistence.Column;
 import javax.persistence.Convert;
@@ -35,7 +34,6 @@
 import org.springframework.data.domain.Persistable;
 
 import com.hedera.mirror.importer.converter.AccountIdConverter;
-import com.hedera.mirror.importer.converter.EntityIdSerializer;
 
 @Data
 @Entity
@@ -74,7 +72,6 @@ public static class Id implements Serializable {
         private String ipAddressV4;
 
         @Convert(converter = AccountIdConverter.class)
-        @JsonSerialize(using = EntityIdSerializer.class)
         private EntityId nodeId;
 
         private int port;

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/CryptoTransfer.java

@@ -22,7 +22,6 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonUnwrapped;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import java.io.Serializable;
 import javax.persistence.Convert;
 import javax.persistence.Embeddable;
@@ -34,15 +33,14 @@
 import org.springframework.data.domain.Persistable;
 
 import com.hedera.mirror.importer.converter.AccountIdConverter;
-import com.hedera.mirror.importer.converter.EntityIdSerializer;
 
 @Data
 @Entity
 @NoArgsConstructor
 public class CryptoTransfer implements Persistable<CryptoTransfer.Id> {
 
     public CryptoTransfer(long consensusTimestamp, long amount, EntityId entityId) {
-        this.id = new CryptoTransfer.Id(amount, consensusTimestamp, entityId);
+        id = new CryptoTransfer.Id(amount, consensusTimestamp, entityId);
     }
 
     /*
@@ -74,7 +72,6 @@ public static class Id implements Serializable {
         private long consensusTimestamp;
 
         @Convert(converter = AccountIdConverter.class)
-        @JsonSerialize(using = EntityIdSerializer.class)
         private EntityId entityId;
     }
 }

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/Entity.java

@@ -20,37 +20,41 @@
  * ‍
  */
 
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import javax.persistence.Convert;
 import javax.persistence.Id;
 import lombok.Data;
 import lombok.ToString;
 import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.StringUtils;
 
 import com.hedera.mirror.importer.converter.AccountIdConverter;
+import com.hedera.mirror.importer.converter.NullableStringSerializer;
 import com.hedera.mirror.importer.util.Utility;
 
 @Data
 @javax.persistence.Entity
 @Log4j2
 @ToString(exclude = {"key", "submitKey"})
 public class Entity {
-    @Id
-    private Long id;
-
     @Convert(converter = AccountIdConverter.class)
     private EntityId autoRenewAccountId;
 
     private Long autoRenewPeriod;
 
     private Long createdTimestamp;
 
-    private boolean deleted;
+    private Boolean deleted;
 
     private Long expirationTimestamp;
 
+    @Id
+    private Long id;
+
     private byte[] key;
 
-    private String memo = "";
+    @JsonSerialize(using = NullableStringSerializer.class)
+    private String memo;
 
     private Long modifiedTimestamp;
 
@@ -59,6 +63,7 @@ public class Entity {
     @Convert(converter = AccountIdConverter.class)
     private EntityId proxyAccountId;
 
+    @JsonSerialize(using = NullableStringSerializer.class)
     private String publicKey;
 
     private Long realm;
@@ -75,7 +80,7 @@ public void setKey(byte[] key) {
     }
 
     public void setMemo(String memo) {
-        this.memo = Utility.sanitize(memo);
+        this.memo = StringUtils.isEmpty(memo) ? "" : Utility.sanitize(memo);
     }
 
     public EntityId toEntityId() {

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/EntityId.java

@@ -48,7 +48,6 @@
 @Value
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class EntityId implements Serializable, Comparable<EntityId> {
-
     private static final Comparator<EntityId> COMPARATOR = Comparator
             .nullsFirst(Comparator.comparingLong(EntityId::getId));
     public static final EntityId EMPTY = new EntityId(0L, 0L, 0L, EntityTypeEnum.ACCOUNT.getId());

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/FileData.java

@@ -21,7 +21,6 @@
  */
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import javax.persistence.Convert;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -31,7 +30,6 @@
 import lombok.ToString;
 import org.springframework.data.domain.Persistable;
 
-import com.hedera.mirror.importer.converter.EntityIdSerializer;
 import com.hedera.mirror.importer.converter.FileIdConverter;
 
 @Data
@@ -47,7 +45,6 @@ public class FileData implements Persistable<Long> {
     private byte[] fileData;
 
     @Convert(converter = FileIdConverter.class)
-    @JsonSerialize(using = EntityIdSerializer.class)
     private EntityId entityId;
 
     private Integer transactionType;

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/NonFeeTransfer.java

@@ -22,7 +22,6 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonUnwrapped;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import java.io.Serializable;
 import javax.persistence.Convert;
 import javax.persistence.Embeddable;
@@ -34,7 +33,6 @@
 import org.springframework.data.domain.Persistable;
 
 import com.hedera.mirror.importer.converter.AccountIdConverter;
-import com.hedera.mirror.importer.converter.EntityIdSerializer;
 
 @Data
 @Entity
@@ -65,7 +63,6 @@ public static class Id implements Serializable {
         private long consensusTimestamp;
 
         @Convert(converter = AccountIdConverter.class)
-        @JsonSerialize(using = EntityIdSerializer.class)
         private EntityId entityId;
     }
 }

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/Schedule.java

@@ -20,7 +20,6 @@
  * ‍
  */
 
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import javax.persistence.Convert;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -29,7 +28,6 @@
 import lombok.ToString;
 
 import com.hedera.mirror.importer.converter.AccountIdConverter;
-import com.hedera.mirror.importer.converter.EntityIdSerializer;
 import com.hedera.mirror.importer.converter.ScheduleIdConverter;
 
 @Data
@@ -40,17 +38,14 @@ public class Schedule {
     private Long consensusTimestamp;
 
     @Convert(converter = AccountIdConverter.class)
-    @JsonSerialize(using = EntityIdSerializer.class)
     private EntityId creatorAccountId;
 
     private Long executedTimestamp;
 
     @Convert(converter = AccountIdConverter.class)
-    @JsonSerialize(using = EntityIdSerializer.class)
     private EntityId payerAccountId;
 
     @Convert(converter = ScheduleIdConverter.class)
-    @JsonSerialize(using = EntityIdSerializer.class)
     private EntityId scheduleId;
 
     @ToString.Exclude