From eed93fa5464a0d193a6a0a20446b391d69b94fce Mon Sep 17 00:00:00 2001 From: Steven Sheehy <17552371+steven-sheehy@users.noreply.github.com> Date: Fri, 4 Feb 2022 10:41:15 -0600 Subject: [PATCH] Add a property to control persistence of topic messages (#3255) * Add a property to control persistence of topic messages * Change Rosetta single image to disable persistence of topic messages by default Signed-off-by: Steven Sheehy Signed-off-by: Matheus DallRosa --- docs/configuration.md | 1 + .../record/entity/EntityProperties.java | 2 ++ .../entity/EntityRecordItemListener.java | 4 +++ .../EntityRecordItemListenerTopicTest.java | 34 +++++++++++++++++++ hedera-mirror-rosetta/build/supervisord.conf | 2 +- 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index c5d78c41cf9..21307ea50f2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -117,6 +117,7 @@ value, it is recommended to only populate overridden properties in the custom `a | `hedera.mirror.importer.parser.record.entity.persist.schedules` | true | Persist schedule transactions to the database | | `hedera.mirror.importer.parser.record.entity.persist.systemFiles` | true | Persist only system files (number lower than `1000`) to the database | | `hedera.mirror.importer.parser.record.entity.persist.tokens` | true | Persist token data to the database | +| `hedera.mirror.importer.parser.record.entity.persist.topics` | true | Persist topic messages to the database | | `hedera.mirror.importer.parser.record.entity.persist.transactionBytes` | false | Persist raw transaction bytes to the database | | `hedera.mirror.importer.parser.record.entity.persist.transactionSignatures` | SCHEDULECREATE, SCHEDULESIGN | A list of transaction types whose transaction signatures will be stored | | `hedera.mirror.importer.parser.record.entity.redis.enabled` | true | Whether to use Redis to send messages to the gRPC process. Requires `spring.redis.*` [properties](https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#data-properties) | diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityProperties.java index d4475f08e9d..56e695f9191 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityProperties.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityProperties.java @@ -57,6 +57,8 @@ public class PersistProperties { private boolean tokens = true; + private boolean topics = true; + /** * If configured the mirror node will store the raw transaction bytes on the transaction table */ diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListener.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListener.java index 166fd16f83c..505937bc7e0 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListener.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListener.java @@ -316,6 +316,10 @@ private EntityId getAccountId(AccountID accountID) { } private void insertConsensusTopicMessage(RecordItem recordItem) { + if (!entityProperties.getPersist().isTopics()) { + return; + } + ConsensusSubmitMessageTransactionBody transactionBody = recordItem.getTransactionBody() .getConsensusSubmitMessage(); TransactionRecord transactionRecord = recordItem.getRecord(); diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListenerTopicTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListenerTopicTest.java index 115714122f3..d8144b9106b 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListenerTopicTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/EntityRecordItemListenerTopicTest.java @@ -453,6 +453,40 @@ void submitMessageTestTopicNotFound() { .isEqualTo(topicMessage); } + @Test + void submitMessageDisabled() { + // given + entityProperties.getPersist().setTopics(false); + var responseCode = SUCCESS; + var topicId = TOPIC_ID; + var consensusTimestamp = 10_000_000L; + var message = "message"; + var sequenceNumber = 10_000L; + var runningHash = "running-hash"; + var runningHashVersion = 1; + var chunkNum = 3; + var chunkTotal = 5; + var validStartNs = 7L; + var scheduled = false; + var nonce = 0; + + TransactionID initialTransactionId = createTransactionID(PAYER_ACCOUNT_ID.getEntityNum(), + TestUtils.toTimestamp(validStartNs), scheduled, nonce); + + var transaction = createSubmitMessageTransaction(topicId, message, chunkNum, chunkTotal, initialTransactionId); + var transactionRecord = createTransactionRecord(topicId, sequenceNumber, runningHash + .getBytes(), runningHashVersion, consensusTimestamp, responseCode); + + // when + parseRecordItemAndCommit(new RecordItem(transaction, transactionRecord)); + + // then + assertEquals(0L, entityRepository.count()); + assertEquals(0L, topicMessageRepository.count()); + assertTransactionInRepository(responseCode, consensusTimestamp, TOPIC_ID.getTopicNum()); + entityProperties.getPersist().setTopics(true); + } + @Test void submitMessageTestFiltered() { // given diff --git a/hedera-mirror-rosetta/build/supervisord.conf b/hedera-mirror-rosetta/build/supervisord.conf index 88b56e9728e..08b3a7eba30 100644 --- a/hedera-mirror-rosetta/build/supervisord.conf +++ b/hedera-mirror-rosetta/build/supervisord.conf @@ -27,7 +27,7 @@ redirect_stderr=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 priority=10 -environment=HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_PERSIST_NONFEETRANSFERS=true,HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_REDIS_ENABLED=false,HEDERA_MIRROR_IMPORTER_STARTDATE=1970-01-01T00:00:00Z +environment=HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_PERSIST_NONFEETRANSFERS=true,HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_PERSIST_TOPICS=false,HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_REDIS_ENABLED=false,HEDERA_MIRROR_IMPORTER_STARTDATE=1970-01-01T00:00:00Z [program:rosetta] command=/app/rosetta/hedera-mirror-rosetta