Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add canonical test scenarios to RecordFileParserPerformanceTest #8855

Merged
merged 2 commits into from
Jul 24, 2024
Merged

Add canonical test scenarios to RecordFileParserPerformanceTest #8855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de99a08
Add canonical test scenarios to RecordFileParserPerformanceTest
steven-sheehy Jul 23, 2024
c6e7693
Code smells
steven-sheehy Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ value, it is recommended to only populate overridden properties in the custom `a
| `hedera.mirror.importer.parser.record.entity.persist.pendingReward` | true | Calculate pending reward and update entity stake state |
| `hedera.mirror.importer.parser.record.entity.persist.schedules` | true | Persist schedule transactions to the database |
| `hedera.mirror.importer.parser.record.entity.persist.syntheticContractLogs` | true | Persist synthetic contract logs from HAPI transaction to the database |
| `hedera.mirror.importer.parser.record.entity.persist.syntheticContractResults` | true | Persist synthetic contract results from HAPI transaction to the database |
| `hedera.mirror.importer.parser.record.entity.persist.syntheticContractResults` | false | Persist synthetic contract results from HAPI transaction to the database |
| `hedera.mirror.importer.parser.record.entity.persist.systemFiles` | true | Persist only system files (number lower than `1000`) to the database |
| `hedera.mirror.importer.parser.record.entity.persist.tokens` | true | Persist token data to the database |
| `hedera.mirror.importer.parser.record.entity.persist.topics` | true | Persist topic messages to the database |
Expand Down
1 change: 1 addition & 0 deletions hedera-mirror-importer/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ dependencies {
testImplementation("com.redis.testcontainers:testcontainers-redis-junit-jupiter")
testImplementation("commons-beanutils:commons-beanutils")
testImplementation("io.projectreactor:reactor-test")
testImplementation("org.apache.commons:commons-math3")
testImplementation("org.awaitility:awaitility")
testImplementation("org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api") // Used by s3proxy
testImplementation("org.gaul:s3proxy")
Expand Down
7 changes: 6 additions & 1 deletion ...src/main/java/com/hedera/mirror/importer/downloader/provider/LocalStreamFileProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,12 @@ else if (pathTypeProp != pathType && pathTypeProp != PathType.AUTO) {
*/
private Flux<Path> getBasePaths(StreamFilename streamFilename) {
var basePath = properties.getImporterProperties().getStreamPath();
basePath.toFile().mkdirs();
var baseFile = basePath.toFile();
baseFile.mkdirs();

if (!baseFile.exists()) {
return Flux.error(new RuntimeException("Unable to create directory: " + basePath));
}

try (var subDirs = Files.list(basePath)) {
var date = LocalDate.ofInstant(streamFilename.getInstant(), ZoneOffset.UTC)
Expand Down
4 changes: 2 additions & 2 deletions ...rror-importer/src/test/java/com/hedera/mirror/importer/migration/ErrataMigrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ void migrateMainnet() throws Exception {
errataMigration.doMigrate();

assertBalanceOffsets(EXPECTED_ACCOUNT_BALANCE_FILES);
assertThat(contractResultRepository.count()).isEqualTo(13L);
assertThat(contractResultRepository.count()).isEqualTo(1L);
assertThat(entityRepository.count()).isZero();
assertThat(findHistory(Entity.class)).isEmpty();
assertThat(tokenTransferRepository.count()).isEqualTo(24L);
Expand Down Expand Up @@ -196,7 +196,7 @@ void migrateIdempotency() throws Exception {
assertErrataTransactions(ErrataType.DELETE, 0);
assertErrataTransfers(ErrataType.INSERT, 566);
assertErrataTransfers(ErrataType.DELETE, 6);
assertThat(contractResultRepository.count()).isEqualTo(13L);
assertThat(contractResultRepository.count()).isEqualTo(1L);
assertThat(entityRepository.count()).isZero();
assertThat(findHistory(Entity.class)).isEmpty();
assertThat(tokenTransferRepository.count()).isEqualTo(24L);
Expand Down
90 changes: 63 additions & 27 deletions ...or-importer/src/test/java/com/hedera/mirror/importer/parser/domain/RecordFileBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,26 @@
import com.hedera.mirror.common.domain.transaction.RecordItem;
import com.hedera.mirror.common.domain.transaction.TransactionType;
import com.hedera.mirror.importer.domain.StreamFilename;
import com.hedera.mirror.importer.util.Utility;
import com.hedera.mirror.importer.parser.domain.RecordItemBuilder.TransferType;
import com.hedera.mirror.importer.parser.record.ParserPerformanceProperties.SubType;
import jakarta.inject.Named;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Supplier;
import lombok.AccessLevel;
import lombok.CustomLog;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.springframework.util.Assert;

/**
* Generates record files using pre-defined templates for bulk creation of record items.
*/
@CustomLog
@Named
@RequiredArgsConstructor
public class RecordFileBuilder {
Expand Down Expand Up @@ -70,7 +73,8 @@ public RecordFile build() {
recordFile.customize(r -> r.items(recordItems));

if (previous != null) {
recordItemBuilder.reset(Instant.ofEpochSecond(0, previous.getConsensusStart() + CLOSE_INTERVAL));
var now = Instant.ofEpochSecond(0, previous.getConsensusStart() + CLOSE_INTERVAL);
recordItemBuilder.setNow(now);
}

for (var itemBuilder : itemBuilders) {
Expand Down Expand Up @@ -107,7 +111,7 @@ public Builder previous(RecordFile recordFile) {
return this;
}

public Builder recordItem(RecordItemBuilder.Builder<?> recordItem) {
public Builder recordItem(Supplier<RecordItemBuilder.Builder<?>> recordItem) {
return recordItems(i -> i.count(1).entities(1).template(recordItem));
}

Expand All @@ -125,13 +129,14 @@ public Builder recordItems(Consumer<ItemBuilder> recordItems) {

public class ItemBuilder {

private final AtomicLong id = new AtomicLong(0L);
private int count = 100;
private int entities = 10;
private RecordItemBuilder.Builder<?> template;
private TransactionType type = TransactionType.CRYPTOTRANSFER;
private boolean entityAutoCreation = false;
private SubType subType = SubType.STANDARD;
private TransactionType type = TransactionType.UNKNOWN;
private Supplier<RecordItemBuilder.Builder<?>> template;

@Getter(lazy = true)
@Getter(lazy = true, value = AccessLevel.PRIVATE)
private final List<RecordItemBuilder.Builder<?>> builders = createBuilders();

private ItemBuilder() {}
Expand All @@ -148,13 +153,26 @@ public ItemBuilder entities(int entities) {
return this;
}

public ItemBuilder template(RecordItemBuilder.Builder<?> template) {
public ItemBuilder entityAutoCreation(boolean entityAutoCreation) {
this.entityAutoCreation = entityAutoCreation;
return this;
}

public ItemBuilder subType(SubType subType) {
Assert.notNull(subType, "subType must not be null");
this.subType = subType;
return this;
}

public ItemBuilder template(Supplier<RecordItemBuilder.Builder<?>> template) {
Assert.notNull(template, "template must not be null");
this.template = template;
return this;
}

public ItemBuilder type(TransactionType type) {
Assert.notNull(type, "type must not be null");
Assert.isTrue(type != TransactionType.UNKNOWN, "type must not be unknown");
this.type = type;
return this;
}
Expand All @@ -163,28 +181,56 @@ private Supplier<RecordItem> build() {
var recordItemBuilders = getBuilders();
int builderSize = recordItemBuilders.size();
var counter = new AtomicInteger(count);
var creates = new ArrayDeque<>(recordItemBuilder.getCreateTransactions());

return () -> {
int count = counter.getAndDecrement();
if (count <= 0) {
if (entityAutoCreation && !creates.isEmpty()) {
var builder = creates.remove();
var recordItem = builder.build();
log.info("Creating {}", TransactionType.of(recordItem.getTransactionType()));
return recordItem;
}

int remaining = counter.getAndDecrement();
if (remaining <= 0) {
return null;
}

var index = count % builderSize;
var index = remaining % builderSize;
var builder = recordItemBuilders.get(index);
return wrap(builder);
return builder.build();
};
}

private Supplier<RecordItemBuilder.Builder<?>> buildTemplate() {
if (template != null) {
return template;
}

if (subType != SubType.STANDARD) {
return switch (subType) {
case TOKEN_TRANSFER -> () -> recordItemBuilder.cryptoTransfer(TransferType.TOKEN);
default -> throw new IllegalArgumentException("subType not supported: " + subType);
};
}

if (type == TransactionType.UNKNOWN) {
throw new IllegalArgumentException("type must not be unknown");
}

return recordItem(type);
}

private List<RecordItemBuilder.Builder<?>> createBuilders() {
List<RecordItemBuilder.Builder<?>> builders = new ArrayList<>();
List<RecordItemBuilder.Builder<?>> builderList = new ArrayList<>();
int maxEntities = Math.min(entities, count);
var builderTemplate = buildTemplate();

for (int i = 0; i < maxEntities; i++) {
builders.add(template != null ? template : recordItem(type).get());
builderList.add(builderTemplate.get());
}

return builders;
return builderList;
}

private Supplier<RecordItemBuilder.Builder<?>> recordItem(TransactionType transactionType) {
Expand All @@ -194,15 +240,5 @@ private Supplier<RecordItemBuilder.Builder<?>> recordItem(TransactionType transa
}
return supplier;
}

private RecordItem wrap(RecordItemBuilder.Builder<?> builder) {
var consensusTimestamp = recordItemBuilder.timestamp(ChronoUnit.NANOS);
var validStart = Utility.instantToTimestamp(
Utility.convertToInstant(consensusTimestamp).minusNanos(10));
return builder.record(r -> r.setConsensusTimestamp(consensusTimestamp))
.receipt(r -> r.setTopicSequenceNumber(id.getAndIncrement()))
.transactionBodyWrapper(tb -> tb.getTransactionIDBuilder().setTransactionValidStart(validStart))
.build();
}
}
}
Loading
Loading