diff --git a/docs/configuration.md b/docs/configuration.md index c5d3bf03efd..7af74cd2563 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -43,6 +43,7 @@ value, it is recommended to only populate overridden properties in the custom `a | `hedera.mirror.importer.downloader.balance.threads` | 15 | The number of threads to search for new files to download | | `hedera.mirror.importer.downloader.bucketName` | | The cloud storage bucket name to download streamed files. This value takes priority over network hardcoded bucket names regardless of `hedera.mirror.importer.network` value.| | `hedera.mirror.importer.downloader.cloudProvider` | S3 | The cloud provider to download files from. Either `S3` or `GCP` | +| `hedera.mirror.importer.downloader.consensusRatio` | 0.333 | The ratio of verified nodes (nodes used to come to consensus on the signature file hash) to total number of nodes available | | `hedera.mirror.importer.downloader.endpointOverride` | | Can be specified to download streams from a source other than S3 and GCP. Should be S3 compatible | | `hedera.mirror.importer.downloader.event.batchSize` | 100 | The number of signature files to download per node before downloading the signed files | | `hedera.mirror.importer.downloader.event.enabled` | false | Whether to enable event file downloads | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index dd63fe7e682..aff7ab9e8cc 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -99,7 +99,7 @@ Following is list of error messages and how to begin handling issues when they a - There is no immediate fix. Bring to team's attention immediately (during reasonable hours, otherwise next morning). -- `File could not be verified by at least 1/3 of nodes` +- `Insufficient downloaded signature file count, requires at least` This can happen if 1. Some mainnet nodes are still in the process of uploading their signatures for the latest file (benign case). @@ -249,7 +249,7 @@ Alerts: Low-Priority PagerDuty Alert during business hours only Response: Requir | `Missing signature for file` | LOW | | | `Error saving file in database` | NONE | HIGH (if 30 entries in 1 min) | | `Failed downloading` | NONE | HIGH (if 30 entries in 1 min) | -| `File could not be verified by at least 1/3 of nodes | NONE | HIGH (if 30 entries in 1 min) | +| `Insufficient downloaded signature file count, requires at least` | NONE | HIGH (if 30 entries in 1 min) | | `Signature verification failed` | NONE | HIGH (if 30 entries in 1 min) | | `Unable to connect to database` | NONE | HIGH (if 30 entries in 1 min) | | `Unable to fetch entity types` | NONE | HIGH (if 30 entries in 1 min) | diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/FileStreamSignature.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/FileStreamSignature.java index 9e084eec696..1ad95c7ebaa 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/FileStreamSignature.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/FileStreamSignature.java @@ -45,6 +45,7 @@ public class FileStreamSignature implements Comparable { private SignatureStatus status = SignatureStatus.DOWNLOADED; private byte[] metadataHash; private byte[] metadataHashSignature; + private StreamType streamType; @Override public int compareTo(FileStreamSignature other) { @@ -64,9 +65,10 @@ public String getNodeAccountIdString() { } public enum SignatureStatus { - DOWNLOADED, // Signature has been downloaded and parsed but not verified - VERIFIED, // Signature has been verified against the node's public key - CONSENSUS_REACHED // At least 1/3 of all nodes have been verified + DOWNLOADED, // Signature has been downloaded and parsed but not verified + VERIFIED, // Signature has been verified against the node's public key + CONSENSUS_REACHED, // Signature verification consensus reached by a node count greater than the consensusRatio + NOT_FOUND, // Signature for given node was not found for download } @Getter diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java index aa5f5a544bf..52ddc1fd359 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java @@ -20,6 +20,7 @@ * ‍ */ +import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import lombok.Data; @@ -46,6 +47,10 @@ public String getBucketName() { return StringUtils.isNotBlank(bucketName) ? bucketName : mirrorProperties.getNetwork().getBucketName(); } + @Max(1) + @Min(0) + private float consensusRatio = 0.333f; + @NotNull private CloudProvider cloudProvider = CloudProvider.S3; diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java index 4e5b89f219b..b19a5e513de 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java @@ -29,7 +29,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.common.collect.TreeMultimap; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import java.nio.file.Path; @@ -99,7 +98,6 @@ public abstract class Downloader { private final MeterRegistry meterRegistry; private final Timer cloudStorageLatencyMetric; private final Timer downloadLatencyMetric; - private final Counter.Builder signatureVerificationMetric; private final Timer streamCloseMetric; private final Timer.Builder streamVerificationMetric; @@ -138,10 +136,6 @@ protected Downloader(S3AsyncClient s3Client, .tag("type", streamType.toString()) .register(meterRegistry); - signatureVerificationMetric = Counter.builder("hedera.mirror.download.signature.verification") - .description("The number of signatures verified from a particular node") - .tag("type", streamType.toString()); - streamCloseMetric = Timer.builder("hedera.mirror.stream.close.latency") .description("The difference between the consensus time of the last and first transaction in the " + "stream file") @@ -313,6 +307,7 @@ private Optional parseSignatureFile(PendingDownload pending StreamFileData streamFileData = new StreamFileData(streamFilename, pendingDownload.getBytes()); FileStreamSignature fileStreamSignature = signatureFileReader.read(streamFileData); fileStreamSignature.setNodeAccountId(nodeAccountId); + fileStreamSignature.setStreamType(streamType); return Optional.of(fileStreamSignature); } @@ -395,16 +390,6 @@ private void verifySigsAndDownloadDataFiles(Multimap nodeSignatureStatusMetricMap = new ConcurrentHashMap<>(); + + public NodeSignatureVerifier(AddressBookService addressBookService, + CommonDownloaderProperties commonDownloaderProperties, + MeterRegistry meterRegistry) { + this.addressBookService = addressBookService; + this.commonDownloaderProperties = commonDownloaderProperties; + this.meterRegistry = meterRegistry; + } - private static boolean canReachConsensus(long actualNodes, long expectedNodes) { - return actualNodes >= Math.ceil(expectedNodes / 3.0); + private boolean canReachConsensus(long actualNodes, long expectedNodes) { + return actualNodes >= Math.ceil(expectedNodes * commonDownloaderProperties.getConsensusRatio()); } /** @@ -76,9 +94,14 @@ public void verify(Collection signatures) throws SignatureV long sigFileCount = signatures.size(); long nodeCount = nodeAccountIDPubKeyMap.size(); if (!canReachConsensus(sigFileCount, nodeCount)) { - throw new SignatureVerificationException("Require at least 1/3 signature files to reach consensus, got " + - sigFileCount + " out of " + nodeCount + " for file " + filename + ": " + statusMap(signatures, - nodeAccountIDPubKeyMap)); + throw new SignatureVerificationException(String.format( + "Insufficient downloaded signature file count, requires at least %.03f to reach consensus, got %d" + + " out of %d for file %s: %s", + commonDownloaderProperties.getConsensusRatio(), + sigFileCount, + nodeCount, + filename, + statusMap(signatures, nodeAccountIDPubKeyMap))); } for (FileStreamSignature fileStreamSignature : signatures) { @@ -88,6 +111,11 @@ public void verify(Collection signatures) throws SignatureV } } + if (commonDownloaderProperties.getConsensusRatio() == 0 && signatureHashMap.size() > 0) { + log.debug("Signature file {} does not require consensus, skipping consensus check", filename); + return; + } + for (String key : signatureHashMap.keySet()) { Collection validatedSignatures = signatureHashMap.get(key); @@ -153,17 +181,44 @@ private boolean verifySignature(FileStreamSignature fileStreamSignature, return false; } - private Map> statusMap(Collection signatures, Map> statusMap(Collection signatures, Map nodeAccountIDPubKeyMap) { - Map> statusMap = signatures.stream() + Map> statusMap = signatures.stream() .collect(Collectors.groupingBy(fss -> fss.getStatus().toString(), - Collectors.mapping(FileStreamSignature::getNodeAccountIdString, Collectors + Collectors.mapping(FileStreamSignature::getNodeAccountId, Collectors .toCollection(TreeSet::new)))); - Set seenNodes = signatures.stream().map(FileStreamSignature::getNodeAccountIdString) - .collect(Collectors.toSet()); - Set missingNodes = new TreeSet<>(Sets.difference(nodeAccountIDPubKeyMap.keySet(), seenNodes)); - statusMap.put("MISSING", missingNodes); - statusMap.remove(SignatureStatus.CONSENSUS_REACHED.toString()); + + Set seenNodes = new HashSet<>(); + signatures.forEach(signature -> seenNodes.add(signature.getNodeAccountId())); + + Set missingNodes = new TreeSet<>(Sets.difference( + nodeAccountIDPubKeyMap.keySet().stream().map(x -> EntityId.of(x, EntityTypeEnum.ACCOUNT)) + .collect(Collectors.toSet()), + seenNodes)); + statusMap.put(SignatureStatus.NOT_FOUND.toString(), missingNodes); + + String streamType = CollectionUtils.isEmpty(signatures) ? "unknown" : + signatures.stream().map(FileStreamSignature::getStreamType).findFirst().toString(); + for (Map.Entry> entry : statusMap.entrySet()) { + entry.getValue().forEach(nodeAccountId -> { + Counter counter = nodeSignatureStatusMetricMap.computeIfAbsent( + nodeAccountId, + n -> newStatusMetric(nodeAccountId, streamType, entry.getKey())); + counter.increment(); + }); + } + return statusMap; } + + private Counter newStatusMetric(EntityId entityId, String streamType, String status) { + return Counter.builder("hedera.mirror.download.signature.verification") + .description("The number of signatures verified from a particular node") + .tag("nodeAccount", entityId.getEntityNum().toString()) + .tag("realm", entityId.getRealmNum().toString()) + .tag("shard", entityId.getShardNum().toString()) + .tag("type", streamType) + .tag("status", status) + .register(meterRegistry); + } } diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java index 2a367e28284..0fc52b57b05 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java @@ -200,7 +200,10 @@ mirrorProperties, commonDownloaderProperties, new MetricsExecutionInterceptor(me signatureFileReader = new CompositeSignatureFileReader(new SignatureFileReaderV2(), new SignatureFileReaderV5()); - nodeSignatureVerifier = new NodeSignatureVerifier(addressBookService); + nodeSignatureVerifier = new NodeSignatureVerifier( + addressBookService, + downloaderProperties.getCommon(), + meterRegistry); downloader = getDownloader(); streamType = downloaderProperties.getStreamType(); diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/NodeSignatureVerifierTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/NodeSignatureVerifierTest.java index 03e74beb620..92314891d6d 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/NodeSignatureVerifierTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/NodeSignatureVerifierTest.java @@ -23,6 +23,8 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.when; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.logging.LoggingMeterRegistry; import java.security.GeneralSecurityException; import java.security.KeyPair; import java.security.KeyPairGenerator; @@ -48,6 +50,7 @@ import com.hedera.mirror.importer.domain.EntityTypeEnum; import com.hedera.mirror.importer.domain.FileStreamSignature; import com.hedera.mirror.importer.domain.FileStreamSignature.SignatureType; +import com.hedera.mirror.importer.domain.StreamType; import com.hedera.mirror.importer.exception.SignatureVerificationException; @ExtendWith(MockitoExtension.class) @@ -57,11 +60,15 @@ class NodeSignatureVerifierTest { private static PublicKey publicKey; private static final EntityId nodeId = new EntityId(0L, 0L, 3L, EntityTypeEnum.ACCOUNT.getId()); + private static final MeterRegistry meterRegistry = new LoggingMeterRegistry(); private Signature signer; @Mock private AddressBookService addressBookService; + @Mock + private CommonDownloaderProperties commonDownloaderProperties; + @Mock private AddressBook currentAddressBook; @@ -76,13 +83,17 @@ static void generateKeys() throws NoSuchAlgorithmException { @BeforeEach void setup() throws GeneralSecurityException { - nodeSignatureVerifier = new NodeSignatureVerifier(addressBookService); + nodeSignatureVerifier = new NodeSignatureVerifier( + addressBookService, + commonDownloaderProperties, + meterRegistry); signer = Signature.getInstance("SHA384withRSA", "SunRsaSign"); signer.initSign(privateKey); Map nodeAccountIDPubKeyMap = new HashMap(); nodeAccountIDPubKeyMap.put("0.0.3", publicKey); when(addressBookService.getCurrent()).thenReturn(currentAddressBook); when(currentAddressBook.getNodeAccountIDPubKeyMap()).thenReturn(nodeAccountIDPubKeyMap); + when(commonDownloaderProperties.getConsensusRatio()).thenReturn(0.333f); } @Test @@ -149,7 +160,52 @@ void testCannotReachConsensus() { List fileStreamSignatures = Arrays.asList(buildBareBonesFileStreamSignature()); Exception e = assertThrows(SignatureVerificationException.class, () -> nodeSignatureVerifier .verify(fileStreamSignatures)); - assertTrue(e.getMessage().contains("Require at least 1/3 signature files to reach consensus")); + assertTrue(e.getMessage().contains("Insufficient downloaded signature file count, requires at least 0.333")); + } + + @Test + void testNoConsensusRequiredWithVerifiedSignatureFiles() throws GeneralSecurityException { + Map nodeAccountIDPubKeyMap = new HashMap(); + nodeAccountIDPubKeyMap.put("0.0.3", publicKey); + nodeAccountIDPubKeyMap.put("0.0.4", publicKey); + nodeAccountIDPubKeyMap.put("0.0.5", publicKey); + nodeAccountIDPubKeyMap.put("0.0.6", publicKey); + nodeAccountIDPubKeyMap.put("0.0.7", publicKey); + nodeAccountIDPubKeyMap.put("0.0.8", publicKey); + nodeAccountIDPubKeyMap.put("0.0.9", publicKey); + nodeAccountIDPubKeyMap.put("0.0.10", publicKey); + + when(currentAddressBook.getNodeAccountIDPubKeyMap()).thenReturn(nodeAccountIDPubKeyMap); + when(commonDownloaderProperties.getConsensusRatio()).thenReturn(0f); + + byte[] fileHash = TestUtils.generateRandomByteArray(48); + byte[] fileHashSignature = signHash(fileHash); + + FileStreamSignature fileStreamSignatureNode = buildFileStreamSignature(fileHash, fileHashSignature, + null, null); + + // only 1 node node necessary + nodeSignatureVerifier.verify(List.of(fileStreamSignatureNode)); + } + + @Test + void testNoConsensusRequiredWithNoVerifiedSignatureFiles() throws GeneralSecurityException { + Map nodeAccountIDPubKeyMap = new HashMap(); + nodeAccountIDPubKeyMap.put("0.0.3", publicKey); + nodeAccountIDPubKeyMap.put("0.0.4", publicKey); + nodeAccountIDPubKeyMap.put("0.0.5", publicKey); + nodeAccountIDPubKeyMap.put("0.0.6", publicKey); + nodeAccountIDPubKeyMap.put("0.0.7", publicKey); + nodeAccountIDPubKeyMap.put("0.0.8", publicKey); + nodeAccountIDPubKeyMap.put("0.0.9", publicKey); + nodeAccountIDPubKeyMap.put("0.0.10", publicKey); + + when(currentAddressBook.getNodeAccountIDPubKeyMap()).thenReturn(nodeAccountIDPubKeyMap); + when(commonDownloaderProperties.getConsensusRatio()).thenReturn(0f); + + Exception e = assertThrows(SignatureVerificationException.class, () -> nodeSignatureVerifier + .verify(List.of())); + assertTrue(e.getMessage().contains("Signature verification failed for file")); } @Test @@ -163,6 +219,22 @@ void testVerifiedWithOneThirdConsensus() throws GeneralSecurityException { byte[] fileHash = TestUtils.generateRandomByteArray(48); byte[] fileHashSignature = signHash(fileHash); + nodeSignatureVerifier + .verify(Arrays.asList(buildFileStreamSignature(fileHash, fileHashSignature, + null, null))); + } + + @Test + void testVerifiedWithOneThirdConsensusWithMissingSignatures() throws GeneralSecurityException { + Map nodeAccountIDPubKeyMap = new HashMap(); + nodeAccountIDPubKeyMap.put("0.0.3", publicKey); + nodeAccountIDPubKeyMap.put("0.0.4", publicKey); + nodeAccountIDPubKeyMap.put("0.0.5", publicKey); + when(currentAddressBook.getNodeAccountIDPubKeyMap()).thenReturn(nodeAccountIDPubKeyMap); + + byte[] fileHash = TestUtils.generateRandomByteArray(48); + byte[] fileHashSignature = signHash(fileHash); + FileStreamSignature fileStreamSignatureNode3 = buildFileStreamSignature(fileHash, fileHashSignature, null, null); @@ -172,9 +244,38 @@ void testVerifiedWithOneThirdConsensus() throws GeneralSecurityException { fileStreamSignatureNode4.setNodeAccountId(new EntityId(0L, 0L, 4L, EntityTypeEnum.ACCOUNT.getId())); FileStreamSignature fileStreamSignatureNode5 = buildFileStreamSignature(fileHash, null, null, null); - fileStreamSignatureNode4.setNodeAccountId(new EntityId(0L, 0L, 5L, EntityTypeEnum.ACCOUNT.getId())); + fileStreamSignatureNode5.setNodeAccountId(new EntityId(0L, 0L, 5L, EntityTypeEnum.ACCOUNT.getId())); + + nodeSignatureVerifier + .verify(Arrays.asList(fileStreamSignatureNode3, fileStreamSignatureNode4, fileStreamSignatureNode5)); + } + + @Test + void testVerifiedWithFullConsensusRequired() throws GeneralSecurityException { + Map nodeAccountIDPubKeyMap = new HashMap(); + nodeAccountIDPubKeyMap.put("0.0.3", publicKey); + nodeAccountIDPubKeyMap.put("0.0.4", publicKey); + nodeAccountIDPubKeyMap.put("0.0.5", publicKey); + when(currentAddressBook.getNodeAccountIDPubKeyMap()).thenReturn(nodeAccountIDPubKeyMap); + when(commonDownloaderProperties.getConsensusRatio()).thenReturn(1f); + + byte[] fileHash = TestUtils.generateRandomByteArray(48); + byte[] fileHashSignature = signHash(fileHash); + + FileStreamSignature fileStreamSignatureNode3 = buildFileStreamSignature(fileHash, fileHashSignature, + null, null); + fileStreamSignatureNode3.setNodeAccountId(new EntityId(0L, 0L, 3L, EntityTypeEnum.ACCOUNT.getId())); + + FileStreamSignature fileStreamSignatureNode4 = buildFileStreamSignature(fileHash, fileHashSignature, + null, null); + fileStreamSignatureNode4.setNodeAccountId(new EntityId(0L, 0L, 4L, EntityTypeEnum.ACCOUNT.getId())); + + FileStreamSignature fileStreamSignatureNode5 = buildFileStreamSignature(fileHash, fileHashSignature, + null, null); + fileStreamSignatureNode5.setNodeAccountId(new EntityId(0L, 0L, 5L, EntityTypeEnum.ACCOUNT.getId())); - nodeSignatureVerifier.verify(Arrays.asList(fileStreamSignatureNode3, fileStreamSignatureNode5)); + nodeSignatureVerifier + .verify(Arrays.asList(fileStreamSignatureNode3, fileStreamSignatureNode4, fileStreamSignatureNode5)); } @Test @@ -259,6 +360,7 @@ private FileStreamSignature buildBareBonesFileStreamSignature() { fileStreamSignature.setFilename(""); fileStreamSignature.setNodeAccountId(nodeId); fileStreamSignature.setSignatureType(SignatureType.SHA_384_WITH_RSA); + fileStreamSignature.setStreamType(StreamType.RECORD); return fileStreamSignature; }