Cleanup record access

hedera-node/src/main/java/com/hedera/services/context/properties/BootstrapProperties.java

@@ -240,6 +240,7 @@ public Set<String> allPropertyNames() {
 			"ledger.transfers.maxLen",
 			"ledger.tokenTransfers.maxLen",
 			"ledger.nftTransfers.maxLen",
+			"ledger.records.maxQueryableByAccount",
 			"ledger.schedule.txExpiryTimeSecs",
 			"rates.intradayChangeLimitPercent",
 			"rates.midnightCheckInterval",
@@ -383,6 +384,7 @@ public static Function<String, Object> transformFor(String prop) {
 			entry("ledger.nftTransfers.maxLen", AS_INT),
 			entry("ledger.totalTinyBarFloat", AS_LONG),
 			entry("ledger.schedule.txExpiryTimeSecs", AS_INT),
+			entry("ledger.records.maxQueryableByAccount", AS_INT),
 			entry("iss.dumpFcms", AS_BOOLEAN),
 			entry("iss.resetPeriod", AS_INT),
 			entry("iss.roundsToDump", AS_INT),

hedera-node/src/main/java/com/hedera/services/context/properties/GlobalDynamicProperties.java

@@ -96,6 +96,7 @@ public class GlobalDynamicProperties {
 	private boolean expandSigsFromLastSignedState;
 	private long maxAggregateContractKvPairs;
 	private int maxIndividualContractKvPairs;
+	private int maxMostRecentQueryableRecords;
 
 	@Inject
 	public GlobalDynamicProperties(
@@ -174,6 +175,7 @@ public void reload() {
 		expandSigsFromLastSignedState = properties.getBooleanProperty("sigs.expandFromLastSignedState");
 		maxAggregateContractKvPairs = properties.getLongProperty("contracts.maxKvPairs.aggregate");
 		maxIndividualContractKvPairs = properties.getIntProperty("contracts.maxKvPairs.individual");
+		maxMostRecentQueryableRecords = properties.getIntProperty("ledger.records.maxQueryableByAccount");
 	}
 
 	public int maxTokensPerAccount() {
@@ -403,4 +405,8 @@ public long maxAggregateContractKvPairs() {
 	public int maxIndividualContractKvPairs() {
 		return maxIndividualContractKvPairs;
 	}
+
+	public int maxNumQueryableRecords() {
+		return maxMostRecentQueryableRecords;
+	}
 }

hedera-node/src/main/java/com/hedera/services/fees/calculation/crypto/queries/GetAccountRecordsResourceUsage.java

@@ -28,11 +28,14 @@
 import com.hederahashgraph.api.proto.java.ResponseType;
 import com.hederahashgraph.fee.CryptoFeeBuilder;
 
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.util.Map;
 
+import static com.hedera.services.queries.meta.GetTxnRecordAnswer.PAYER_RECORDS_CTX_KEY;
 import static com.hedera.services.utils.EntityNum.fromAccountId;
+import static com.hedera.services.utils.MiscUtils.putIfNotNull;
 
 @Singleton
 public final class GetAccountRecordsResourceUsage implements QueryResourceUsageEstimator {
@@ -54,12 +57,21 @@ public boolean applicableTo(final Query query) {
 	}
 
 	@Override
-	public FeeData usageGiven(final Query query, final StateView view, final Map<String, Object> ignoreCtx) {
-		return usageGivenType(query, view, query.getCryptoGetAccountRecords().getHeader().getResponseType());
+	public FeeData usageGiven(final Query query, final StateView view, final Map<String, Object> queryCtx) {
+		return usageFor(query, view, query.getCryptoGetAccountRecords().getHeader().getResponseType(), queryCtx);
 	}
 
 	@Override
 	public FeeData usageGivenType(final Query query, final StateView view, final ResponseType type) {
+		return usageFor(query, view, type, null);
+	}
+
+	private FeeData usageFor(
+			final Query query,
+			final StateView view,
+			final ResponseType stateProofType,
+			@Nullable final Map<String, Object> queryCtx
+	) {
 		final var op = query.getCryptoGetAccountRecords();
 		final var target = fromAccountId(op.getAccountID());
 		if (!view.accounts().containsKey(target)) {
@@ -69,7 +81,8 @@ public FeeData usageGivenType(final Query query, final StateView view, final Res
 			 * status code); so just return the default {@code FeeData} here. */
 			return FeeData.getDefaultInstance();
 		}
-		final var records = answerFunctions.accountRecords(view, op);
-		return usageEstimator.getCryptoAccountRecordsQueryFeeMatrices(records, type);
+		final var records = answerFunctions.mostRecentRecords(view, op);
+		putIfNotNull(queryCtx, PAYER_RECORDS_CTX_KEY, records);
+		return usageEstimator.getCryptoAccountRecordsQueryFeeMatrices(records, stateProofType);
 	}
 }

hedera-node/src/main/java/com/hedera/services/fees/calculation/crypto/queries/GetTxnRecordResourceUsage.java

@@ -71,24 +71,21 @@ public boolean applicableTo(final Query query) {
 
 	@Override
 	public FeeData usageGiven(final Query query, final StateView view, @Nullable final Map<String, Object> queryCtx) {
-		return usageFor(query, view, query.getTransactionGetRecord().getHeader().getResponseType(), queryCtx);
+		return usageFor(query, query.getTransactionGetRecord().getHeader().getResponseType(), queryCtx);
 	}
 
 	@Override
 	public FeeData usageGivenType(final Query query, final StateView view, final ResponseType type) {
-		return usageFor(query, view, type, null);
+		return usageFor(query, type, null);
 	}
 
 	private FeeData usageFor(
 			final Query query,
-			final StateView view,
 			final ResponseType stateProofType,
 			@Nullable final Map<String, Object> queryCtx
 	) {
 		final var op = query.getTransactionGetRecord();
-		final var txnRecord = answerFunctions
-				.txnRecord(recordCache, view, op)
-				.orElse(MISSING_RECORD_STANDIN);
+		final var txnRecord = answerFunctions.txnRecord(recordCache, op).orElse(MISSING_RECORD_STANDIN);
 		var usages = usageEstimator.getTransactionRecordQueryFeeMatrices(txnRecord, stateProofType);
 		if (txnRecord != MISSING_RECORD_STANDIN) {
 			putIfNotNull(queryCtx, PRIORITY_RECORD_CTX_KEY, txnRecord);

hedera-node/src/main/java/com/hedera/services/queries/answering/AnswerFunctions.java

@@ -21,54 +21,110 @@
  */
 
 import com.hedera.services.context.primitives.StateView;
+import com.hedera.services.context.properties.GlobalDynamicProperties;
 import com.hedera.services.records.RecordCache;
+import com.hedera.services.state.merkle.MerkleAccount;
 import com.hedera.services.state.submerkle.ExpirableTxnRecord;
-import com.hedera.services.state.submerkle.TxnId;
 import com.hedera.services.utils.EntityNum;
 import com.hederahashgraph.api.proto.java.CryptoGetAccountRecordsQuery;
 import com.hederahashgraph.api.proto.java.TransactionGetRecordQuery;
 import com.hederahashgraph.api.proto.java.TransactionRecord;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 
 @Singleton
 public class AnswerFunctions {
+	private final GlobalDynamicProperties dynamicProperties;
+
 	@Inject
-	public AnswerFunctions() {
-		// Default Constructor
+	public AnswerFunctions(GlobalDynamicProperties dynamicProperties) {
+		this.dynamicProperties = dynamicProperties;
 	}
 
-	public List<TransactionRecord> accountRecords(final StateView view, final CryptoGetAccountRecordsQuery op) {
-		final var key = EntityNum.fromAccountId(op.getAccountID());
-		final var account = view.accounts().get(key);
-		return ExpirableTxnRecord.allToGrpc(account.recordList());
+	/**
+	 * Returns the most recent payer records available for an account in the given {@link StateView}.
+	 *
+	 * Note that at <b>most</b> {@link GlobalDynamicProperties#maxNumQueryableRecords()} records will be available,
+	 * even if the given account has paid for more than this number of transactions in the last 180 seconds.
+	 *
+	 * @param view the view of the world state to get payer records from
+	 * @param op the query with the target payer account
+	 * @return the most recent available records for the given payer
+	 */
+	public List<TransactionRecord> mostRecentRecords(final StateView view, final CryptoGetAccountRecordsQuery op) {
+		final var targetId = EntityNum.fromAccountId(op.getAccountID());
+		final var targetAccount = view.accounts().get(targetId);
+		if (targetAccount == null) {
+			return Collections.emptyList();
+		}
+		final var numAvailable = targetAccount.numRecords();
+		final var maxQueryable = dynamicProperties.maxNumQueryableRecords();
+		return numAvailable <= maxQueryable
+				? recordsFrom(targetAccount, numAvailable)
+				: mostRecentFrom(targetAccount, maxQueryable, numAvailable);
 	}
 
-	public Optional<TransactionRecord> txnRecord(
-			final RecordCache recordCache,
-			final StateView view,
-			final TransactionGetRecordQuery query
-	) {
-		final var txnId = query.getTransactionID();
+	/**
+	 * Returns the record of the requested transaction from the given {@link RecordCache}, if available.
+	 *
+	 * @param recordCache the cache to get the record from
+	 * @param op the query with the target transaction id
+	 * @return the transaction record if available
+	 */
+	public Optional<TransactionRecord> txnRecord(final RecordCache recordCache, final TransactionGetRecordQuery op) {
+		final var txnId = op.getTransactionID();
 		final var expirableTxnRecord = recordCache.getPriorityRecord(txnId);
-		if (expirableTxnRecord != null) {
-			return Optional.of(expirableTxnRecord.asGrpc());
-		} else {
-			try {
-				final var id = txnId.getAccountID();
-				final var account = view.accounts().get(EntityNum.fromAccountId(id));
-				final var searchableId = TxnId.fromGrpc(txnId);
-				return account.recordList()
-						.stream()
-						.filter(r -> r.getTxnId().equals(searchableId))
-						.findAny()
-						.map(ExpirableTxnRecord::asGrpc);
-			} catch (final Exception ignore) {
-				return Optional.empty();
+		return Optional.ofNullable(expirableTxnRecord).map(ExpirableTxnRecord::asGrpc);
+	}
+
+	/* --- Internal helpers --- */
+	/**
+	 * Returns up to {@code n} payer records from an account in gRPC form.
+	 *
+	 * @param account the account of interest
+	 * @param n the expected number of records in the account
+	 * @return the available records
+	 */
+	private List<TransactionRecord> recordsFrom(final MerkleAccount account, final int n) {
+		return mostRecentFrom(account, n, n);
+	}
+
+	/**
+	 * Returns up to the last {@code m}-of-{@code n} payer records from an account in gRPC form.
+	 *
+	 * Since records are added FIFO to the payer account, the last records are the most recent;
+	 * and presumably the most interesting.
+	 *
+	 * If the given {@link MerkleAccount} is from the working state (as in release 0.22), then
+	 * this method acts on a best-effort basis, and returns only the relevant records it could
+	 * iterate over before hitting a {@link ConcurrentModificationException} or {@link NoSuchElementException}.
+	 *
+	 * @param account the account of interest
+	 * @param m the maximum number of records to return
+	 * @param n the expected number of records in the account
+	 * @return the available records
+	 */
+	private List<TransactionRecord> mostRecentFrom(final MerkleAccount account, final int m, final int n) {
+		final List<TransactionRecord> ans = new ArrayList<>();
+		final Iterator<ExpirableTxnRecord> iter = account.recordIterator();
+		try {
+			for (int i = 0, cutoff = n - m; i < n; i++) {
+				final var nextRecord = iter.next();
+				if (i >= cutoff) {
+					ans.add(nextRecord.asGrpc());
+				}
 			}
+		} catch (ConcurrentModificationException | NoSuchElementException ignore) {
+			/* Records expired while we were iterating the list, return only what we could find. */
 		}
+		return ans;
 	}
 }

hedera-node/src/main/java/com/hedera/services/queries/crypto/GetAccountRecordsAnswer.java

@@ -25,16 +25,22 @@
 import com.hedera.services.queries.answering.AnswerFunctions;
 import com.hedera.services.txns.validation.OptionValidator;
 import com.hedera.services.utils.SignedTxnAccessor;
+import com.hederahashgraph.api.proto.java.CryptoGetAccountRecordsQuery;
 import com.hederahashgraph.api.proto.java.CryptoGetAccountRecordsResponse;
 import com.hederahashgraph.api.proto.java.HederaFunctionality;
 import com.hederahashgraph.api.proto.java.Query;
 import com.hederahashgraph.api.proto.java.Response;
 import com.hederahashgraph.api.proto.java.ResponseCodeEnum;
+import com.hederahashgraph.api.proto.java.TransactionRecord;
 
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Singleton;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
+import static com.hedera.services.queries.meta.GetTxnRecordAnswer.PAYER_RECORDS_CTX_KEY;
 import static com.hederahashgraph.api.proto.java.HederaFunctionality.CryptoGetAccountRecords;
 import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.OK;
 import static com.hederahashgraph.api.proto.java.ResponseType.COST_ANSWER;
@@ -69,6 +75,27 @@ public Response responseGiven(
 			final StateView view,
 			final ResponseCodeEnum validity,
 			final long cost
+	) {
+		return responseFor(query, view, validity, cost, null);
+	}
+
+	@Override
+	public Response responseGiven(
+			final Query query,
+			final StateView view,
+			final ResponseCodeEnum validity,
+			final long cost,
+			final Map<String, Object> queryCtx
+	) {
+		return responseFor(query, view, validity, cost, queryCtx);
+	}
+
+	private Response responseFor(
+			final Query query,
+			final StateView view,
+			final ResponseCodeEnum validity,
+			final long cost,
+			final @Nullable Map<String, Object> queryCtx
 	) {
 		final var op = query.getCryptoGetAccountRecords();
 		final var response = CryptoGetAccountRecordsResponse.newBuilder();
@@ -81,9 +108,7 @@ public Response responseGiven(
 				response.setAccountID(op.getAccountID());
 				response.setHeader(costAnswerHeader(OK, cost));
 			} else {
-				response.setHeader(answerOnlyHeader(OK));
-				response.setAccountID(op.getAccountID());
-				response.addAllRecords(answerFunctions.accountRecords(view, op));
+				setAnswerOnly(response, view, op, queryCtx);
 			}
 		}
 
@@ -92,6 +117,22 @@ public Response responseGiven(
 				.build();
 	}
 
+	@SuppressWarnings("unchecked")
+	private void setAnswerOnly(
+			final CryptoGetAccountRecordsResponse.Builder response,
+			final StateView view,
+			final CryptoGetAccountRecordsQuery op,
+			final @Nullable Map<String, Object> queryCtx
+	) {
+		response.setHeader(answerOnlyHeader(OK));
+		response.setAccountID(op.getAccountID());
+		if (queryCtx != null && queryCtx.containsKey(PAYER_RECORDS_CTX_KEY)) {
+			response.addAllRecords((List<TransactionRecord>) queryCtx.get(PAYER_RECORDS_CTX_KEY));
+		} else {
+			response.addAllRecords(answerFunctions.mostRecentRecords(view, op));
+		}
+	}
+
 	@Override
 	public ResponseCodeEnum checkValidity(final Query query, final StateView view) {
 		final var id = query.getCryptoGetAccountRecords().getAccountID();
@@ -112,6 +153,6 @@ public HederaFunctionality canonicalFunction() {
 	@Override
 	public Optional<SignedTxnAccessor> extractPaymentFrom(final Query query) {
 		final var paymentTxn = query.getCryptoGetAccountRecords().getHeader().getPayment();
-		return Optional.ofNullable(SignedTxnAccessor.uncheckedFrom(paymentTxn));
+		return Optional.of(SignedTxnAccessor.uncheckedFrom(paymentTxn));
 	}
 }