From 320020dc841e957238c6c8fd3207e16ee58b8a7f Mon Sep 17 00:00:00 2001 From: Sergii Mikhtoniuk Date: Mon, 7 Oct 2024 17:26:22 -0700 Subject: [PATCH] Add query commitments example --- examples/query-commitments/README.md | 17 ++ examples/query-commitments/example.py | 266 +++++++++++++++++++++ examples/query-commitments/requirements.in | 4 + examples/reth-vs-snp500/init-odf-all.sh | 23 ++ examples/reth-vs-snp500/init-odf.sh | 17 ++ 5 files changed, 327 insertions(+) create mode 100644 examples/query-commitments/README.md create mode 100755 examples/query-commitments/example.py create mode 100644 examples/query-commitments/requirements.in create mode 100755 examples/reth-vs-snp500/init-odf-all.sh create mode 100755 examples/reth-vs-snp500/init-odf.sh diff --git a/examples/query-commitments/README.md b/examples/query-commitments/README.md new file mode 100644 index 000000000..1b83763ae --- /dev/null +++ b/examples/query-commitments/README.md @@ -0,0 +1,17 @@ +# Examples of Verifiable Batch Queries +Query proofs allow you to hold any ODF node forever accountable for the result it provided you, no matter how much time had passed. + +See [batch query commitments documentation](https://docs.kamu.dev/node/commitments/) and [REST API reference](https://docs.kamu.dev/node/protocols/rest-api/) for the overview of this mechanism. + +The included script illustrates: +- how to query data and receive a cryptographic proof +- how to validate commitment consistency on the client side +- and how to ask another node to verify the commitment by reproducing the query. + +Running: +```sh +pip install -r requirements.in +python ./example.py --node-url https://node.example.com --private-key +``` + +Private key is used to show examples of failed verification (by forging a signature). If you don't provide it - those cases will be skipped. diff --git a/examples/query-commitments/example.py b/examples/query-commitments/example.py new file mode 100755 index 000000000..5a2b78e6e --- /dev/null +++ b/examples/query-commitments/example.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +import argparse +import copy +import canonicaljson +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +import hashlib +import requests +import json +import base64 +import base58 + + +def decode_multibase(multibase): + prefix, value = multibase[0], multibase[1:] + if prefix == 'u': + b64_urlsafe_nopad = value + b64_urlsafe = b64_urlsafe_nopad + "=" * ((4 - len(b64_urlsafe_nopad) % 4) % 4) + return base64.urlsafe_b64decode(b64_urlsafe) + elif prefix == 'f': + return bytes.fromhex(value) + elif prefix == 'z': + return base58.b58decode(value) + else: + raise("Malformed multibase value") + + +def encode_multibase_base64_urlsafe_nopad(bin): + return 'u' + base64.urlsafe_b64encode(bin).decode('utf-8').rstrip("=") + + +def multihash_sha3_256_multibase_base16(bin): + return 'f1620' + hashlib.sha3_256(bin).hexdigest() + + +def public_key_from_did(did): + assert did.startswith('did:key:') + multicodec = decode_multibase(did[len('did:key:'):]) + assert multicodec[0:1].hex() == 'ed' # 0xed is multicodec value for Ed25519Pub + return Ed25519PublicKey.from_public_bytes(multicodec[2:]) + +def main(args): + # Query data + resp = requests.get( + args.node_url + "/query", + params=dict( + query=args.query, + include="proof", + ) + ) + resp.raise_for_status() + resp_data = resp.json() + + print(">>> Node's response:") + print(json.dumps(resp_data, indent=2)) + print() + print() + + + # Verify commitment consistency + # This should always be done by the client after receiving a proof to ensure its disputable + assert resp_data["commitment"]["inputHash"] == multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json(resp_data["input"]) + ) + assert resp_data["commitment"]["outputHash"] == multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json(resp_data["output"]) + ) + assert resp_data["commitment"]["subQueriesHash"] == multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json(resp_data["subQueries"]) + ) + + signature = decode_multibase(resp_data["proof"]["proofValue"]) + public_key = public_key_from_did(resp_data["proof"]["verificationMethod"]) + public_key.verify(signature, canonicaljson.encode_canonical_json(resp_data["commitment"])) + print("Commitment is consistent!") + + + print(">>> Commitment:") + commitment = resp_data.copy() + del commitment["output"] + print(json.dumps(commitment, indent=2)) + print() + print() + + + # Remote validation through reproducibility + print(">>> Verifying original commitment:") + resp = requests.post( + args.node_url + "/verify", + json=commitment + ) + resp.raise_for_status() + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == True + + + # Invalid request: input hash + print(">>> Simulating invalid request (input hash):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["commitment"]["inputHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "InvalidRequest::InputHash" + + + # Invalid request: subQueries hash + print(">>> Simulating invalid request (subQueries hash):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["commitment"]["subQueriesHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "InvalidRequest::SubQueriesHash" + + + # Invalid request: bad signature + print(">>> Simulating invalid request (bad signature):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["proof"]["proofValue"] = "uZbm7fFcWc4l6iyvaKe_txdKntL3h3kvsGHOaKIbPV6c42PH1VnSmpYHMopv4TU68syzgoEdcS26AvpkSQb9dBQ" + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "InvalidRequest::BadSignature" + + + if args.private_key is None: + print("Private key is not provided - skipping tests that require signature forging") + return + + private_key = Ed25519PrivateKey.from_private_bytes(decode_multibase(args.private_key)) + + + # Cannot reproduce the query: output mismatch + # Dataset stays the same but we fake the output hash and the signature + print(">>> Simulating invalid request (output mismatch):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["commitment"]["outputHash"] = "f1620ff7f5beaf16900218a3ac4aae82cdccf764816986c7c739c716cf7dc03112a2d" + + canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) + signature = private_key.sign(canonical_commitment) + invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) + + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "VerificationFailed::OutputMismatch" + + + # Cannot reproduce the query: dataset is missing + # Dataset stays the same but we fake the output hash and the signature + print(">>> Simulating invalid request (dataset is missing):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["input"]["datasets"][0]["id"] = invalid_commitment["input"]["datasets"][0]["id"][:-4] + "beef" + invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json( + invalid_commitment["input"] + ) + ) + + canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) + signature = private_key.sign(canonical_commitment) + invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) + + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "VerificationFailed::DatasetNotFound" + + + # Cannot reproduce the query: block is missing + # Dataset stays the same but we fake the output hash and the signature + print(">>> Simulating invalid request (block is missing):") + invalid_commitment = copy.deepcopy(commitment) + invalid_commitment["input"]["datasets"][0]["blockHash"] = invalid_commitment["input"]["datasets"][0]["blockHash"][:-4] + "beef" + invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( + canonicaljson.encode_canonical_json( + invalid_commitment["input"] + ) + ) + + canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) + signature = private_key.sign(canonical_commitment) + invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) + + print(json.dumps(invalid_commitment, indent=2)) + print() + print() + + print("Verification result:") + resp = requests.post( + args.node_url + "/verify", + json=invalid_commitment + ) + resp_data = resp.json() + print(json.dumps(resp_data, indent=2)) + print() + print() + assert resp_data["ok"] == False + assert resp_data["error"]["kind"] == "VerificationFailed::DatasetBlockNotFound" + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--node-url', required=True) + parser.add_argument('--private-key', required=False) + parser.add_argument('--query', default='select block_hash, to from "kamu/net.rocketpool.reth.tokens-minted" order by offset desc limit 1') + args = parser.parse_args() + main(args) \ No newline at end of file diff --git a/examples/query-commitments/requirements.in b/examples/query-commitments/requirements.in new file mode 100644 index 000000000..ab209766b --- /dev/null +++ b/examples/query-commitments/requirements.in @@ -0,0 +1,4 @@ +base58 +canonicaljson +cryptography +requests \ No newline at end of file diff --git a/examples/reth-vs-snp500/init-odf-all.sh b/examples/reth-vs-snp500/init-odf-all.sh new file mode 100755 index 000000000..1739ff387 --- /dev/null +++ b/examples/reth-vs-snp500/init-odf-all.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +NODE_URL="odf+https://node.demo.kamu.dev/kamu/" + +kamu init || true + +# Root +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" +kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" +kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" + +kamu pull "${NODE_URL}account.transactions" +kamu pull "${NODE_URL}account.tokens.transfers" + +# Deriv +kamu pull "${NODE_URL}net.rocketpool.reth.mint-burn" +kamu pull "${NODE_URL}account.tokens.portfolio" +kamu pull "${NODE_URL}account.tokens.portfolio.market-value" +kamu pull "${NODE_URL}account.tokens.portfolio.usd" +kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.market-value" +kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.portfolio" diff --git a/examples/reth-vs-snp500/init-odf.sh b/examples/reth-vs-snp500/init-odf.sh new file mode 100755 index 000000000..abec551cc --- /dev/null +++ b/examples/reth-vs-snp500/init-odf.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +NODE_URL="odf+https://node.demo.kamu.dev/kamu/" + +kamu init || true + +# Root +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" +kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" +kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" +kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" + +kamu pull "${NODE_URL}account.transactions" +kamu pull "${NODE_URL}account.tokens.transfers" + +kamu add -r .