From 3ac2703cd79d091a340c7c64b167f9af618ecde7 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 3 Jul 2024 22:59:45 +0530 Subject: [PATCH 01/92] Initializing Caravan Health and adding privacy score Signed-off-by: Harshil-Jani --- package-lock.json | 11 ++ packages/caravan-health/.eslintrc.js | 11 ++ packages/caravan-health/README.md | 1 + packages/caravan-health/jest.config.js | 5 + packages/caravan-health/package.json | 39 +++++ packages/caravan-health/src/index.ts | 1 + packages/caravan-health/src/privacy.ts | 199 +++++++++++++++++++++++++ packages/caravan-health/tsconfig.json | 4 + packages/caravan-health/tsup.config.js | 6 + 9 files changed, 277 insertions(+) create mode 100644 packages/caravan-health/.eslintrc.js create mode 100644 packages/caravan-health/README.md create mode 100644 packages/caravan-health/jest.config.js create mode 100644 packages/caravan-health/package.json create mode 100644 packages/caravan-health/src/index.ts create mode 100644 packages/caravan-health/src/privacy.ts create mode 100644 packages/caravan-health/tsconfig.json create mode 100644 packages/caravan-health/tsup.config.js diff --git a/package-lock.json b/package-lock.json index cbdc5ec5..511ffb15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2721,6 +2721,10 @@ "resolved": "packages/eslint-config", "link": true }, + "node_modules/@caravan/health": { + "resolved": "packages/health", + "link": true + }, "node_modules/@caravan/multisig": { "resolved": "packages/multisig", "link": true @@ -28457,6 +28461,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/health": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "packages/multisig": { "name": "@caravan/multisig", "version": "0.0.0", diff --git a/packages/caravan-health/.eslintrc.js b/packages/caravan-health/.eslintrc.js new file mode 100644 index 00000000..f93c6274 --- /dev/null +++ b/packages/caravan-health/.eslintrc.js @@ -0,0 +1,11 @@ +// .eslintrc.js +module.exports = { + root: true, + extends: [ + "@caravan/eslint-config/library.js" + ], + parser: "@typescript-eslint/parser", + parserOptions: { + project: true, + }, +}; diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md new file mode 100644 index 00000000..5154f3f5 --- /dev/null +++ b/packages/caravan-health/README.md @@ -0,0 +1 @@ +# Health \ No newline at end of file diff --git a/packages/caravan-health/jest.config.js b/packages/caravan-health/jest.config.js new file mode 100644 index 00000000..b6c0f588 --- /dev/null +++ b/packages/caravan-health/jest.config.js @@ -0,0 +1,5 @@ +// jest.config.js +module.exports = { + preset: "ts-jest", + testEnvironment: "node" +}; diff --git a/packages/caravan-health/package.json b/packages/caravan-health/package.json new file mode 100644 index 00000000..25fb2e46 --- /dev/null +++ b/packages/caravan-health/package.json @@ -0,0 +1,39 @@ +{ + "name": "@caravan/health", + "version": "1.0.0", + "author": "Harshil Jani", + "description": "The core logic for analysing wallet health for privacy concerns and nature of spending fees.", + "private": true, + "engines": { + "node": ">=20" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "module": "./dist/index.mjs", + "files": [ + "./dist/index.js", + "./dist/index.mjs", + "./dist/index.d.ts" + ], + "exports": { + ".": { + "import": "./dist/index.mjs", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "license": "MIT", + "scripts": { + "build": "tsup src/index.ts --format cjs,esm --dts", + "dev": "npm run build -- --watch", + "lint": "eslint src/", + "ci": "npm run lint && npm run test", + "test": "jest src", + "test:watch": "jest --watch src", + "test:debug": "node --inspect-brk ../../node_modules/.bin/jest --runInBand" + }, + "dependencies": { + "@caravan/clients": "*" + } + +} diff --git a/packages/caravan-health/src/index.ts b/packages/caravan-health/src/index.ts new file mode 100644 index 00000000..1a42f385 --- /dev/null +++ b/packages/caravan-health/src/index.ts @@ -0,0 +1 @@ +export {privacyScore} from "./privacy"; \ No newline at end of file diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts new file mode 100644 index 00000000..a918bed5 --- /dev/null +++ b/packages/caravan-health/src/privacy.ts @@ -0,0 +1,199 @@ +/* +The methodology for calculating a privacy score (p_score) for Bitcoin transactions based +on the number of inputs and outputs is the primary point to define wallet health for privacy. +The score is further influenced by several factors such as address reuse, +address types and UTXO set fingerprints etc. + +More on the algorithms for scoring privacy: +*/ + +// A normalizing quantity that increases the score by a certain factor in cases of self-payment. +// More about deniability : https://www.truthcoin.info/blog/deniability/ +const DENIABILITY_FACTOR = 1.5; + +/* +The p_score is calculated by evaluating the likelihood of self-payments, the involvement of +change outputs and the type of transaction based on number of inputs and outputs. + +We have 5 categories of transaction type +- Sweep Spend +- Simple Spend +- UTXO Fragmentation +- Consolidation +- CoinJoin +*/ +function privacyScoreOnIO(transaction: any): number { + const numberOfInputs: number = transaction.vin.length; + const numberOfOutputs: number = transaction.vout.length; + + let score: number; + + if (numberOfInputs === 1) { + if (numberOfOutputs === 1) { + // Sweep Spend (No change Output) + // #Input = 1, #Output = 1 + score = 1 / 2; + } else if (numberOfOutputs === 2) { + // Simple Spend (Single change output) + // #Input = 1, #Output = 2 + score = 4 / 9; + } else { + // UTXO Fragmentation + // #Input = 1, #Output > 2 + score = 2 / 3 - 1 / numberOfOutputs; + } + if (isSelfPayment(transaction)) { + return score * DENIABILITY_FACTOR; + } + } else { + if (numberOfOutputs === 1) { + // Consolidation + // #Input >= 2, #Output = 1 + score = 1 / numberOfInputs; + + // No D.F for consolidation + } else { + // Mixing or CoinJoin + // #Input >= 2, #Output >= 2 + let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; + score = (0.75 * x) / (1 + x); + if (isSelfPayment(transaction)) { + return score * DENIABILITY_FACTOR; + } + } + } + return score; +} + +/* +Determine whether a Bitcoin transaction is a self-payment, meaning the sender and recipient +are the same entity. To check this, we make an RPC call to the watcher wallet asking for the +amount that a given address holds. If the call returns a number then it is part of wallet +otherwise it is not a self payment. +*/ +function isSelfPayment(transaction: any): boolean { + /*TODO : Call getAddressStatus to check against bitcoind or block explorer*/ + return false; +} + +/* TODO : replace any type to custom types for Transactions and UTXOs*/ +/* +In order to score for address reuse we can check the amount being hold by reused UTXOs +with respect to the total amount +*/ +function addressReuseFactor(utxos: Array): number { + let reused_amount : number = 0; + let total_amount : number = 0; + utxos.forEach((utxo) => { + if (utxo.reused) { + reused_amount += utxo.amount; + } + total_amount += utxo.amount; + }); + return reused_amount / total_amount; +} + +/* +If we are making payment to other wallet types then the privacy score should decrease because +the change received will be at address of our wallet type and it will lead to derivation that +we still own that amount. +*/ +function addressTypeFactor(transactions : Array, walletAddressType : string): number { + let P2WSH : number = 0; + let P2PKH : number = 0; + let P2SH : number = 0; + let atf : number = 1; + transactions.forEach(tx => { + tx.vout.forEach(output => { + let address = output.scriptPubKey.address; + if (address.startsWith("bc")) { + //Bech 32 Native Segwit (P2WSH) or Taproot + P2WSH += 1; + } else if (address.startsWith("1")) { + // Legacy (P2PKH) + P2PKH += 1; + } else { + // Segwith (P2SH) + P2SH += 1; + } + }) + }); + + if (walletAddressType == "P2WSH" && (P2WSH != 0 && (P2SH != 0 || P2PKH != 0))) { + atf = 1 / (P2WSH + 1); + } else if (walletAddressType == "P2PKH" && (P2PKH != 0 && (P2SH != 0 || P2WSH != 0))) { + atf = 1 / (P2PKH + 1); + } else if (walletAddressType == "P2SH" && (P2SH != 0 && (P2WSH != 0 || P2PKH != 0))) { + atf = 1 / (P2SH + 1); + } else { + atf = 1; + } + return atf; +} + +/* +The spread factor using standard deviation helps in assessing the dispersion of UTXO values. +In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries +to link transactions and deduce the ownership and spending patterns of users. +*/ +function utxoSpreadFactor(utxos : Array) : number { + const amounts : Array = utxos.map(utxo => utxo.amount); + const mean : number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; + const variance : number = amounts.reduce((sum, amount) => sum + Math.pow(amount - mean, 2), 0) / amounts.length; + const stdDev : number = Math.sqrt(variance); + return stdDev / (stdDev + 1); +} + +/* +The weightage is ad-hoc to normalize the privacy score based on the number of UTXOs in the set. +- 0 for UTXO set length >= 50 +- 0.25 for UTXO set length >= 25 and <= 49 +- 0.5 for UTXO set length >= 15 and <= 24 +- 0.75 for UTXO set length >= 5 and <= 14 +- 1 for UTXO set length < 5 +*/ +function utxoSetLengthWeight(utxos : Array) : number { + let utxo_set_length : number = utxos.length; + let weight : number; + if (utxo_set_length >= 50) { + weight = 0; + } else if (utxo_set_length >= 25 && utxo_set_length <= 49) { + weight = 0.25; + } else if (utxo_set_length >= 15 && utxo_set_length <= 24) { + weight = 0.5; + } else if (utxo_set_length >= 5 && utxo_set_length <= 14) { + weight = 0.75; + } else { + weight = 1; + } + return weight; +} + +/* +UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. +It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. +*/ +function utxoValueWeightageFactor(utxos: Array): number { + let W : number = utxoSetLengthWeight(utxos); + let USF : number = utxoSpreadFactor(utxos); + return (USF + W)*0.15 -0.15; +} + +/* +The privacy score is a combination of all the factors calculated above. +- Privacy Score based on Inputs and Outputs +- Address Reuse Factor (R.F) : p_adjusted = p_score * (1 - 0.5 * r.f) + 0.10 * (1 - r.f) +- Address Type Factor (A.T.F) : p_adjusted = p_score * (1-A.T.F) +- UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F +*/ +export function privacyScore(transactions : Array, utxos : Array, walletAddressType : string) : number { + let privacy_score = transactions.reduce((sum, tx) => sum + privacyScoreOnIO(tx), 0) / transactions.length; + // Adjusting the privacy score based on the address reuse factor + privacy_score = (privacy_score * (1 - (0.5 * addressReuseFactor(utxos)))) + (0.10 * (1 - addressReuseFactor(utxos))); + // Adjusting the privacy score based on the address type factor + privacy_score = privacy_score * (1 - addressTypeFactor(transactions,walletAddressType)); + // Adjusting the privacy score based on the UTXO set length and value weightage factor + privacy_score = privacy_score + 0.1 * utxoValueWeightageFactor(utxos) + + return privacy_score +} diff --git a/packages/caravan-health/tsconfig.json b/packages/caravan-health/tsconfig.json new file mode 100644 index 00000000..1f2d76fd --- /dev/null +++ b/packages/caravan-health/tsconfig.json @@ -0,0 +1,4 @@ +// tsconfig.json +{ + "extends": "@caravan/typescript-config/base.json" +} diff --git a/packages/caravan-health/tsup.config.js b/packages/caravan-health/tsup.config.js new file mode 100644 index 00000000..2aee42e6 --- /dev/null +++ b/packages/caravan-health/tsup.config.js @@ -0,0 +1,6 @@ +import { defineConfig } from 'tsup'; +import { polyfillNode } from "esbuild-plugin-polyfill-node"; + +export default defineConfig({ + esbuildPlugins: [polyfillNode()], +}); From 822cf54a03f1ca1e939799ce636d773b0fbb81cb Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 8 Jul 2024 22:18:45 +0530 Subject: [PATCH 02/92] Utilize client methods from @caravan/client Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index a918bed5..e89ee4dc 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,3 +1,4 @@ +import {BlockchainClient} from '@caravan/clients' /* The methodology for calculating a privacy score (p_score) for Bitcoin transactions based on the number of inputs and outputs is the primary point to define wallet health for privacy. @@ -22,7 +23,7 @@ We have 5 categories of transaction type - Consolidation - CoinJoin */ -function privacyScoreOnIO(transaction: any): number { +function privacyScoreOnIO(transaction: any, client: BlockchainClient): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -42,7 +43,7 @@ function privacyScoreOnIO(transaction: any): number { // #Input = 1, #Output > 2 score = 2 / 3 - 1 / numberOfOutputs; } - if (isSelfPayment(transaction)) { + if (isSelfPayment(transaction,client)) { return score * DENIABILITY_FACTOR; } } else { @@ -57,7 +58,7 @@ function privacyScoreOnIO(transaction: any): number { // #Input >= 2, #Output >= 2 let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; score = (0.75 * x) / (1 + x); - if (isSelfPayment(transaction)) { + if (isSelfPayment(transaction,client)) { return score * DENIABILITY_FACTOR; } } @@ -71,9 +72,13 @@ are the same entity. To check this, we make an RPC call to the watcher wallet as amount that a given address holds. If the call returns a number then it is part of wallet otherwise it is not a self payment. */ -function isSelfPayment(transaction: any): boolean { - /*TODO : Call getAddressStatus to check against bitcoind or block explorer*/ - return false; +function isSelfPayment(transaction: any, client: BlockchainClient): boolean { + transaction.vout.forEach(async op => { + if(await client.getAddressStatus(op.scriptPubKey.address) === undefined){ + return false; + } + }) + return true; } /* TODO : replace any type to custom types for Transactions and UTXOs*/ @@ -186,8 +191,8 @@ The privacy score is a combination of all the factors calculated above. - Address Type Factor (A.T.F) : p_adjusted = p_score * (1-A.T.F) - UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F */ -export function privacyScore(transactions : Array, utxos : Array, walletAddressType : string) : number { - let privacy_score = transactions.reduce((sum, tx) => sum + privacyScoreOnIO(tx), 0) / transactions.length; +export function privacyScore(transactions : Array, utxos : Array, walletAddressType : string, client: BlockchainClient) : number { + let privacy_score = transactions.reduce((sum, tx) => sum + privacyScoreOnIO(tx,client), 0) / transactions.length; // Adjusting the privacy score based on the address reuse factor privacy_score = (privacy_score * (1 - (0.5 * addressReuseFactor(utxos)))) + (0.10 * (1 - addressReuseFactor(utxos))); // Adjusting the privacy score based on the address type factor From e3989b97b91606c5b55c9ea9b0c591a199891159 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 9 Jul 2024 00:25:21 +0530 Subject: [PATCH 03/92] Added fees score method to @caravan/health Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.ts | 91 +++++++++++++++++++++++++ packages/caravan-health/src/privacy.ts | 2 +- 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 packages/caravan-health/src/feescore.ts diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts new file mode 100644 index 00000000..89c1df92 --- /dev/null +++ b/packages/caravan-health/src/feescore.ts @@ -0,0 +1,91 @@ +import { utxoSetLengthWeight } from "./privacy"; + +function getFeeRateForTransaction(transaction: any): number { + // TODO : Please check that do we really get the fee rate and weight both from the transaction object + let fees: number = transaction.fee; + let weight: number = transaction.weight; + return fees / weight; +} + +// Function to call the Mempool block fee rates +async function getFeeRatePercentileForTransaction( + timestamp: any, + feeRate: number +) { + const url: string = + "https://mempool.space/api/v1/mining/blocks/fee-rates/all"; + const headers: Headers = new Headers(); + headers.set("Content-Type", "application/json"); + + const response: Response = await fetch(url, { + method: "GET", + headers: headers, + }); + + const data: Array = await response.json(); + + // Find the closest entry by timestamp + let closestEntry: any; + let closestDifference: number = Infinity; + + data.forEach((item) => { + const difference = Math.abs(item.timestamp - timestamp); + if (difference < closestDifference) { + closestDifference = difference; + closestEntry = item; + } + }); + + switch (true) { + case feeRate < closestEntry.avgFee_10: + return 1; + case feeRate < closestEntry.avgFee_25: + return 0.9; + case feeRate < closestEntry.avgFee_50: + return 0.75; + case feeRate < closestEntry.avgFee_75: + return 0.5; + case feeRate < closestEntry.avgFee_90: + return 0.25; + case feeRate < closestEntry.avgFee_100: + return 0.1; + default: + return 0; + } +} + +export function RelativeFeesScore(transactions: Array): number { + let sumRFS: number = 0; + let numberOfSendTx: number = 0; + transactions.forEach(async (tx: any) => { + if (tx.category == "send") { + numberOfSendTx++; + let feeRate: number = getFeeRateForTransaction(tx); + let RFS: number = await getFeeRatePercentileForTransaction( + tx.blocktime, + feeRate + ); + sumRFS += RFS; + } + }); + return sumRFS / numberOfSendTx; +} + +export function feesToAmountRatio(transactions: Array): number { + let sumFeesToAmountRatio: number = 0; + let numberOfSendTx: number = 0; + transactions.forEach((tx: any) => { + if (tx.category === "send") { + sumFeesToAmountRatio += tx.fee / tx.amount; + numberOfSendTx++; + } + }); + return sumFeesToAmountRatio / numberOfSendTx; +} + +export function feesScore(transactions: Array, utxos: Array): number { + let RFS: number = RelativeFeesScore(transactions); + let FAR: number = feesToAmountRatio(transactions); + let W : number = utxoSetLengthWeight(utxos); + return (0.35* RFS) + (0.35 * FAR) + (0.3 * W); +} diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index e89ee4dc..c45ce24e 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -157,7 +157,7 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ -function utxoSetLengthWeight(utxos : Array) : number { +export function utxoSetLengthWeight(utxos : Array) : number { let utxo_set_length : number = utxos.length; let weight : number; if (utxo_set_length >= 50) { From 7737e3e1551372f66ebfb6d20347b47e5dd6f5e7 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 9 Jul 2024 00:26:49 +0530 Subject: [PATCH 04/92] Export all privacy methods Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index c45ce24e..c2c5fa32 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -23,7 +23,7 @@ We have 5 categories of transaction type - Consolidation - CoinJoin */ -function privacyScoreOnIO(transaction: any, client: BlockchainClient): number { +export function privacyScoreOnIO(transaction: any, client: BlockchainClient): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -86,7 +86,7 @@ function isSelfPayment(transaction: any, client: BlockchainClient): boolean { In order to score for address reuse we can check the amount being hold by reused UTXOs with respect to the total amount */ -function addressReuseFactor(utxos: Array): number { +export function addressReuseFactor(utxos: Array): number { let reused_amount : number = 0; let total_amount : number = 0; utxos.forEach((utxo) => { @@ -103,7 +103,7 @@ If we are making payment to other wallet types then the privacy score should dec the change received will be at address of our wallet type and it will lead to derivation that we still own that amount. */ -function addressTypeFactor(transactions : Array, walletAddressType : string): number { +export function addressTypeFactor(transactions : Array, walletAddressType : string): number { let P2WSH : number = 0; let P2PKH : number = 0; let P2SH : number = 0; @@ -141,7 +141,7 @@ The spread factor using standard deviation helps in assessing the dispersion of In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries to link transactions and deduce the ownership and spending patterns of users. */ -function utxoSpreadFactor(utxos : Array) : number { +export function utxoSpreadFactor(utxos : Array) : number { const amounts : Array = utxos.map(utxo => utxo.amount); const mean : number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; const variance : number = amounts.reduce((sum, amount) => sum + Math.pow(amount - mean, 2), 0) / amounts.length; @@ -178,7 +178,7 @@ export function utxoSetLengthWeight(utxos : Array) : number { UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. */ -function utxoValueWeightageFactor(utxos: Array): number { +export function utxoValueWeightageFactor(utxos: Array): number { let W : number = utxoSetLengthWeight(utxos); let USF : number = utxoSpreadFactor(utxos); return (USF + W)*0.15 -0.15; From 742137604878887ea3d7cd6fb6cd56df5ec8d757 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 9 Jul 2024 01:15:35 +0530 Subject: [PATCH 05/92] Add some context on the fee score functions Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.ts | 33 +++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 89c1df92..5f3320a5 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,5 +1,6 @@ import { utxoSetLengthWeight } from "./privacy"; +// Utility function that helps to obtain the fee rate of the transaction function getFeeRateForTransaction(transaction: any): number { // TODO : Please check that do we really get the fee rate and weight both from the transaction object let fees: number = transaction.fee; @@ -7,7 +8,8 @@ function getFeeRateForTransaction(transaction: any): number { return fees / weight; } -// Function to call the Mempool block fee rates +// TODO : Implement Caching or Ticker based mechanism to reduce network latency +// Utility function that helps to obtain the percentile of the fees paid by user in tx block async function getFeeRatePercentileForTransaction( timestamp: any, feeRate: number @@ -54,6 +56,13 @@ async function getFeeRatePercentileForTransaction( } } +/* +R.F.S can be associated with all the transactions and we can give a measure +if any transaction was done at expensive fees or nominal fees. + +This can be done by calculating the percentile of the fees paid by the user +in the block of the transaction. +*/ export function RelativeFeesScore(transactions: Array): number { let sumRFS: number = 0; let numberOfSendTx: number = 0; @@ -71,6 +80,16 @@ export function RelativeFeesScore(transactions: Array): number { return sumRFS / numberOfSendTx; } +/* +Measure of how much the wallet is burning in fees is that we take the ratio of +amount being paid and the fees consumed. + +Mastercard charges 0.6% cross-border fee for international transactions in US dollars, +but if the transaction is in any other currency the fee goes up to 1%. +Source : https://www.clearlypayments.com/blog/what-are-cross-border-fees-in-credit-card-payments/ + +This ratio is a measure of our fees spending against the fiat charges we pay. +*/ export function feesToAmountRatio(transactions: Array): number { let sumFeesToAmountRatio: number = 0; let numberOfSendTx: number = 0; @@ -80,9 +99,19 @@ export function feesToAmountRatio(transactions: Array): number { numberOfSendTx++; } }); - return sumFeesToAmountRatio / numberOfSendTx; + return 100*(sumFeesToAmountRatio / numberOfSendTx); } +/* +35% Weightage of fees score depends on Percentile of fees paid +35% Weightage of fees score depends fees paid with respect to amount spend +30% Weightage of fees score depends on the number of UTXOs present in the wallet. + +Q : What role does W plays in the fees score? +Assume the wallet is being consolidated, Thus number of UTXO will decrease and thus +W (Weightage of number of UTXO) will increase and this justifies that, consolidation +increases the fees health since you don’t overpay them in long run. +*/ export function feesScore(transactions: Array, utxos: Array): number { let RFS: number = RelativeFeesScore(transactions); let FAR: number = feesToAmountRatio(transactions); From c3f167edc7300d543b77ea3075fd9a7faa4f05f0 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sun, 14 Jul 2024 13:13:59 +0530 Subject: [PATCH 06/92] Rename to camelCase and fix wordings Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.ts | 4 +-- packages/caravan-health/src/privacy.ts | 34 ++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 5f3320a5..7a3fe198 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -63,7 +63,7 @@ if any transaction was done at expensive fees or nominal fees. This can be done by calculating the percentile of the fees paid by the user in the block of the transaction. */ -export function RelativeFeesScore(transactions: Array): number { +export function relativeFeesScore(transactions: Array): number { let sumRFS: number = 0; let numberOfSendTx: number = 0; transactions.forEach(async (tx: any) => { @@ -113,7 +113,7 @@ W (Weightage of number of UTXO) will increase and this justifies that, consolida increases the fees health since you don’t overpay them in long run. */ export function feesScore(transactions: Array, utxos: Array): number { - let RFS: number = RelativeFeesScore(transactions); + let RFS: number = relativeFeesScore(transactions); let FAR: number = feesToAmountRatio(transactions); let W : number = utxoSetLengthWeight(utxos); return (0.35* RFS) + (0.35 * FAR) + (0.3 * W); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index c2c5fa32..29528066 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -23,7 +23,7 @@ We have 5 categories of transaction type - Consolidation - CoinJoin */ -export function privacyScoreOnIO(transaction: any, client: BlockchainClient): number { +export function privscyScoreByTxTopology(transaction: any, client: BlockchainClient): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -87,20 +87,20 @@ In order to score for address reuse we can check the amount being hold by reused with respect to the total amount */ export function addressReuseFactor(utxos: Array): number { - let reused_amount : number = 0; - let total_amount : number = 0; + let reusedAmount : number = 0; + let totalAmount : number = 0; utxos.forEach((utxo) => { if (utxo.reused) { - reused_amount += utxo.amount; + reusedAmount += utxo.amount; } - total_amount += utxo.amount; + totalAmount += utxo.amount; }); - return reused_amount / total_amount; + return reusedAmount / totalAmount; } /* If we are making payment to other wallet types then the privacy score should decrease because -the change received will be at address of our wallet type and it will lead to derivation that +the change received will be to an address type matching our wallet and it will lead to a deduction that we still own that amount. */ export function addressTypeFactor(transactions : Array, walletAddressType : string): number { @@ -158,15 +158,15 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 1 for UTXO set length < 5 */ export function utxoSetLengthWeight(utxos : Array) : number { - let utxo_set_length : number = utxos.length; + let utxoSetLength : number = utxos.length; let weight : number; - if (utxo_set_length >= 50) { + if (utxoSetLength >= 50) { weight = 0; - } else if (utxo_set_length >= 25 && utxo_set_length <= 49) { + } else if (utxoSetLength >= 25 && utxoSetLength <= 49) { weight = 0.25; - } else if (utxo_set_length >= 15 && utxo_set_length <= 24) { + } else if (utxoSetLength >= 15 && utxoSetLength <= 24) { weight = 0.5; - } else if (utxo_set_length >= 5 && utxo_set_length <= 14) { + } else if (utxoSetLength >= 5 && utxoSetLength <= 14) { weight = 0.75; } else { weight = 1; @@ -192,13 +192,13 @@ The privacy score is a combination of all the factors calculated above. - UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F */ export function privacyScore(transactions : Array, utxos : Array, walletAddressType : string, client: BlockchainClient) : number { - let privacy_score = transactions.reduce((sum, tx) => sum + privacyScoreOnIO(tx,client), 0) / transactions.length; + let privacyScore = transactions.reduce((sum, tx) => sum + privscyScoreByTxTopology(tx,client), 0) / transactions.length; // Adjusting the privacy score based on the address reuse factor - privacy_score = (privacy_score * (1 - (0.5 * addressReuseFactor(utxos)))) + (0.10 * (1 - addressReuseFactor(utxos))); + privacyScore = (privacyScore * (1 - (0.5 * addressReuseFactor(utxos)))) + (0.10 * (1 - addressReuseFactor(utxos))); // Adjusting the privacy score based on the address type factor - privacy_score = privacy_score * (1 - addressTypeFactor(transactions,walletAddressType)); + privacyScore = privacyScore * (1 - addressTypeFactor(transactions,walletAddressType)); // Adjusting the privacy score based on the UTXO set length and value weightage factor - privacy_score = privacy_score + 0.1 * utxoValueWeightageFactor(utxos) + privacyScore = privacyScore + 0.1 * utxoValueWeightageFactor(utxos) - return privacy_score + return privacyScore } From 27cacf168c62b472c7699964c956d96ff8e0de88 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sun, 14 Jul 2024 13:19:49 +0530 Subject: [PATCH 07/92] Setup prettier for @caravan/health and format Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 2 +- packages/caravan-health/package.json | 4 + packages/caravan-health/src/feescore.ts | 12 +- packages/caravan-health/src/index.ts | 2 +- packages/caravan-health/src/privacy.ts | 157 ++++++++++++++---------- 5 files changed, 106 insertions(+), 71 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 5154f3f5..c5593854 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -1 +1 @@ -# Health \ No newline at end of file +# Health diff --git a/packages/caravan-health/package.json b/packages/caravan-health/package.json index 25fb2e46..d2a7552b 100644 --- a/packages/caravan-health/package.json +++ b/packages/caravan-health/package.json @@ -28,12 +28,16 @@ "dev": "npm run build -- --watch", "lint": "eslint src/", "ci": "npm run lint && npm run test", + "format": "prettier --write \"**/*.{ts,tsx,md}\"", "test": "jest src", "test:watch": "jest --watch src", "test:debug": "node --inspect-brk ../../node_modules/.bin/jest --runInBand" }, "dependencies": { "@caravan/clients": "*" + }, + "devDependencies": { + "prettier": "^3.2.5" } } diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 7a3fe198..4fdfbfad 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,6 +1,6 @@ import { utxoSetLengthWeight } from "./privacy"; -// Utility function that helps to obtain the fee rate of the transaction +// Utility function that helps to obtain the fee rate of the transaction function getFeeRateForTransaction(transaction: any): number { // TODO : Please check that do we really get the fee rate and weight both from the transaction object let fees: number = transaction.fee; @@ -12,7 +12,7 @@ function getFeeRateForTransaction(transaction: any): number { // Utility function that helps to obtain the percentile of the fees paid by user in tx block async function getFeeRatePercentileForTransaction( timestamp: any, - feeRate: number + feeRate: number, ) { const url: string = "https://mempool.space/api/v1/mining/blocks/fee-rates/all"; @@ -72,7 +72,7 @@ export function relativeFeesScore(transactions: Array): number { let feeRate: number = getFeeRateForTransaction(tx); let RFS: number = await getFeeRatePercentileForTransaction( tx.blocktime, - feeRate + feeRate, ); sumRFS += RFS; } @@ -99,7 +99,7 @@ export function feesToAmountRatio(transactions: Array): number { numberOfSendTx++; } }); - return 100*(sumFeesToAmountRatio / numberOfSendTx); + return 100 * (sumFeesToAmountRatio / numberOfSendTx); } /* @@ -115,6 +115,6 @@ increases the fees health since you don’t overpay them in long run. export function feesScore(transactions: Array, utxos: Array): number { let RFS: number = relativeFeesScore(transactions); let FAR: number = feesToAmountRatio(transactions); - let W : number = utxoSetLengthWeight(utxos); - return (0.35* RFS) + (0.35 * FAR) + (0.3 * W); + let W: number = utxoSetLengthWeight(utxos); + return 0.35 * RFS + 0.35 * FAR + 0.3 * W; } diff --git a/packages/caravan-health/src/index.ts b/packages/caravan-health/src/index.ts index 1a42f385..79e01ac2 100644 --- a/packages/caravan-health/src/index.ts +++ b/packages/caravan-health/src/index.ts @@ -1 +1 @@ -export {privacyScore} from "./privacy"; \ No newline at end of file +export { privacyScore } from "./privacy"; diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 29528066..1c779b3b 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,4 +1,4 @@ -import {BlockchainClient} from '@caravan/clients' +import { BlockchainClient } from "@caravan/clients"; /* The methodology for calculating a privacy score (p_score) for Bitcoin transactions based on the number of inputs and outputs is the primary point to define wallet health for privacy. @@ -23,7 +23,10 @@ We have 5 categories of transaction type - Consolidation - CoinJoin */ -export function privscyScoreByTxTopology(transaction: any, client: BlockchainClient): number { +export function privscyScoreByTxTopology( + transaction: any, + client: BlockchainClient, +): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -43,7 +46,7 @@ export function privscyScoreByTxTopology(transaction: any, client: BlockchainCli // #Input = 1, #Output > 2 score = 2 / 3 - 1 / numberOfOutputs; } - if (isSelfPayment(transaction,client)) { + if (isSelfPayment(transaction, client)) { return score * DENIABILITY_FACTOR; } } else { @@ -58,7 +61,7 @@ export function privscyScoreByTxTopology(transaction: any, client: BlockchainCli // #Input >= 2, #Output >= 2 let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; score = (0.75 * x) / (1 + x); - if (isSelfPayment(transaction,client)) { + if (isSelfPayment(transaction, client)) { return score * DENIABILITY_FACTOR; } } @@ -73,11 +76,13 @@ amount that a given address holds. If the call returns a number then it is part otherwise it is not a self payment. */ function isSelfPayment(transaction: any, client: BlockchainClient): boolean { - transaction.vout.forEach(async op => { - if(await client.getAddressStatus(op.scriptPubKey.address) === undefined){ - return false; - } - }) + transaction.vout.forEach(async (op) => { + if ( + (await client.getAddressStatus(op.scriptPubKey.address)) === undefined + ) { + return false; + } + }); return true; } @@ -87,8 +92,8 @@ In order to score for address reuse we can check the amount being hold by reused with respect to the total amount */ export function addressReuseFactor(utxos: Array): number { - let reusedAmount : number = 0; - let totalAmount : number = 0; + let reusedAmount: number = 0; + let totalAmount: number = 0; utxos.forEach((utxo) => { if (utxo.reused) { reusedAmount += utxo.amount; @@ -103,37 +108,48 @@ If we are making payment to other wallet types then the privacy score should dec the change received will be to an address type matching our wallet and it will lead to a deduction that we still own that amount. */ -export function addressTypeFactor(transactions : Array, walletAddressType : string): number { - let P2WSH : number = 0; - let P2PKH : number = 0; - let P2SH : number = 0; - let atf : number = 1; - transactions.forEach(tx => { - tx.vout.forEach(output => { - let address = output.scriptPubKey.address; - if (address.startsWith("bc")) { - //Bech 32 Native Segwit (P2WSH) or Taproot - P2WSH += 1; - } else if (address.startsWith("1")) { - // Legacy (P2PKH) - P2PKH += 1; - } else { - // Segwith (P2SH) - P2SH += 1; - } - }) +export function addressTypeFactor( + transactions: Array, + walletAddressType: string, +): number { + let P2WSH: number = 0; + let P2PKH: number = 0; + let P2SH: number = 0; + let atf: number = 1; + transactions.forEach((tx) => { + tx.vout.forEach((output) => { + let address = output.scriptPubKey.address; + if (address.startsWith("bc")) { + //Bech 32 Native Segwit (P2WSH) or Taproot + P2WSH += 1; + } else if (address.startsWith("1")) { + // Legacy (P2PKH) + P2PKH += 1; + } else { + // Segwith (P2SH) + P2SH += 1; + } }); + }); - if (walletAddressType == "P2WSH" && (P2WSH != 0 && (P2SH != 0 || P2PKH != 0))) { - atf = 1 / (P2WSH + 1); - } else if (walletAddressType == "P2PKH" && (P2PKH != 0 && (P2SH != 0 || P2WSH != 0))) { - atf = 1 / (P2PKH + 1); - } else if (walletAddressType == "P2SH" && (P2SH != 0 && (P2WSH != 0 || P2PKH != 0))) { - atf = 1 / (P2SH + 1); - } else { - atf = 1; - } - return atf; + if (walletAddressType == "P2WSH" && P2WSH != 0 && (P2SH != 0 || P2PKH != 0)) { + atf = 1 / (P2WSH + 1); + } else if ( + walletAddressType == "P2PKH" && + P2PKH != 0 && + (P2SH != 0 || P2WSH != 0) + ) { + atf = 1 / (P2PKH + 1); + } else if ( + walletAddressType == "P2SH" && + P2SH != 0 && + (P2WSH != 0 || P2PKH != 0) + ) { + atf = 1 / (P2SH + 1); + } else { + atf = 1; + } + return atf; } /* @@ -141,12 +157,15 @@ The spread factor using standard deviation helps in assessing the dispersion of In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries to link transactions and deduce the ownership and spending patterns of users. */ -export function utxoSpreadFactor(utxos : Array) : number { - const amounts : Array = utxos.map(utxo => utxo.amount); - const mean : number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; - const variance : number = amounts.reduce((sum, amount) => sum + Math.pow(amount - mean, 2), 0) / amounts.length; - const stdDev : number = Math.sqrt(variance); - return stdDev / (stdDev + 1); +export function utxoSpreadFactor(utxos: Array): number { + const amounts: Array = utxos.map((utxo) => utxo.amount); + const mean: number = + amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; + const variance: number = + amounts.reduce((sum, amount) => sum + Math.pow(amount - mean, 2), 0) / + amounts.length; + const stdDev: number = Math.sqrt(variance); + return stdDev / (stdDev + 1); } /* @@ -157,19 +176,19 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ -export function utxoSetLengthWeight(utxos : Array) : number { - let utxoSetLength : number = utxos.length; - let weight : number; +export function utxoSetLengthWeight(utxos: Array): number { + let utxoSetLength: number = utxos.length; + let weight: number; if (utxoSetLength >= 50) { - weight = 0; + weight = 0; } else if (utxoSetLength >= 25 && utxoSetLength <= 49) { - weight = 0.25; + weight = 0.25; } else if (utxoSetLength >= 15 && utxoSetLength <= 24) { - weight = 0.5; + weight = 0.5; } else if (utxoSetLength >= 5 && utxoSetLength <= 14) { - weight = 0.75; + weight = 0.75; } else { - weight = 1; + weight = 1; } return weight; } @@ -179,9 +198,9 @@ UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. */ export function utxoValueWeightageFactor(utxos: Array): number { - let W : number = utxoSetLengthWeight(utxos); - let USF : number = utxoSpreadFactor(utxos); - return (USF + W)*0.15 -0.15; + let W: number = utxoSetLengthWeight(utxos); + let USF: number = utxoSpreadFactor(utxos); + return (USF + W) * 0.15 - 0.15; } /* @@ -191,14 +210,26 @@ The privacy score is a combination of all the factors calculated above. - Address Type Factor (A.T.F) : p_adjusted = p_score * (1-A.T.F) - UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F */ -export function privacyScore(transactions : Array, utxos : Array, walletAddressType : string, client: BlockchainClient) : number { - let privacyScore = transactions.reduce((sum, tx) => sum + privscyScoreByTxTopology(tx,client), 0) / transactions.length; +export function privacyScore( + transactions: Array, + utxos: Array, + walletAddressType: string, + client: BlockchainClient, +): number { + let privacyScore = + transactions.reduce( + (sum, tx) => sum + privscyScoreByTxTopology(tx, client), + 0, + ) / transactions.length; // Adjusting the privacy score based on the address reuse factor - privacyScore = (privacyScore * (1 - (0.5 * addressReuseFactor(utxos)))) + (0.10 * (1 - addressReuseFactor(utxos))); + privacyScore = + privacyScore * (1 - 0.5 * addressReuseFactor(utxos)) + + 0.1 * (1 - addressReuseFactor(utxos)); // Adjusting the privacy score based on the address type factor - privacyScore = privacyScore * (1 - addressTypeFactor(transactions,walletAddressType)); + privacyScore = + privacyScore * (1 - addressTypeFactor(transactions, walletAddressType)); // Adjusting the privacy score based on the UTXO set length and value weightage factor - privacyScore = privacyScore + 0.1 * utxoValueWeightageFactor(utxos) + privacyScore = privacyScore + 0.1 * utxoValueWeightageFactor(utxos); - return privacyScore + return privacyScore; } From 29c220734f64099e90bda6ddc7806679ce7407a9 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 16 Jul 2024 00:44:13 +0530 Subject: [PATCH 08/92] Refactor privacyScoreByTxTopology for improved readability and maintainability Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.ts | 88 ++++++++++++++++---------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 1c779b3b..88c54ce2 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -23,6 +23,45 @@ We have 5 categories of transaction type - Consolidation - CoinJoin */ +enum SpendType { + SweepSpend = "SweepSpend", + SimpleSpend = "SimpleSpend", + UTXOFragmentation = "UTXOFragmentation", + Consolidation = "Consolidation", + MixingOrCoinJoin = "MixingOrCoinJoin", +} + +function SpendTypeScores( + spendType: SpendType, + numberOfInputs: number, + numberOfOutputs: number, +): number { + switch (spendType) { + case SpendType.SweepSpend: + return 1 / 2; + case SpendType.SimpleSpend: + return 4 / 9; + case SpendType.UTXOFragmentation: + return 2 / 3 - 1 / numberOfOutputs; + case SpendType.Consolidation: + return 1 / numberOfInputs; + case SpendType.MixingOrCoinJoin: + let x = Math.pow(numberOfOutputs, 2); + return (0.75 * x) / (1 + x); + } +} + +function determineSpendType(inputs: number, outputs: number): SpendType { + if (inputs === 1) { + if (outputs === 1) return SpendType.SweepSpend; + if (outputs === 2) return SpendType.SimpleSpend; + return SpendType.UTXOFragmentation; + } else { + if (outputs === 1) return SpendType.Consolidation; + return SpendType.MixingOrCoinJoin; + } +} + export function privscyScoreByTxTopology( transaction: any, client: BlockchainClient, @@ -30,41 +69,20 @@ export function privscyScoreByTxTopology( const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; - let score: number; - - if (numberOfInputs === 1) { - if (numberOfOutputs === 1) { - // Sweep Spend (No change Output) - // #Input = 1, #Output = 1 - score = 1 / 2; - } else if (numberOfOutputs === 2) { - // Simple Spend (Single change output) - // #Input = 1, #Output = 2 - score = 4 / 9; - } else { - // UTXO Fragmentation - // #Input = 1, #Output > 2 - score = 2 / 3 - 1 / numberOfOutputs; - } - if (isSelfPayment(transaction, client)) { - return score * DENIABILITY_FACTOR; - } - } else { - if (numberOfOutputs === 1) { - // Consolidation - // #Input >= 2, #Output = 1 - score = 1 / numberOfInputs; - - // No D.F for consolidation - } else { - // Mixing or CoinJoin - // #Input >= 2, #Output >= 2 - let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; - score = (0.75 * x) / (1 + x); - if (isSelfPayment(transaction, client)) { - return score * DENIABILITY_FACTOR; - } - } + const spendType: SpendType = determineSpendType( + numberOfInputs, + numberOfOutputs, + ); + const score: number = SpendTypeScores( + spendType, + numberOfInputs, + numberOfOutputs, + ); + if ( + isSelfPayment(transaction, client) && + spendType !== SpendType.Consolidation + ) { + return score * DENIABILITY_FACTOR; } return score; } From 26dfd30b755331bef782f01bd859af66ebd141a9 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 17 Jul 2024 19:45:57 +0530 Subject: [PATCH 09/92] Add types for Transaction and UTXO for privacy.ts Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 22 +++++++++++++++ packages/caravan-health/README.md | 6 +++++ packages/caravan-health/src/privacy.ts | 37 +++++++++++++++----------- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 1bc2ef98..e7ed2f47 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -36,6 +36,28 @@ export interface UTXO { }; } +export interface Transaction { + txid: string; + vin: Input[]; + vout: Output[]; + size: number; + weight: number; + fee: number; +} + +interface Input { + txid: string; + vout: number; + witness: string[]; + sequence: number; +} + +interface Output { + scriptPubkeyHex: string; + scriptPubkeyAddress: string; + value: number; +} + export enum ClientType { PRIVATE = "private", BLOCKSTREAM = "blockstream", diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index c5593854..32c6bbd2 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -1 +1,7 @@ # Health + +# TODO +- [] Write logic for `isAddressReused` function in `privacy.ts` +- [] Replace `any` types from `feescore.ts` +- [] Add link to blog post +- [] Cover test-cases diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 88c54ce2..ae888ecd 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,4 +1,5 @@ import { BlockchainClient } from "@caravan/clients"; +import type { UTXO, Transaction } from "@caravan/clients/src/client"; /* The methodology for calculating a privacy score (p_score) for Bitcoin transactions based on the number of inputs and outputs is the primary point to define wallet health for privacy. @@ -63,7 +64,7 @@ function determineSpendType(inputs: number, outputs: number): SpendType { } export function privscyScoreByTxTopology( - transaction: any, + transaction: Transaction, client: BlockchainClient, ): number { const numberOfInputs: number = transaction.vin.length; @@ -93,10 +94,10 @@ are the same entity. To check this, we make an RPC call to the watcher wallet as amount that a given address holds. If the call returns a number then it is part of wallet otherwise it is not a self payment. */ -function isSelfPayment(transaction: any, client: BlockchainClient): boolean { +function isSelfPayment(transaction: Transaction, client: BlockchainClient): boolean { transaction.vout.forEach(async (op) => { if ( - (await client.getAddressStatus(op.scriptPubKey.address)) === undefined + (await client.getAddressStatus(op.scriptPubkeyAddress)) === undefined ) { return false; } @@ -104,19 +105,23 @@ function isSelfPayment(transaction: any, client: BlockchainClient): boolean { return true; } -/* TODO : replace any type to custom types for Transactions and UTXOs*/ +function isReusedAddress(): boolean { + // TODO : Fix this function + return false; +} + /* In order to score for address reuse we can check the amount being hold by reused UTXOs with respect to the total amount */ -export function addressReuseFactor(utxos: Array): number { +export function addressReuseFactor(utxos: Array): number { let reusedAmount: number = 0; let totalAmount: number = 0; utxos.forEach((utxo) => { - if (utxo.reused) { - reusedAmount += utxo.amount; + if (isReusedAddress()) { + reusedAmount += utxo.value; } - totalAmount += utxo.amount; + totalAmount += utxo.value; }); return reusedAmount / totalAmount; } @@ -127,7 +132,7 @@ the change received will be to an address type matching our wallet and it will l we still own that amount. */ export function addressTypeFactor( - transactions: Array, + transactions: Array, walletAddressType: string, ): number { let P2WSH: number = 0; @@ -136,7 +141,7 @@ export function addressTypeFactor( let atf: number = 1; transactions.forEach((tx) => { tx.vout.forEach((output) => { - let address = output.scriptPubKey.address; + let address = output.scriptPubkeyAddress; if (address.startsWith("bc")) { //Bech 32 Native Segwit (P2WSH) or Taproot P2WSH += 1; @@ -175,8 +180,8 @@ The spread factor using standard deviation helps in assessing the dispersion of In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries to link transactions and deduce the ownership and spending patterns of users. */ -export function utxoSpreadFactor(utxos: Array): number { - const amounts: Array = utxos.map((utxo) => utxo.amount); +export function utxoSpreadFactor(utxos: Array): number { + const amounts: Array = utxos.map((utxo) => utxo.value); const mean: number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; const variance: number = @@ -194,7 +199,7 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ -export function utxoSetLengthWeight(utxos: Array): number { +export function utxoSetLengthWeight(utxos: Array): number { let utxoSetLength: number = utxos.length; let weight: number; if (utxoSetLength >= 50) { @@ -215,7 +220,7 @@ export function utxoSetLengthWeight(utxos: Array): number { UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. */ -export function utxoValueWeightageFactor(utxos: Array): number { +export function utxoValueWeightageFactor(utxos: Array): number { let W: number = utxoSetLengthWeight(utxos); let USF: number = utxoSpreadFactor(utxos); return (USF + W) * 0.15 - 0.15; @@ -229,8 +234,8 @@ The privacy score is a combination of all the factors calculated above. - UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F */ export function privacyScore( - transactions: Array, - utxos: Array, + transactions: Array, + utxos: Array, walletAddressType: string, client: BlockchainClient, ): number { From f1e6a3886584e175ff6789336b3f1f6db07997b9 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 17 Jul 2024 21:38:13 +0530 Subject: [PATCH 10/92] Changing UTXO type to Wallet UTXO which brings addresses from memory Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 18 ++++++++++ packages/caravan-health/README.md | 2 +- packages/caravan-health/src/privacy.ts | 46 +++++++++++++++++--------- 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index e7ed2f47..a37025c3 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -194,6 +194,24 @@ export class BlockchainClient extends ClientBase { } } + public async getAddressTransactions(address: string): Promise { + try { + if (this.type === ClientType.PRIVATE) { + return await callBitcoind( + this.bitcoindParams.url, + this.bitcoindParams.auth, + "listtransactions", + [this.bitcoindParams.walletName, 1000], + ); + } + return await this.Get(`/address/${address}/txs`); + } catch (error: any) { + throw new Error( + `Failed to get transactions for address ${address}: ${error.message}`, + ); + } + } + public async broadcastTransaction(rawTx: string): Promise { try { if (this.type === ClientType.PRIVATE) { diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 32c6bbd2..de5efde4 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -1,7 +1,7 @@ # Health # TODO -- [] Write logic for `isAddressReused` function in `privacy.ts` +- [x] Write logic for `isAddressReused` function in `privacy.ts` - [] Replace `any` types from `feescore.ts` - [] Add link to blog post - [] Cover test-cases diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index ae888ecd..877c9ff2 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,5 +1,11 @@ import { BlockchainClient } from "@caravan/clients"; import type { UTXO, Transaction } from "@caravan/clients/src/client"; + +interface WalletUTXOs { + address: string; + utxos : Array; +} + /* The methodology for calculating a privacy score (p_score) for Bitcoin transactions based on the number of inputs and outputs is the primary point to define wallet health for privacy. @@ -105,27 +111,33 @@ function isSelfPayment(transaction: Transaction, client: BlockchainClient): bool return true; } -function isReusedAddress(): boolean { - // TODO : Fix this function - return false; -} /* In order to score for address reuse we can check the amount being hold by reused UTXOs with respect to the total amount */ -export function addressReuseFactor(utxos: Array): number { +export function addressReuseFactor(utxos: Array, client: BlockchainClient): number { let reusedAmount: number = 0; let totalAmount: number = 0; + utxos.forEach((utxo) => { - if (isReusedAddress()) { - reusedAmount += utxo.value; + let address = utxo.address; + if (isReusedAddress(address,client)) { + reusedAmount += utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0); } - totalAmount += utxo.value; + totalAmount += utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0); }); return reusedAmount / totalAmount; } +function isReusedAddress(address: string, client: BlockchainClient): boolean { + client.getAddressTransactions(address).then((txs) => { + if (txs.length > 1) { + return true; + } + }); + return false; +} /* If we are making payment to other wallet types then the privacy score should decrease because the change received will be to an address type matching our wallet and it will lead to a deduction that @@ -180,8 +192,8 @@ The spread factor using standard deviation helps in assessing the dispersion of In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries to link transactions and deduce the ownership and spending patterns of users. */ -export function utxoSpreadFactor(utxos: Array): number { - const amounts: Array = utxos.map((utxo) => utxo.value); +export function utxoSpreadFactor(utxos: Array): number { + const amounts : Array = utxos.map((utxo) => utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0)); const mean: number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; const variance: number = @@ -199,7 +211,7 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ -export function utxoSetLengthWeight(utxos: Array): number { +export function utxoSetLengthWeight(utxos: Array): number { let utxoSetLength: number = utxos.length; let weight: number; if (utxoSetLength >= 50) { @@ -220,7 +232,7 @@ export function utxoSetLengthWeight(utxos: Array): number { UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. */ -export function utxoValueWeightageFactor(utxos: Array): number { +export function utxoValueWeightageFactor(utxos: Array): number { let W: number = utxoSetLengthWeight(utxos); let USF: number = utxoSpreadFactor(utxos); return (USF + W) * 0.15 - 0.15; @@ -235,22 +247,26 @@ The privacy score is a combination of all the factors calculated above. */ export function privacyScore( transactions: Array, - utxos: Array, + utxos: Array, walletAddressType: string, client: BlockchainClient, ): number { + let privacyScore = transactions.reduce( (sum, tx) => sum + privscyScoreByTxTopology(tx, client), 0, ) / transactions.length; + // Adjusting the privacy score based on the address reuse factor privacyScore = - privacyScore * (1 - 0.5 * addressReuseFactor(utxos)) + - 0.1 * (1 - addressReuseFactor(utxos)); + privacyScore * (1 - 0.5 * addressReuseFactor(utxos,client)) + + 0.1 * (1 - addressReuseFactor(utxos,client)); + // Adjusting the privacy score based on the address type factor privacyScore = privacyScore * (1 - addressTypeFactor(transactions, walletAddressType)); + // Adjusting the privacy score based on the UTXO set length and value weightage factor privacyScore = privacyScore + 0.1 * utxoValueWeightageFactor(utxos); From 7f7ed1db6f22432695e6ebb1f0eef295053dec9f Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 18 Jul 2024 02:30:17 +0530 Subject: [PATCH 11/92] Fix UTXO set length comming wrong Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 877c9ff2..bd7d5899 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,7 +1,7 @@ import { BlockchainClient } from "@caravan/clients"; import type { UTXO, Transaction } from "@caravan/clients/src/client"; -interface WalletUTXOs { +export interface WalletUTXOs { address: string; utxos : Array; } @@ -212,7 +212,7 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 1 for UTXO set length < 5 */ export function utxoSetLengthWeight(utxos: Array): number { - let utxoSetLength: number = utxos.length; + let utxoSetLength = utxos.reduce((sum, utxo) => sum + utxo.utxos.length, 0); let weight: number; if (utxoSetLength >= 50) { weight = 0; From 1f70cab1a9dbb8bccff55fd0f08364a52ca26513 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 19 Jul 2024 02:09:52 +0530 Subject: [PATCH 12/92] Adding Test Cases for Privacy Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 6 +- packages/caravan-health/src/privacy.test.ts | 237 ++++++++++++++++++++ packages/caravan-health/src/privacy.ts | 63 +++--- 3 files changed, 278 insertions(+), 28 deletions(-) create mode 100644 packages/caravan-health/src/privacy.test.ts diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index de5efde4..218c9ef0 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -1,7 +1,9 @@ # Health # TODO -- [x] Write logic for `isAddressReused` function in `privacy.ts` + +- [x] Write logic for `isAddressReused` function in `privacy.ts`. +- [x] Cover test cases for `privacy.ts`. - [] Replace `any` types from `feescore.ts` +- [] Cover test-cases for `feescore.ts` - [] Add link to blog post -- [] Cover test-cases diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts new file mode 100644 index 00000000..a61008a6 --- /dev/null +++ b/packages/caravan-health/src/privacy.test.ts @@ -0,0 +1,237 @@ +import { + privscyScoreByTxTopology, + addressReuseFactor, + addressTypeFactor, + utxoSpreadFactor, + utxoSetLengthWeight, + utxoValueWeightageFactor, + privacyScore, + WalletUTXOs, +} from "./privacy"; // Adjust the import according to the actual file name +import { BlockchainClient } from "@caravan/clients"; +import type { UTXO, Transaction } from "@caravan/clients/src/client"; + +describe("Privacy Score Functions", () => { + let mockClient: BlockchainClient; + + beforeEach(() => { + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest.fn(), + } as unknown as BlockchainClient; + }); + + describe("privscyScoreByTxTopology", () => { + it("should calculate the privacy score based on transaction topology (i.e number of inputs and outputs)", () => { + const transaction: Transaction = { + vin: [ + { + txid: "input1", + vout: 0, + witness: [], + sequence: 0, + }, + { + txid: "input2", + vout: 0, + witness: [], + sequence: 0, + }, + ], // 2 inputs + vout: [ + { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + ], // 3 Outputs + txid: "", + size: 0, + weight: 0, + fee: 0, + }; + + jest + .spyOn(mockClient, "getAddressStatus") + .mockResolvedValue(Promise.resolve(undefined)); + + const score: number = +privscyScoreByTxTopology( + transaction, + mockClient, + ).toFixed(3); + expect(score).toBe(0.92); // Example expected score based on the given inputs/outputs + }); + }); + + describe("addressReuseFactor", () => { + it("should calculate the address reuse factor", async () => { + const utxos: Array = [ + { + address: "address1", + utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], + }, + { + address: "address2", + utxos: [{ value: 8 } as UTXO], + }, + ]; + + // Mocking the client behavior for reused addresses + jest + .spyOn(mockClient, "getAddressTransactions") + .mockImplementation((address: string) => { + if (address === "address1") { + return Promise.resolve([ + { txid: "tx1" } as Transaction, + { txid: "tx2" } as Transaction, + ]); // Reused address + } else { + return Promise.resolve([{ txid: "tx3" } as Transaction]); // Not reused address + } + }); + + const factor = await addressReuseFactor(utxos, mockClient); + expect(factor).toBeCloseTo(0.652); // Example expected factor + }); + }); + + describe("addressTypeFactor", () => { + it("should calculate the address type factor", () => { + const transactions: Transaction[] = [ + { + vin: [ + { + txid: "input1", + vout: 0, + witness: [], + sequence: 0, + }, + { + txid: "input2", + vout: 1, + witness: [], + sequence: 0, + }, + ], // 2 inputs + vout: [ + { scriptPubkeyHex: "", scriptPubkeyAddress: "123", value: 0 }, + { scriptPubkeyHex: "", scriptPubkeyAddress: "bc123", value: 0 }, + { scriptPubkeyHex: "", scriptPubkeyAddress: "bc1213", value: 0 }, + ], + txid: "", + size: 0, + weight: 0, + fee: 0, + }, + ]; + + const factor = addressTypeFactor(transactions, "P2WSH"); + expect(factor).toBeCloseTo(0.333); // Example expected factor + }); + }); + + describe("utxoSpreadFactor", () => { + it("should calculate the UTXO spread factor", () => { + const utxos: Array = [ + { + address: "address1", + utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], + }, + { + address: "address2", + utxos: [{ value: 8 } as UTXO], + }, + ]; + + const factor = utxoSpreadFactor(utxos); + expect(factor).toBeCloseTo(0.778); // Example expected factor + }); + }); + + describe("utxoSetLengthWeight", () => { + it("should calculate the UTXO set length weight", () => { + const utxos: Array = [ + { + address: "address1", + utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], + }, + { + address: "address2", + utxos: [{ value: 8 } as UTXO], + }, + { + address: "address3", + utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], + }, + { + address: "address4", + utxos: [{ value: 8 } as UTXO], + }, + ]; + + const weight = utxoSetLengthWeight(utxos); + expect(weight).toBe(0.75); // Example expected weight + }); + }); + + describe("utxoValueWeightageFactor", () => { + it("should calculate the UTXO value weightage factor", () => { + const utxos: Array = [ + { + address: "address1", + utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], + }, + { + address: "address2", + utxos: [{ value: 8 } as UTXO], + }, + ]; + + const factor = utxoValueWeightageFactor(utxos); + expect(factor).toBeCloseTo(0.116); // Example expected factor + }); + }); + + // describe("privacyScore", () => { + // it("should calculate the overall privacy score", () => { + // const transactions: Transaction[] = [ + // { + // vin: [ + // { + // txid: "input1", + // vout: 0, + // witness: [], + // sequence: 0, + // }, + // { + // txid: "input2", + // vout: 1, + // witness: [], + // sequence: 0, + // }, + // ], // 2 inputs + // vout: [ + // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + // ], + // txid: "", + // size: 0, + // weight: 0, + // fee: 0, + // }, + // ]; + // const utxos: Array = [ + // { + // address: "address1", + // utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], + // }, + // { + // address: "address2", + // utxos: [{ value: 8 } as UTXO], + // }, + // ]; + + // const score = privacyScore(transactions, utxos, "P2WSH", mockClient); + // expect(score).toBeCloseTo(0.5); // Example expected overall score + // }); + // }); +}); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index bd7d5899..877fb38b 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -3,7 +3,7 @@ import type { UTXO, Transaction } from "@caravan/clients/src/client"; export interface WalletUTXOs { address: string; - utxos : Array; + utxos: Array; } /* @@ -53,7 +53,7 @@ function SpendTypeScores( case SpendType.Consolidation: return 1 / numberOfInputs; case SpendType.MixingOrCoinJoin: - let x = Math.pow(numberOfOutputs, 2); + let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; return (0.75 * x) / (1 + x); } } @@ -100,42 +100,51 @@ are the same entity. To check this, we make an RPC call to the watcher wallet as amount that a given address holds. If the call returns a number then it is part of wallet otherwise it is not a self payment. */ -function isSelfPayment(transaction: Transaction, client: BlockchainClient): boolean { +function isSelfPayment( + transaction: Transaction, + client: BlockchainClient, +): boolean { transaction.vout.forEach(async (op) => { - if ( - (await client.getAddressStatus(op.scriptPubkeyAddress)) === undefined - ) { + if ((await client.getAddressStatus(op.scriptPubkeyAddress)) === undefined) { return false; } }); return true; } - /* In order to score for address reuse we can check the amount being hold by reused UTXOs with respect to the total amount */ -export function addressReuseFactor(utxos: Array, client: BlockchainClient): number { +export async function addressReuseFactor( + utxos: Array, + client: BlockchainClient, +): Promise { let reusedAmount: number = 0; let totalAmount: number = 0; - utxos.forEach((utxo) => { - let address = utxo.address; - if (isReusedAddress(address,client)) { - reusedAmount += utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0); + for (const utxo of utxos) { + const address = utxo.address; + const reused = await isReusedAddress(address, client); + + const amount = utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0); + if (reused) { + reusedAmount += amount; } - totalAmount += utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0); - }); + totalAmount += amount; + } + return reusedAmount / totalAmount; } -function isReusedAddress(address: string, client: BlockchainClient): boolean { - client.getAddressTransactions(address).then((txs) => { - if (txs.length > 1) { - return true; - } - }); +async function isReusedAddress( + address: string, + client: BlockchainClient, +): Promise { + let txs: Array = await client.getAddressTransactions(address); + if (txs.length > 1) { + return true; + } return false; } /* @@ -193,7 +202,9 @@ In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for to link transactions and deduce the ownership and spending patterns of users. */ export function utxoSpreadFactor(utxos: Array): number { - const amounts : Array = utxos.map((utxo) => utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0)); + const amounts: Array = utxos.map((utxo) => + utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0), + ); const mean: number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; const variance: number = @@ -245,13 +256,12 @@ The privacy score is a combination of all the factors calculated above. - Address Type Factor (A.T.F) : p_adjusted = p_score * (1-A.T.F) - UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F */ -export function privacyScore( +export async function privacyScore( transactions: Array, utxos: Array, walletAddressType: string, client: BlockchainClient, -): number { - +): Promise { let privacyScore = transactions.reduce( (sum, tx) => sum + privscyScoreByTxTopology(tx, client), @@ -259,9 +269,10 @@ export function privacyScore( ) / transactions.length; // Adjusting the privacy score based on the address reuse factor + let addressReusedFactor = await addressReuseFactor(utxos, client); privacyScore = - privacyScore * (1 - 0.5 * addressReuseFactor(utxos,client)) + - 0.1 * (1 - addressReuseFactor(utxos,client)); + privacyScore * (1 - 0.5 * addressReusedFactor) + + 0.1 * (1 - addressReusedFactor); // Adjusting the privacy score based on the address type factor privacyScore = From afe92adadd041e6739428e4e4242fbd451ee269e Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 22 Jul 2024 03:06:43 +0530 Subject: [PATCH 13/92] Type Improvements and code improvement Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.ts | 27 +- packages/caravan-health/src/privacy.test.ts | 474 ++++++++++---------- packages/caravan-health/src/privacy.ts | 156 +++---- packages/caravan-health/src/types.ts | 30 ++ 4 files changed, 349 insertions(+), 338 deletions(-) create mode 100644 packages/caravan-health/src/types.ts diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 4fdfbfad..a052fc74 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,8 +1,10 @@ -import { utxoSetLengthWeight } from "./privacy"; +import { utxoSetLengthScore } from "./privacy"; +import { Transaction, AddressUtxos } from "./types"; // Utility function that helps to obtain the fee rate of the transaction -function getFeeRateForTransaction(transaction: any): number { +function getFeeRateForTransaction(transaction: Transaction): number { // TODO : Please check that do we really get the fee rate and weight both from the transaction object + // No we don't get fees let fees: number = transaction.fee; let weight: number = transaction.weight; return fees / weight; @@ -11,7 +13,7 @@ function getFeeRateForTransaction(transaction: any): number { // TODO : Implement Caching or Ticker based mechanism to reduce network latency // Utility function that helps to obtain the percentile of the fees paid by user in tx block async function getFeeRatePercentileForTransaction( - timestamp: any, + timestamp: number, feeRate: number, ) { const url: string = @@ -63,11 +65,11 @@ if any transaction was done at expensive fees or nominal fees. This can be done by calculating the percentile of the fees paid by the user in the block of the transaction. */ -export function relativeFeesScore(transactions: Array): number { +export function relativeFeesScore(transactions: Transaction[]): number { let sumRFS: number = 0; let numberOfSendTx: number = 0; - transactions.forEach(async (tx: any) => { - if (tx.category == "send") { + transactions.forEach(async (tx: Transaction) => { + if (tx.isSend === true) { numberOfSendTx++; let feeRate: number = getFeeRateForTransaction(tx); let RFS: number = await getFeeRatePercentileForTransaction( @@ -90,11 +92,11 @@ Source : https://www.clearlypayments.com/blog/what-are-cross-border-fees-in-cred This ratio is a measure of our fees spending against the fiat charges we pay. */ -export function feesToAmountRatio(transactions: Array): number { +export function feesToAmountRatio(transactions: Transaction[]): number { let sumFeesToAmountRatio: number = 0; let numberOfSendTx: number = 0; - transactions.forEach((tx: any) => { - if (tx.category === "send") { + transactions.forEach((tx: Transaction) => { + if (tx.isSend === true) { sumFeesToAmountRatio += tx.fee / tx.amount; numberOfSendTx++; } @@ -112,9 +114,12 @@ Assume the wallet is being consolidated, Thus number of UTXO will decrease and t W (Weightage of number of UTXO) will increase and this justifies that, consolidation increases the fees health since you don’t overpay them in long run. */ -export function feesScore(transactions: Array, utxos: Array): number { +export function feesScore( + transactions: Transaction[], + utxos: AddressUtxos, +): number { let RFS: number = relativeFeesScore(transactions); let FAR: number = feesToAmountRatio(transactions); - let W: number = utxoSetLengthWeight(utxos); + let W: number = utxoSetLengthScore(utxos); return 0.35 * RFS + 0.35 * FAR + 0.3 * W; } diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index a61008a6..498e5dab 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,237 +1,237 @@ -import { - privscyScoreByTxTopology, - addressReuseFactor, - addressTypeFactor, - utxoSpreadFactor, - utxoSetLengthWeight, - utxoValueWeightageFactor, - privacyScore, - WalletUTXOs, -} from "./privacy"; // Adjust the import according to the actual file name -import { BlockchainClient } from "@caravan/clients"; -import type { UTXO, Transaction } from "@caravan/clients/src/client"; - -describe("Privacy Score Functions", () => { - let mockClient: BlockchainClient; - - beforeEach(() => { - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest.fn(), - } as unknown as BlockchainClient; - }); - - describe("privscyScoreByTxTopology", () => { - it("should calculate the privacy score based on transaction topology (i.e number of inputs and outputs)", () => { - const transaction: Transaction = { - vin: [ - { - txid: "input1", - vout: 0, - witness: [], - sequence: 0, - }, - { - txid: "input2", - vout: 0, - witness: [], - sequence: 0, - }, - ], // 2 inputs - vout: [ - { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - ], // 3 Outputs - txid: "", - size: 0, - weight: 0, - fee: 0, - }; - - jest - .spyOn(mockClient, "getAddressStatus") - .mockResolvedValue(Promise.resolve(undefined)); - - const score: number = +privscyScoreByTxTopology( - transaction, - mockClient, - ).toFixed(3); - expect(score).toBe(0.92); // Example expected score based on the given inputs/outputs - }); - }); - - describe("addressReuseFactor", () => { - it("should calculate the address reuse factor", async () => { - const utxos: Array = [ - { - address: "address1", - utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], - }, - { - address: "address2", - utxos: [{ value: 8 } as UTXO], - }, - ]; - - // Mocking the client behavior for reused addresses - jest - .spyOn(mockClient, "getAddressTransactions") - .mockImplementation((address: string) => { - if (address === "address1") { - return Promise.resolve([ - { txid: "tx1" } as Transaction, - { txid: "tx2" } as Transaction, - ]); // Reused address - } else { - return Promise.resolve([{ txid: "tx3" } as Transaction]); // Not reused address - } - }); - - const factor = await addressReuseFactor(utxos, mockClient); - expect(factor).toBeCloseTo(0.652); // Example expected factor - }); - }); - - describe("addressTypeFactor", () => { - it("should calculate the address type factor", () => { - const transactions: Transaction[] = [ - { - vin: [ - { - txid: "input1", - vout: 0, - witness: [], - sequence: 0, - }, - { - txid: "input2", - vout: 1, - witness: [], - sequence: 0, - }, - ], // 2 inputs - vout: [ - { scriptPubkeyHex: "", scriptPubkeyAddress: "123", value: 0 }, - { scriptPubkeyHex: "", scriptPubkeyAddress: "bc123", value: 0 }, - { scriptPubkeyHex: "", scriptPubkeyAddress: "bc1213", value: 0 }, - ], - txid: "", - size: 0, - weight: 0, - fee: 0, - }, - ]; - - const factor = addressTypeFactor(transactions, "P2WSH"); - expect(factor).toBeCloseTo(0.333); // Example expected factor - }); - }); - - describe("utxoSpreadFactor", () => { - it("should calculate the UTXO spread factor", () => { - const utxos: Array = [ - { - address: "address1", - utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], - }, - { - address: "address2", - utxos: [{ value: 8 } as UTXO], - }, - ]; - - const factor = utxoSpreadFactor(utxos); - expect(factor).toBeCloseTo(0.778); // Example expected factor - }); - }); - - describe("utxoSetLengthWeight", () => { - it("should calculate the UTXO set length weight", () => { - const utxos: Array = [ - { - address: "address1", - utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], - }, - { - address: "address2", - utxos: [{ value: 8 } as UTXO], - }, - { - address: "address3", - utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], - }, - { - address: "address4", - utxos: [{ value: 8 } as UTXO], - }, - ]; - - const weight = utxoSetLengthWeight(utxos); - expect(weight).toBe(0.75); // Example expected weight - }); - }); - - describe("utxoValueWeightageFactor", () => { - it("should calculate the UTXO value weightage factor", () => { - const utxos: Array = [ - { - address: "address1", - utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], - }, - { - address: "address2", - utxos: [{ value: 8 } as UTXO], - }, - ]; - - const factor = utxoValueWeightageFactor(utxos); - expect(factor).toBeCloseTo(0.116); // Example expected factor - }); - }); - - // describe("privacyScore", () => { - // it("should calculate the overall privacy score", () => { - // const transactions: Transaction[] = [ - // { - // vin: [ - // { - // txid: "input1", - // vout: 0, - // witness: [], - // sequence: 0, - // }, - // { - // txid: "input2", - // vout: 1, - // witness: [], - // sequence: 0, - // }, - // ], // 2 inputs - // vout: [ - // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - // ], - // txid: "", - // size: 0, - // weight: 0, - // fee: 0, - // }, - // ]; - // const utxos: Array = [ - // { - // address: "address1", - // utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], - // }, - // { - // address: "address2", - // utxos: [{ value: 8 } as UTXO], - // }, - // ]; - - // const score = privacyScore(transactions, utxos, "P2WSH", mockClient); - // expect(score).toBeCloseTo(0.5); // Example expected overall score - // }); - // }); -}); +// import { +// privscyScoreByTxTopology, +// addressReuseFactor, +// addressTypeFactor, +// utxoSpreadFactor, +// utxoSetLengthWeight, +// utxoValueWeightageFactor, +// privacyScore, +// WalletUTXOs, +// } from "./privacy"; // Adjust the import according to the actual file name +// import { BlockchainClient } from "@caravan/clients"; +// import type { UTXO, Transaction } from "@caravan/clients/src/client"; + +// describe("Privacy Score Functions", () => { +// let mockClient: BlockchainClient; + +// beforeEach(() => { +// mockClient = { +// getAddressStatus: jest.fn(), +// getAddressTransactions: jest.fn(), +// } as unknown as BlockchainClient; +// }); + +// describe("privscyScoreByTxTopology", () => { +// it("should calculate the privacy score based on transaction topology (i.e number of inputs and outputs)", () => { +// const transaction: Transaction = { +// vin: [ +// { +// txid: "input1", +// vout: 0, +// witness: [], +// sequence: 0, +// }, +// { +// txid: "input2", +// vout: 0, +// witness: [], +// sequence: 0, +// }, +// ], // 2 inputs +// vout: [ +// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// ], // 3 Outputs +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// }; + +// jest +// .spyOn(mockClient, "getAddressStatus") +// .mockResolvedValue(Promise.resolve(undefined)); + +// const score: number = +privscyScoreByTxTopology( +// transaction, +// mockClient, +// ).toFixed(3); +// expect(score).toBe(0.92); // Example expected score based on the given inputs/outputs +// }); +// }); + +// describe("addressReuseFactor", () => { +// it("should calculate the address reuse factor", async () => { +// const utxos: Array = [ +// { +// address: "address1", +// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], +// }, +// { +// address: "address2", +// utxos: [{ value: 8 } as UTXO], +// }, +// ]; + +// // Mocking the client behavior for reused addresses +// jest +// .spyOn(mockClient, "getAddressTransactions") +// .mockImplementation((address: string) => { +// if (address === "address1") { +// return Promise.resolve([ +// { txid: "tx1" } as Transaction, +// { txid: "tx2" } as Transaction, +// ]); // Reused address +// } else { +// return Promise.resolve([{ txid: "tx3" } as Transaction]); // Not reused address +// } +// }); + +// const factor = await addressReuseFactor(utxos, mockClient); +// expect(factor).toBeCloseTo(0.652); // Example expected factor +// }); +// }); + +// describe("addressTypeFactor", () => { +// it("should calculate the address type factor", () => { +// const transactions: Transaction[] = [ +// { +// vin: [ +// { +// txid: "input1", +// vout: 0, +// witness: [], +// sequence: 0, +// }, +// { +// txid: "input2", +// vout: 1, +// witness: [], +// sequence: 0, +// }, +// ], // 2 inputs +// vout: [ +// { scriptPubkeyHex: "", scriptPubkeyAddress: "123", value: 0 }, +// { scriptPubkeyHex: "", scriptPubkeyAddress: "bc123", value: 0 }, +// { scriptPubkeyHex: "", scriptPubkeyAddress: "bc1213", value: 0 }, +// ], +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// }, +// ]; + +// const factor = addressTypeFactor(transactions, "P2WSH"); +// expect(factor).toBeCloseTo(0.333); // Example expected factor +// }); +// }); + +// describe("utxoSpreadFactor", () => { +// it("should calculate the UTXO spread factor", () => { +// const utxos: Array = [ +// { +// address: "address1", +// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], +// }, +// { +// address: "address2", +// utxos: [{ value: 8 } as UTXO], +// }, +// ]; + +// const factor = utxoSpreadFactor(utxos); +// expect(factor).toBeCloseTo(0.778); // Example expected factor +// }); +// }); + +// describe("utxoSetLengthWeight", () => { +// it("should calculate the UTXO set length weight", () => { +// const utxos: Array = [ +// { +// address: "address1", +// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], +// }, +// { +// address: "address2", +// utxos: [{ value: 8 } as UTXO], +// }, +// { +// address: "address3", +// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], +// }, +// { +// address: "address4", +// utxos: [{ value: 8 } as UTXO], +// }, +// ]; + +// const weight = utxoSetLengthWeight(utxos); +// expect(weight).toBe(0.75); // Example expected weight +// }); +// }); + +// describe("utxoValueWeightageFactor", () => { +// it("should calculate the UTXO value weightage factor", () => { +// const utxos: Array = [ +// { +// address: "address1", +// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], +// }, +// { +// address: "address2", +// utxos: [{ value: 8 } as UTXO], +// }, +// ]; + +// const factor = utxoValueWeightageFactor(utxos); +// expect(factor).toBeCloseTo(0.116); // Example expected factor +// }); +// }); + +// // describe("privacyScore", () => { +// // it("should calculate the overall privacy score", () => { +// // const transactions: Transaction[] = [ +// // { +// // vin: [ +// // { +// // txid: "input1", +// // vout: 0, +// // witness: [], +// // sequence: 0, +// // }, +// // { +// // txid: "input2", +// // vout: 1, +// // witness: [], +// // sequence: 0, +// // }, +// // ], // 2 inputs +// // vout: [ +// // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// // ], +// // txid: "", +// // size: 0, +// // weight: 0, +// // fee: 0, +// // }, +// // ]; +// // const utxos: Array = [ +// // { +// // address: "address1", +// // utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], +// // }, +// // { +// // address: "address2", +// // utxos: [{ value: 8 } as UTXO], +// // }, +// // ]; + +// // const score = privacyScore(transactions, utxos, "P2WSH", mockClient); +// // expect(score).toBeCloseTo(0.5); // Example expected overall score +// // }); +// // }); +// }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 877fb38b..2e4f1a13 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,10 +1,6 @@ import { BlockchainClient } from "@caravan/clients"; -import type { UTXO, Transaction } from "@caravan/clients/src/client"; - -export interface WalletUTXOs { - address: string; - utxos: Array; -} +import { Transaction, AddressUtxos } from "./types"; +import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; /* The methodology for calculating a privacy score (p_score) for Bitcoin transactions based @@ -26,7 +22,7 @@ change outputs and the type of transaction based on number of inputs and outputs We have 5 categories of transaction type - Sweep Spend - Simple Spend -- UTXO Fragmentation +- UTXO Fragmentation (any transaction with more than the standard 2 outputs) - Consolidation - CoinJoin */ @@ -38,7 +34,7 @@ enum SpendType { MixingOrCoinJoin = "MixingOrCoinJoin", } -function SpendTypeScores( +function spendTypeScores( spendType: SpendType, numberOfInputs: number, numberOfOutputs: number, @@ -55,6 +51,8 @@ function SpendTypeScores( case SpendType.MixingOrCoinJoin: let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; return (0.75 * x) / (1 + x); + default: + return -1; } } @@ -69,7 +67,7 @@ function determineSpendType(inputs: number, outputs: number): SpendType { } } -export function privscyScoreByTxTopology( +export function privacyScoreByTxTopology( transaction: Transaction, client: BlockchainClient, ): number { @@ -80,60 +78,38 @@ export function privscyScoreByTxTopology( numberOfInputs, numberOfOutputs, ); - const score: number = SpendTypeScores( + const score: number = spendTypeScores( spendType, numberOfInputs, numberOfOutputs, ); - if ( - isSelfPayment(transaction, client) && - spendType !== SpendType.Consolidation - ) { + if (transaction.isSend && spendType !== SpendType.Consolidation) { return score * DENIABILITY_FACTOR; } return score; } -/* -Determine whether a Bitcoin transaction is a self-payment, meaning the sender and recipient -are the same entity. To check this, we make an RPC call to the watcher wallet asking for the -amount that a given address holds. If the call returns a number then it is part of wallet -otherwise it is not a self payment. -*/ -function isSelfPayment( - transaction: Transaction, - client: BlockchainClient, -): boolean { - transaction.vout.forEach(async (op) => { - if ((await client.getAddressStatus(op.scriptPubkeyAddress)) === undefined) { - return false; - } - }); - return true; -} - /* In order to score for address reuse we can check the amount being hold by reused UTXOs with respect to the total amount */ export async function addressReuseFactor( - utxos: Array, + utxos: AddressUtxos, client: BlockchainClient, ): Promise { let reusedAmount: number = 0; let totalAmount: number = 0; - for (const utxo of utxos) { - const address = utxo.address; - const reused = await isReusedAddress(address, client); - - const amount = utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0); - if (reused) { - reusedAmount += amount; + for (const address in utxos) { + const addressUtxos = utxos[address]; + for (const utxo of addressUtxos) { + totalAmount += utxo.value; + let isReused = await isReusedAddress(address, client); + if (isReused) { + reusedAmount += utxo.value; + } } - totalAmount += amount; } - return reusedAmount / totalAmount; } @@ -141,7 +117,7 @@ async function isReusedAddress( address: string, client: BlockchainClient, ): Promise { - let txs: Array = await client.getAddressTransactions(address); + let txs: Transaction[] = await client.getAddressTransactions(address); if (txs.length > 1) { return true; } @@ -153,47 +129,36 @@ the change received will be to an address type matching our wallet and it will l we still own that amount. */ export function addressTypeFactor( - transactions: Array, - walletAddressType: string, + transactions: Transaction[], + walletAddressType: MultisigAddressType, + network: Network, ): number { - let P2WSH: number = 0; - let P2PKH: number = 0; - let P2SH: number = 0; - let atf: number = 1; + const addressCounts: Record = { + P2WSH: 0, + P2SH: 0, + P2PKH: 0, + P2TR: 0, + UNKNOWN: 0, + "P2SH-P2WSH": 0, + }; + transactions.forEach((tx) => { tx.vout.forEach((output) => { - let address = output.scriptPubkeyAddress; - if (address.startsWith("bc")) { - //Bech 32 Native Segwit (P2WSH) or Taproot - P2WSH += 1; - } else if (address.startsWith("1")) { - // Legacy (P2PKH) - P2PKH += 1; - } else { - // Segwith (P2SH) - P2SH += 1; - } + const addressType = getAddressType(output.scriptPubkeyAddress, network); + addressCounts[addressType]++; }); }); - if (walletAddressType == "P2WSH" && P2WSH != 0 && (P2SH != 0 || P2PKH != 0)) { - atf = 1 / (P2WSH + 1); - } else if ( - walletAddressType == "P2PKH" && - P2PKH != 0 && - (P2SH != 0 || P2WSH != 0) - ) { - atf = 1 / (P2PKH + 1); - } else if ( - walletAddressType == "P2SH" && - P2SH != 0 && - (P2WSH != 0 || P2PKH != 0) - ) { - atf = 1 / (P2SH + 1); - } else { - atf = 1; + const totalAddresses = Object.values(addressCounts).reduce( + (a, b) => a + b, + 0, + ); + const walletTypeCount = addressCounts[walletAddressType]; + + if (walletTypeCount === 0 || totalAddresses === walletTypeCount) { + return 1; } - return atf; + return 1 / (walletTypeCount + 1); } /* @@ -201,10 +166,15 @@ The spread factor using standard deviation helps in assessing the dispersion of In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries to link transactions and deduce the ownership and spending patterns of users. */ -export function utxoSpreadFactor(utxos: Array): number { - const amounts: Array = utxos.map((utxo) => - utxo.utxos.reduce((sum, utxo) => sum + utxo.value, 0), - ); +export function utxoSpreadFactor(utxos: AddressUtxos): number { + const amounts: number[] = []; + for (const address in utxos) { + const addressUtxos = utxos[address]; + addressUtxos.forEach((utxo) => { + amounts.push(utxo.value); + }); + } + const mean: number = amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; const variance: number = @@ -222,8 +192,12 @@ The weightage is ad-hoc to normalize the privacy score based on the number of UT - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ -export function utxoSetLengthWeight(utxos: Array): number { - let utxoSetLength = utxos.reduce((sum, utxo) => sum + utxo.utxos.length, 0); +export function utxoSetLengthScore(utxos: AddressUtxos): number { + let utxoSetLength = 0; + for (const address in utxos) { + const addressUtxos = utxos[address]; + utxoSetLength += addressUtxos.length; + } let weight: number; if (utxoSetLength >= 50) { weight = 0; @@ -243,8 +217,8 @@ export function utxoSetLengthWeight(utxos: Array): number { UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. */ -export function utxoValueWeightageFactor(utxos: Array): number { - let W: number = utxoSetLengthWeight(utxos); +export function utxoValueWeightageFactor(utxos: AddressUtxos): number { + let W: number = utxoSetLengthScore(utxos); let USF: number = utxoSpreadFactor(utxos); return (USF + W) * 0.15 - 0.15; } @@ -257,14 +231,15 @@ The privacy score is a combination of all the factors calculated above. - UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F */ export async function privacyScore( - transactions: Array, - utxos: Array, - walletAddressType: string, + transactions: Transaction[], + utxos: AddressUtxos, + walletAddressType: MultisigAddressType, client: BlockchainClient, + network: Network, ): Promise { let privacyScore = transactions.reduce( - (sum, tx) => sum + privscyScoreByTxTopology(tx, client), + (sum, tx) => sum + privacyScoreByTxTopology(tx, client), 0, ) / transactions.length; @@ -276,7 +251,8 @@ export async function privacyScore( // Adjusting the privacy score based on the address type factor privacyScore = - privacyScore * (1 - addressTypeFactor(transactions, walletAddressType)); + privacyScore * + (1 - addressTypeFactor(transactions, walletAddressType, network)); // Adjusting the privacy score based on the UTXO set length and value weightage factor privacyScore = privacyScore + 0.1 * utxoValueWeightageFactor(utxos); diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts new file mode 100644 index 00000000..b1ee21b4 --- /dev/null +++ b/packages/caravan-health/src/types.ts @@ -0,0 +1,30 @@ +import { UTXO } from "@caravan/clients"; + +export interface AddressUtxos { + [address: string]: UTXO[]; +} + +export interface Transaction { + txid: string; + vin: Input[]; + vout: Output[]; + size: number; + weight: number; + fee: number; + isSend: boolean; + amount: number; + blocktime: number; +} + +interface Input { + txid: string; + vout: number; + witness: string[]; + sequence: number; +} + +interface Output { + scriptPubkeyHex: string; + scriptPubkeyAddress: string; + value: number; +} From 0011b2181a7231b1157a47902eaf3eecd3a47701 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 22 Jul 2024 03:09:14 +0530 Subject: [PATCH 14/92] Better handling of exports and address functionality Signed-off-by: Harshil-Jani --- .../caravan-bitcoin/src/addresses.test.ts | 53 ++++++++++++++++++- packages/caravan-bitcoin/src/addresses.ts | 29 ++++++++++ .../caravan-bitcoin/src/types/addresses.ts | 2 +- packages/caravan-clients/src/client.ts | 22 -------- packages/caravan-clients/src/index.ts | 1 + packages/caravan-health/src/index.ts | 4 +- 6 files changed, 86 insertions(+), 25 deletions(-) diff --git a/packages/caravan-bitcoin/src/addresses.test.ts b/packages/caravan-bitcoin/src/addresses.test.ts index b79a8252..45dd8218 100644 --- a/packages/caravan-bitcoin/src/addresses.test.ts +++ b/packages/caravan-bitcoin/src/addresses.test.ts @@ -1,9 +1,10 @@ -import { validateAddress } from "./addresses"; +import { validateAddress, getAddressType } from "./addresses"; import * as multisig from "./multisig"; import { Network } from "./networks"; const P2PKH = "P2PKH"; const P2TR = "P2TR"; +const P2WSH = "P2WSH"; const ADDRESSES = {}; ADDRESSES[Network.MAINNET] = {}; @@ -104,4 +105,54 @@ describe("addresses", () => { }); }); }); + + describe("getAddressType", () => { + it("correctly identifies P2SH addresses", () => { + ADDRESSES[Network.MAINNET][P2PKH].forEach((address) => { + expect(getAddressType(address, Network.MAINNET)).toBe(P2PKH); + }); + ADDRESSES[Network.TESTNET][P2PKH].forEach((address) => { + expect(getAddressType(address, Network.TESTNET)).toBe("P2PKH"); + }); + ADDRESSES[Network.REGTEST][P2PKH].forEach((address) => { + expect(getAddressType(address, Network.REGTEST)).toBe(P2PKH); + }); + }); + + it("correctly identifies P2WSH addresses", () => { + ADDRESSES[Network.MAINNET][(multisig as any).P2WSH].forEach((address) => { + expect(getAddressType(address, Network.MAINNET)).toBe(P2WSH); + }); + ADDRESSES[Network.TESTNET][(multisig as any).P2WSH].forEach((address) => { + expect(getAddressType(address, Network.TESTNET)).toBe(P2WSH); + }); + ADDRESSES[Network.REGTEST][(multisig as any).P2WSH].forEach((address) => { + expect(getAddressType(address, Network.REGTEST)).toBe(P2WSH); + }); + }); + + it("correctly identifies P2TR addresses", () => { + ADDRESSES[Network.MAINNET][P2TR].forEach((address) => { + expect(getAddressType(address, Network.MAINNET)).toBe(P2TR); + }); + ADDRESSES[Network.TESTNET][P2TR].forEach((address) => { + expect(getAddressType(address, Network.TESTNET)).toBe(P2TR); + }); + ADDRESSES[Network.REGTEST][P2TR].forEach((address) => { + expect(getAddressType(address, Network.REGTEST)).toBe(P2TR); + }); + }); + + it("returns UNKNOWN for unrecognized addresses", () => { + expect(getAddressType("unknownaddress1", Network.MAINNET)).toBe( + "UNKNOWN", + ); + expect(getAddressType("unknownaddress2", Network.TESTNET)).toBe( + "UNKNOWN", + ); + expect(getAddressType("unknownaddress3", Network.REGTEST)).toBe( + "UNKNOWN", + ); + }); + }); }); diff --git a/packages/caravan-bitcoin/src/addresses.ts b/packages/caravan-bitcoin/src/addresses.ts index 557a083a..acb288dd 100644 --- a/packages/caravan-bitcoin/src/addresses.ts +++ b/packages/caravan-bitcoin/src/addresses.ts @@ -8,6 +8,7 @@ import { } from "bitcoin-address-validation"; import { Network } from "./networks"; +import { MultisigAddressType } from "./types"; const MAINNET_ADDRESS_MAGIC_BYTE_PATTERN = "^(bc1|[13])"; const TESTNET_ADDRESS_MAGIC_BYTE_PATTERN = "^(tb1|bcrt1|[mn2])"; @@ -60,3 +61,31 @@ export function validateAddress(address: string, network: Network) { return valid ? "" : "Address is invalid."; } + +export function getAddressType( + address: string, + network: Network, +): MultisigAddressType { + if (validateAddress(address, network) !== "") { + return "UNKNOWN"; + } + const bech32Regex = /^(bc1|tb1|bcrt1)/; + const p2pkhRegex = /^(1|m|n)/; + const p2shRegex = /^(3|2)/; + + if (address.match(bech32Regex)) { + if ( + address.startsWith("bc1p") || + address.startsWith("tb1p") || + address.startsWith("bcrt1p") + ) { + return "P2TR"; + } + return "P2WSH"; + } else if (address.match(p2pkhRegex)) { + return "P2PKH"; + } else if (address.match(p2shRegex)) { + return "P2SH"; + } + return "UNKNOWN"; +} diff --git a/packages/caravan-bitcoin/src/types/addresses.ts b/packages/caravan-bitcoin/src/types/addresses.ts index d030b457..ac71e7cb 100644 --- a/packages/caravan-bitcoin/src/types/addresses.ts +++ b/packages/caravan-bitcoin/src/types/addresses.ts @@ -2,4 +2,4 @@ // address type. // We should be able to replace this with use of the MULTISIG_ADDRESS_TYPES // enum when that file (./multisig.js) gets converted to typescript -export type MultisigAddressType = "P2SH" | "P2WSH" | "P2SH-P2WSH" | "P2TR"; +export type MultisigAddressType = "P2SH" | "P2WSH" | "P2SH-P2WSH" | "P2TR" | "P2PKH" | "UNKNOWN"; diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index a37025c3..ce697f12 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -36,28 +36,6 @@ export interface UTXO { }; } -export interface Transaction { - txid: string; - vin: Input[]; - vout: Output[]; - size: number; - weight: number; - fee: number; -} - -interface Input { - txid: string; - vout: number; - witness: string[]; - sequence: number; -} - -interface Output { - scriptPubkeyHex: string; - scriptPubkeyAddress: string; - value: number; -} - export enum ClientType { PRIVATE = "private", BLOCKSTREAM = "blockstream", diff --git a/packages/caravan-clients/src/index.ts b/packages/caravan-clients/src/index.ts index a1f928fc..3774161a 100644 --- a/packages/caravan-clients/src/index.ts +++ b/packages/caravan-clients/src/index.ts @@ -1,2 +1,3 @@ export { bitcoindImportDescriptors } from "./wallet"; export { BlockchainClient, ClientType } from "./client"; +export type { UTXO } from "./client"; diff --git a/packages/caravan-health/src/index.ts b/packages/caravan-health/src/index.ts index 79e01ac2..302971de 100644 --- a/packages/caravan-health/src/index.ts +++ b/packages/caravan-health/src/index.ts @@ -1 +1,3 @@ -export { privacyScore } from "./privacy"; +export * from "./privacy"; +export * from "./types"; +export * from "./feescore"; From 270a0356be6735b9edea1bc460f2788d8b53232d Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 22 Jul 2024 16:28:53 +0530 Subject: [PATCH 15/92] Rewrite test cases for privacy Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 480 ++++++++++---------- packages/caravan-health/src/privacy.ts | 29 +- 2 files changed, 271 insertions(+), 238 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 498e5dab..a6970495 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,237 +1,261 @@ -// import { -// privscyScoreByTxTopology, -// addressReuseFactor, -// addressTypeFactor, -// utxoSpreadFactor, -// utxoSetLengthWeight, -// utxoValueWeightageFactor, -// privacyScore, -// WalletUTXOs, -// } from "./privacy"; // Adjust the import according to the actual file name -// import { BlockchainClient } from "@caravan/clients"; -// import type { UTXO, Transaction } from "@caravan/clients/src/client"; +import { + privacyScoreByTxTopology, + addressReuseFactor, + addressTypeFactor, + utxoSpreadFactor, + utxoSetLengthScore, + utxoValueWeightageFactor, + privacyScore, +} from "./privacy"; // Adjust the import according to the actual file name +import { BlockchainClient } from "@caravan/clients"; +import { Transaction } from "./types"; +import { AddressUtxos } from "./types"; +import { MultisigAddressType, Network } from "@caravan/bitcoin"; -// describe("Privacy Score Functions", () => { -// let mockClient: BlockchainClient; +describe("Privacy Score Functions", () => { + let mockClient: BlockchainClient; -// beforeEach(() => { -// mockClient = { -// getAddressStatus: jest.fn(), -// getAddressTransactions: jest.fn(), -// } as unknown as BlockchainClient; -// }); + beforeEach(() => { + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest.fn().mockResolvedValue([{ txid: "tx1" }]), + } as unknown as BlockchainClient; + }); -// describe("privscyScoreByTxTopology", () => { -// it("should calculate the privacy score based on transaction topology (i.e number of inputs and outputs)", () => { -// const transaction: Transaction = { -// vin: [ -// { -// txid: "input1", -// vout: 0, -// witness: [], -// sequence: 0, -// }, -// { -// txid: "input2", -// vout: 0, -// witness: [], -// sequence: 0, -// }, -// ], // 2 inputs -// vout: [ -// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// ], // 3 Outputs -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// }; + describe("privacyScoreByTxTopology", () => { + it("CoinJoin with reused address", async () => { + const transaction: Transaction = { + vin: [ + { + txid: "input1", + vout: 0, + witness: [], + sequence: 0, + }, + { + txid: "input2", + vout: 0, + witness: [], + sequence: 0, + }, + ], // 2 inputs + vout: [ + { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, + ], // 3 Outputs + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }; + const score: number = +( + await privacyScoreByTxTopology(transaction, mockClient) + ).toFixed(3); + expect(score).toBe(0.92); + }); + }); -// jest -// .spyOn(mockClient, "getAddressStatus") -// .mockResolvedValue(Promise.resolve(undefined)); + describe("addressReuseFactor", () => { + it("UTXOs having same addresses", async () => { + const utxos: AddressUtxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx3", + vout: 0, + value: 3, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const factor: number = +( + await addressReuseFactor(utxos, mockClient) + ).toFixed(3); + expect(factor).toBe(0); + }); + }); -// const score: number = +privscyScoreByTxTopology( -// transaction, -// mockClient, -// ).toFixed(3); -// expect(score).toBe(0.92); // Example expected score based on the given inputs/outputs -// }); -// }); + describe("addressTypeFactor", () => { + it("P2PKH address", () => { + const transactions = [ + { + vin: [ + { + txid: "input1", + vout: 0, + witness: [], + sequence: 0, + }, + ], + vout: [ + { scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }, + ], + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + ]; + const walletAddressType: MultisigAddressType = "P2PKH"; + const network: Network = Network["MAINNET"]; + const factor: number = +addressTypeFactor( + transactions, + walletAddressType, + network, + ).toFixed(3); + expect(factor).toBe(1); + }); + }); -// describe("addressReuseFactor", () => { -// it("should calculate the address reuse factor", async () => { -// const utxos: Array = [ -// { -// address: "address1", -// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], -// }, -// { -// address: "address2", -// utxos: [{ value: 8 } as UTXO], -// }, -// ]; + describe("utxoSpreadFactor", () => { + it("UTXOs spread across multiple addresses", () => { + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const factor: number = +utxoSpreadFactor(utxos).toFixed(3); + expect(factor).toBe(0.333); + }); + }); -// // Mocking the client behavior for reused addresses -// jest -// .spyOn(mockClient, "getAddressTransactions") -// .mockImplementation((address: string) => { -// if (address === "address1") { -// return Promise.resolve([ -// { txid: "tx1" } as Transaction, -// { txid: "tx2" } as Transaction, -// ]); // Reused address -// } else { -// return Promise.resolve([{ txid: "tx3" } as Transaction]); // Not reused address -// } -// }); + describe("utxoSetLengthScore", () => { + it("UTXO set length", () => { + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const score: number = +utxoSetLengthScore(utxos).toFixed(3); + expect(score).toBe(1); + }); + }); -// const factor = await addressReuseFactor(utxos, mockClient); -// expect(factor).toBeCloseTo(0.652); // Example expected factor -// }); -// }); + describe("utxoValueWeightageFactor", () => { + it("UTXO value weightage", () => { + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const factor: number = +utxoValueWeightageFactor(utxos).toFixed(3); + expect(factor).toBe(0.05); + }); + }); -// describe("addressTypeFactor", () => { -// it("should calculate the address type factor", () => { -// const transactions: Transaction[] = [ -// { -// vin: [ -// { -// txid: "input1", -// vout: 0, -// witness: [], -// sequence: 0, -// }, -// { -// txid: "input2", -// vout: 1, -// witness: [], -// sequence: 0, -// }, -// ], // 2 inputs -// vout: [ -// { scriptPubkeyHex: "", scriptPubkeyAddress: "123", value: 0 }, -// { scriptPubkeyHex: "", scriptPubkeyAddress: "bc123", value: 0 }, -// { scriptPubkeyHex: "", scriptPubkeyAddress: "bc1213", value: 0 }, -// ], -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// }, -// ]; - -// const factor = addressTypeFactor(transactions, "P2WSH"); -// expect(factor).toBeCloseTo(0.333); // Example expected factor -// }); -// }); - -// describe("utxoSpreadFactor", () => { -// it("should calculate the UTXO spread factor", () => { -// const utxos: Array = [ -// { -// address: "address1", -// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], -// }, -// { -// address: "address2", -// utxos: [{ value: 8 } as UTXO], -// }, -// ]; - -// const factor = utxoSpreadFactor(utxos); -// expect(factor).toBeCloseTo(0.778); // Example expected factor -// }); -// }); - -// describe("utxoSetLengthWeight", () => { -// it("should calculate the UTXO set length weight", () => { -// const utxos: Array = [ -// { -// address: "address1", -// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], -// }, -// { -// address: "address2", -// utxos: [{ value: 8 } as UTXO], -// }, -// { -// address: "address3", -// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], -// }, -// { -// address: "address4", -// utxos: [{ value: 8 } as UTXO], -// }, -// ]; - -// const weight = utxoSetLengthWeight(utxos); -// expect(weight).toBe(0.75); // Example expected weight -// }); -// }); - -// describe("utxoValueWeightageFactor", () => { -// it("should calculate the UTXO value weightage factor", () => { -// const utxos: Array = [ -// { -// address: "address1", -// utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], -// }, -// { -// address: "address2", -// utxos: [{ value: 8 } as UTXO], -// }, -// ]; - -// const factor = utxoValueWeightageFactor(utxos); -// expect(factor).toBeCloseTo(0.116); // Example expected factor -// }); -// }); - -// // describe("privacyScore", () => { -// // it("should calculate the overall privacy score", () => { -// // const transactions: Transaction[] = [ -// // { -// // vin: [ -// // { -// // txid: "input1", -// // vout: 0, -// // witness: [], -// // sequence: 0, -// // }, -// // { -// // txid: "input2", -// // vout: 1, -// // witness: [], -// // sequence: 0, -// // }, -// // ], // 2 inputs -// // vout: [ -// // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// // { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// // ], -// // txid: "", -// // size: 0, -// // weight: 0, -// // fee: 0, -// // }, -// // ]; -// // const utxos: Array = [ -// // { -// // address: "address1", -// // utxos: [{ value: 10 } as UTXO, { value: 5 } as UTXO], -// // }, -// // { -// // address: "address2", -// // utxos: [{ value: 8 } as UTXO], -// // }, -// // ]; - -// // const score = privacyScore(transactions, utxos, "P2WSH", mockClient); -// // expect(score).toBeCloseTo(0.5); // Example expected overall score -// // }); -// // }); -// }); + describe("privacyScore", () => { + it("Privacy score", async () => { + const transactions = [ + { + vin: [ + { + txid: "input1", + vout: 0, + witness: [], + sequence: 0, + }, + ], + vout: [ + { scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }, + ], + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + ]; + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const walletAddressType: MultisigAddressType = "P2PKH"; + const network: Network = Network["MAINNET"]; + const score: number = +( + await privacyScore( + transactions, + utxos, + walletAddressType, + mockClient, + network, + ) + ).toFixed(3); + expect(score).toBe(0.005); + }); + }); +}); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 2e4f1a13..3f2c236e 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -67,10 +67,10 @@ function determineSpendType(inputs: number, outputs: number): SpendType { } } -export function privacyScoreByTxTopology( +export async function privacyScoreByTxTopology( transaction: Transaction, client: BlockchainClient, -): number { +): Promise { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -83,10 +83,18 @@ export function privacyScoreByTxTopology( numberOfInputs, numberOfOutputs, ); - if (transaction.isSend && spendType !== SpendType.Consolidation) { - return score * DENIABILITY_FACTOR; + + if (spendType === SpendType.Consolidation) { + return score; } - return score; + for (let op of transaction.vout) { + let address = op.scriptPubkeyAddress; + let isResued = await isReusedAddress(address, client); + if (isResued === true) { + return score; + } + } + return score * DENIABILITY_FACTOR; } /* @@ -237,11 +245,12 @@ export async function privacyScore( client: BlockchainClient, network: Network, ): Promise { - let privacyScore = - transactions.reduce( - (sum, tx) => sum + privacyScoreByTxTopology(tx, client), - 0, - ) / transactions.length; + let privacyScore = 0; + for (let tx of transactions) { + let topologyScore = await privacyScoreByTxTopology(tx, client); + privacyScore += topologyScore; + } + privacyScore = privacyScore / transactions.length; // Adjusting the privacy score based on the address reuse factor let addressReusedFactor = await addressReuseFactor(utxos, client); From fc589e2077dfb7874e8444f0e19d61c7807a913c Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 22 Jul 2024 17:46:06 +0530 Subject: [PATCH 16/92] Fix in privacy and it's test case Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 2 +- packages/caravan-health/src/privacy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index a6970495..c1dc0af1 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -6,7 +6,7 @@ import { utxoSetLengthScore, utxoValueWeightageFactor, privacyScore, -} from "./privacy"; // Adjust the import according to the actual file name +} from "./privacy"; import { BlockchainClient } from "@caravan/clients"; import { Transaction } from "./types"; import { AddressUtxos } from "./types"; diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 3f2c236e..1173ce42 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -52,7 +52,7 @@ function spendTypeScores( let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; return (0.75 * x) / (1 + x); default: - return -1; + throw new Error("Invalid spend type"); } } From afb69a479fde8e09377b0ccee6b77c54f9feea69 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 22 Jul 2024 18:19:30 +0530 Subject: [PATCH 17/92] Completed with feescore test cases Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 48 ++++++++++ packages/caravan-health/src/feescore.test.ts | 97 ++++++++++++++++++++ packages/caravan-health/src/feescore.ts | 68 ++++++-------- 3 files changed, 172 insertions(+), 41 deletions(-) create mode 100644 packages/caravan-health/src/feescore.test.ts diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index ce697f12..9f0660a1 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -333,6 +333,54 @@ export class BlockchainClient extends ClientBase { } } + // TODO : Implement Caching or Ticker based mechanism to reduce network latency + public async getFeeRatePercentileForTransaction( + timestamp: number, + feeRate: number, + ): Promise { + try { + switch (this.type) { + case ClientType.PRIVATE: + // TODO : Implement it for private client + case ClientType.BLOCKSTREAM: + // TODO : Implement it for blockstream client + case ClientType.MEMPOOL: + let data = await this.Get(`v1/mining/blocks/fee-rates/all`); + // Find the closest entry by timestamp + let closestEntry: any; + let closestDifference: number = Infinity; + + data.forEach((item) => { + const difference = Math.abs(item.timestamp - timestamp); + if (difference < closestDifference) { + closestDifference = difference; + closestEntry = item; + } + }); + switch (closestEntry) { + case feeRate < closestEntry.avgFee_10: + return 10; + case feeRate < closestEntry.avgFee_25: + return 25; + case feeRate < closestEntry.avgFee_50: + return 50; + case feeRate < closestEntry.avgFee_75: + return 75; + case feeRate < closestEntry.avgFee_90: + return 90; + case feeRate < closestEntry.avgFee_100: + return 100; + default: + return 0; + } + default: + throw new Error("Invalid client type"); + } + } catch (error: any) { + throw new Error(`Failed to get fee estimate: ${error.message}`); + } + } + public async getTransactionHex(txid: string): Promise { try { if (this.type === ClientType.PRIVATE) { diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/feescore.test.ts new file mode 100644 index 00000000..c50edaee --- /dev/null +++ b/packages/caravan-health/src/feescore.test.ts @@ -0,0 +1,97 @@ +import { feesScore, feesToAmountRatio, relativeFeesScore } from "./feescore"; +import { BlockchainClient } from "@caravan/clients"; +import { Transaction } from "./types"; +import { AddressUtxos } from "./types"; +import { MultisigAddressType, Network } from "@caravan/bitcoin"; + +describe("Fees Score Functions", () => { + let mockClient: BlockchainClient; + + beforeEach(() => { + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest.fn().mockResolvedValue([{ txid: "tx1" }]), + getFeeRatePercentileForTransaction: jest.fn().mockResolvedValue(10), + } as unknown as BlockchainClient; + }); + + describe("relativeFeesScore", () => { + it("Relative fees score for transaction", async () => { + const transactions: Transaction[] = [ + { + vin: [], + vout: [], + txid: "tx1", + size: 0, + weight: 0, + fee: 0, + isSend: true, + amount: 0, + blocktime: 0, + }, + ]; + const score: number = +( + await relativeFeesScore(transactions, mockClient) + ).toFixed(3); + expect(score).toBe(1); + }); + }); + + describe("feesToAmountRatio", () => { + it("Fees to amount ratio for transaction", async () => { + const transaction: Transaction[] = [ + { + vin: [], + vout: [], + txid: "tx1", + size: 0, + weight: 0, + fee: 1, + isSend: true, + amount: 10, + blocktime: 0, + }, + ]; + const ratio: number = +feesToAmountRatio(transaction).toFixed(3); + expect(ratio).toBe(0.1); + }); + }); + + describe("feesScore", () => { + it("Fees score for transaction", async () => { + const transaction: Transaction[] = [ + { + vin: [], + vout: [], + txid: "tx1", + size: 0, + weight: 0, + fee: 1, + isSend: true, + amount: 10, + blocktime: 0, + }, + ]; + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const score: number = +( + await feesScore(transaction, utxos, mockClient) + ).toFixed(3); + expect(score).toBe(0.685); + }); + }); +}); diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index a052fc74..fa04ebc0 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,3 +1,4 @@ +import { BlockchainClient } from "@caravan/clients"; import { utxoSetLengthScore } from "./privacy"; import { Transaction, AddressUtxos } from "./types"; @@ -10,48 +11,28 @@ function getFeeRateForTransaction(transaction: Transaction): number { return fees / weight; } -// TODO : Implement Caching or Ticker based mechanism to reduce network latency // Utility function that helps to obtain the percentile of the fees paid by user in tx block -async function getFeeRatePercentileForTransaction( +async function getFeeRatePercentileScore( timestamp: number, feeRate: number, + client: BlockchainClient ) { - const url: string = - "https://mempool.space/api/v1/mining/blocks/fee-rates/all"; - const headers: Headers = new Headers(); - headers.set("Content-Type", "application/json"); - - const response: Response = await fetch(url, { - method: "GET", - headers: headers, - }); - - const data: Array = await response.json(); - - // Find the closest entry by timestamp - let closestEntry: any; - let closestDifference: number = Infinity; - - data.forEach((item) => { - const difference = Math.abs(item.timestamp - timestamp); - if (difference < closestDifference) { - closestDifference = difference; - closestEntry = item; - } - }); - - switch (true) { - case feeRate < closestEntry.avgFee_10: + let percentile: number = await client.getFeeRatePercentileForTransaction( + timestamp, + feeRate + ); + switch (percentile) { + case 10: return 1; - case feeRate < closestEntry.avgFee_25: + case 25: return 0.9; - case feeRate < closestEntry.avgFee_50: + case 50: return 0.75; - case feeRate < closestEntry.avgFee_75: + case 75: return 0.5; - case feeRate < closestEntry.avgFee_90: + case 90: return 0.25; - case feeRate < closestEntry.avgFee_100: + case 100: return 0.1; default: return 0; @@ -65,20 +46,24 @@ if any transaction was done at expensive fees or nominal fees. This can be done by calculating the percentile of the fees paid by the user in the block of the transaction. */ -export function relativeFeesScore(transactions: Transaction[]): number { +export async function relativeFeesScore( + transactions: Transaction[], + client: BlockchainClient +): Promise { let sumRFS: number = 0; let numberOfSendTx: number = 0; - transactions.forEach(async (tx: Transaction) => { + for (const tx of transactions) { if (tx.isSend === true) { numberOfSendTx++; let feeRate: number = getFeeRateForTransaction(tx); - let RFS: number = await getFeeRatePercentileForTransaction( + let RFS: number = await getFeeRatePercentileScore( tx.blocktime, feeRate, + client ); sumRFS += RFS; } - }); + } return sumRFS / numberOfSendTx; } @@ -101,7 +86,7 @@ export function feesToAmountRatio(transactions: Transaction[]): number { numberOfSendTx++; } }); - return 100 * (sumFeesToAmountRatio / numberOfSendTx); + return sumFeesToAmountRatio / numberOfSendTx; } /* @@ -114,11 +99,12 @@ Assume the wallet is being consolidated, Thus number of UTXO will decrease and t W (Weightage of number of UTXO) will increase and this justifies that, consolidation increases the fees health since you don’t overpay them in long run. */ -export function feesScore( +export async function feesScore( transactions: Transaction[], utxos: AddressUtxos, -): number { - let RFS: number = relativeFeesScore(transactions); + client: BlockchainClient +): Promise { + let RFS: number = await relativeFeesScore(transactions, client); let FAR: number = feesToAmountRatio(transactions); let W: number = utxoSetLengthScore(utxos); return 0.35 * RFS + 0.35 * FAR + 0.3 * W; From a4c6ed7476bcdd8677dc6f1b218efd8e962fd822 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 22 Jul 2024 23:43:30 +0530 Subject: [PATCH 18/92] Improving Documentation Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 6 +- packages/caravan-health/src/feescore.ts | 61 +++++++++++++++----- packages/caravan-health/src/privacy.test.ts | 2 +- packages/caravan-health/src/privacy.ts | 62 +++++++++++++++++++-- packages/caravan-health/src/types.ts | 2 + 5 files changed, 109 insertions(+), 24 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 218c9ef0..eca79ea7 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -4,6 +4,6 @@ - [x] Write logic for `isAddressReused` function in `privacy.ts`. - [x] Cover test cases for `privacy.ts`. -- [] Replace `any` types from `feescore.ts` -- [] Cover test-cases for `feescore.ts` -- [] Add link to blog post +- [x] Replace `any` types from `feescore.ts` +- [x] Cover test-cases for `feescore.ts` +- [ ] Add link to blog post diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index fa04ebc0..0020a7d5 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -2,40 +2,57 @@ import { BlockchainClient } from "@caravan/clients"; import { utxoSetLengthScore } from "./privacy"; import { Transaction, AddressUtxos } from "./types"; -// Utility function that helps to obtain the fee rate of the transaction +/* +Utility function that helps to obtain the fee rate of the transaction + +Expected Range : [0, 0.75] +-> Very Poor : [0, 0.15] +-> Poor : (0.15, 0.3] +-> Moderate : (0.3, 0.45] +-> Good : (0.45, 0.6] +-> Very Good : (0.6, 0.75] +*/ function getFeeRateForTransaction(transaction: Transaction): number { - // TODO : Please check that do we really get the fee rate and weight both from the transaction object - // No we don't get fees let fees: number = transaction.fee; let weight: number = transaction.weight; return fees / weight; } -// Utility function that helps to obtain the percentile of the fees paid by user in tx block +/* +Utility function that helps to obtain the percentile of the fees paid by user in tx block + +Expected Range : [0, 0.75] +-> 10% tile : 1 +-> 25% tile : 0.75 +-> 50% tile : 0.5 +-> 75% tile : 0.25 +-> 90% tile : 0.1 +-> 100% tile : 0.05 +*/ async function getFeeRatePercentileScore( timestamp: number, feeRate: number, - client: BlockchainClient + client: BlockchainClient, ) { let percentile: number = await client.getFeeRatePercentileForTransaction( timestamp, - feeRate + feeRate, ); switch (percentile) { case 10: return 1; case 25: - return 0.9; - case 50: return 0.75; - case 75: + case 50: return 0.5; - case 90: + case 75: return 0.25; - case 100: + case 90: return 0.1; + case 100: + return 0.05; default: - return 0; + throw new Error("Invalid percentile"); } } @@ -45,10 +62,17 @@ if any transaction was done at expensive fees or nominal fees. This can be done by calculating the percentile of the fees paid by the user in the block of the transaction. + +Expected Range : [0, 1] +-> Very Poor : [0, 0.2] +-> Poor : (0.2, 0.4] +-> Moderate : (0.4, 0.6] +-> Good : (0.6, 0.8] +-> Very Good : (0.8, 1] */ export async function relativeFeesScore( transactions: Transaction[], - client: BlockchainClient + client: BlockchainClient, ): Promise { let sumRFS: number = 0; let numberOfSendTx: number = 0; @@ -59,7 +83,7 @@ export async function relativeFeesScore( let RFS: number = await getFeeRatePercentileScore( tx.blocktime, feeRate, - client + client, ); sumRFS += RFS; } @@ -98,11 +122,18 @@ Q : What role does W plays in the fees score? Assume the wallet is being consolidated, Thus number of UTXO will decrease and thus W (Weightage of number of UTXO) will increase and this justifies that, consolidation increases the fees health since you don’t overpay them in long run. + +Expected Range : [0, 1] +-> Very Poor : [0, 0.2] +-> Poor : (0.2, 0.4] +-> Moderate : (0.4, 0.6] +-> Good : (0.6, 0.8] +-> Very Good : (0.8, 1] */ export async function feesScore( transactions: Transaction[], utxos: AddressUtxos, - client: BlockchainClient + client: BlockchainClient, ): Promise { let RFS: number = await relativeFeesScore(transactions, client); let FAR: number = feesToAmountRatio(transactions); diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index c1dc0af1..fe9c7858 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -55,7 +55,7 @@ describe("Privacy Score Functions", () => { const score: number = +( await privacyScoreByTxTopology(transaction, mockClient) ).toFixed(3); - expect(score).toBe(0.92); + expect(score).toBe(0.818); }); }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 1173ce42..1d04efea 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -34,6 +34,9 @@ enum SpendType { MixingOrCoinJoin = "MixingOrCoinJoin", } +/* +The deterministic scores or their formula for each spend type are as follows +*/ function spendTypeScores( spendType: SpendType, numberOfInputs: number, @@ -50,7 +53,7 @@ function spendTypeScores( return 1 / numberOfInputs; case SpendType.MixingOrCoinJoin: let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; - return (0.75 * x) / (1 + x); + return ((2 / 3) * x) / (1 + x); default: throw new Error("Invalid spend type"); } @@ -67,6 +70,17 @@ function determineSpendType(inputs: number, outputs: number): SpendType { } } +/* +The transaction topology refers to the type of transaction based on +number of inputs and outputs. + +Expected Range : [0, 0.75] +-> Very Poor : [0, 0.15] +-> Poor : (0.15, 0.3] +-> Moderate : (0.3, 0.45] +-> Good : (0.45, 0.6] +-> Very Good : (0.6, 0.75] +*/ export async function privacyScoreByTxTopology( transaction: Transaction, client: BlockchainClient, @@ -100,6 +114,13 @@ export async function privacyScoreByTxTopology( /* In order to score for address reuse we can check the amount being hold by reused UTXOs with respect to the total amount + +Expected Range : [0,1] +-> Very Poor : (0.8, 1] +-> Poor : [0.6, 0.8) +-> Moderate : [0.4, 0.6) +-> Good : [0.2, 0.4) +-> Very Good : [0 ,0.2) */ export async function addressReuseFactor( utxos: AddressUtxos, @@ -131,10 +152,18 @@ async function isReusedAddress( } return false; } + /* If we are making payment to other wallet types then the privacy score should decrease because the change received will be to an address type matching our wallet and it will lead to a deduction that we still own that amount. + +Expected Range : (0,1] +-> Very Poor : (0, 0.1] +-> Poor : [0.1, 0.3) +-> Moderate : [0.3, 0.4) +-> Good : [0.4, 0.5) +-> Very Good : [0.5 ,1] */ export function addressTypeFactor( transactions: Transaction[], @@ -173,6 +202,13 @@ export function addressTypeFactor( The spread factor using standard deviation helps in assessing the dispersion of UTXO values. In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries to link transactions and deduce the ownership and spending patterns of users. + +Expected Range : [0,1) +-> Very Poor : (0, 0.2] +-> Poor : [0.2, 0.4) +-> Moderate : [0.4, 0.6) +-> Good : [0.6, 0.8) +-> Very Good : [0.8 ,1] */ export function utxoSpreadFactor(utxos: AddressUtxos): number { const amounts: number[] = []; @@ -194,6 +230,8 @@ export function utxoSpreadFactor(utxos: AddressUtxos): number { /* The weightage is ad-hoc to normalize the privacy score based on the number of UTXOs in the set. + +Expected Range : [0,1] - 0 for UTXO set length >= 50 - 0.25 for UTXO set length >= 25 and <= 49 - 0.5 for UTXO set length >= 15 and <= 24 @@ -224,6 +262,13 @@ export function utxoSetLengthScore(utxos: AddressUtxos): number { /* UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. + +Expected Range : [-0.15,0.15] +-> Very Poor : [-0.1, -0.05) +-> Poor : [-0.05, 0) +-> Moderate : [0, 0.05) +-> Good : [0.05, 0.1) +-> Very Good : [0.1 ,0.15] */ export function utxoValueWeightageFactor(utxos: AddressUtxos): number { let W: number = utxoSetLengthScore(utxos); @@ -233,10 +278,17 @@ export function utxoValueWeightageFactor(utxos: AddressUtxos): number { /* The privacy score is a combination of all the factors calculated above. -- Privacy Score based on Inputs and Outputs -- Address Reuse Factor (R.F) : p_adjusted = p_score * (1 - 0.5 * r.f) + 0.10 * (1 - r.f) -- Address Type Factor (A.T.F) : p_adjusted = p_score * (1-A.T.F) -- UTXO Value Weightage Factor (U.V.W.F) : p_adjusted = p_score + U.V.W.F +- Privacy Score based on Inputs and Outputs (i.e Tx Topology) +- Address Reuse Factor (R.F) +- Address Type Factor (A.T.F) +- UTXO Value Weightage Factor (U.V.W.F) + +Expected Range : [0, 1] +-> Very Poor : [0, 0.2] +-> Poor : (0.2, 0.4] +-> Moderate : (0.4, 0.6] +-> Good : (0.6, 0.8] +-> Very Good : (0.8, 1] */ export async function privacyScore( transactions: Transaction[], diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index b1ee21b4..8405d745 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -1,9 +1,11 @@ import { UTXO } from "@caravan/clients"; +// Represents the Unspent Outputs of the address export interface AddressUtxos { [address: string]: UTXO[]; } +// Expected Transaction object which should be built in order to consume @caravan-health functionalities export interface Transaction { txid: string; vin: Input[]; From f4959d7d2fba2b75fed3d89a149e3ff1d1f7e766 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 23 Jul 2024 02:25:37 +0530 Subject: [PATCH 19/92] Type Improvements Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 65 +++++++++++++++----- packages/caravan-health/README.md | 5 +- packages/caravan-health/src/feescore.test.ts | 4 +- packages/caravan-health/src/feescore.ts | 9 ++- packages/caravan-health/src/privacy.ts | 11 +++- 5 files changed, 72 insertions(+), 22 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 9f0660a1..13bf5b51 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -41,6 +41,19 @@ export enum ClientType { BLOCKSTREAM = "blockstream", MEMPOOL = "mempool", } + +export interface FeeRatePercentile { + avgHeight: number; + timestamp: number; + avgFee_0: number; + avgFee_10: number; + avgFee_25: number; + avgFee_50: number; + avgFee_75: number; + avgFee_90: number; + avgFee_100: number; +} + const delay = () => { return new Promise((resolve) => setTimeout(resolve, 500)); }; @@ -341,37 +354,59 @@ export class BlockchainClient extends ClientBase { try { switch (this.type) { case ClientType.PRIVATE: - // TODO : Implement it for private client + // DOUBT : I don't think bitcoind or even blockstream gives this info. + // Maybe we should compare it only against MEMPOOL with given timestamp and feerate. case ClientType.BLOCKSTREAM: - // TODO : Implement it for blockstream client + // DOUBT : Same as above case ClientType.MEMPOOL: let data = await this.Get(`v1/mining/blocks/fee-rates/all`); + let feeRatePercentileBlocks: FeeRatePercentile[] = []; + for (const block of data) { + let feeRatePercentile : FeeRatePercentile = { + avgHeight: block?.avgHeight, + timestamp: block?.timestamp, + avgFee_0: block?.avgFee_0, + avgFee_10: block?.avgFee_10, + avgFee_25: block?.avgFee_25, + avgFee_50: block?.avgFee_50, + avgFee_75: block?.avgFee_75, + avgFee_90: block?.avgFee_90, + avgFee_100: block?.avgFee_100, + } + feeRatePercentileBlocks.push(feeRatePercentile); + } // Find the closest entry by timestamp - let closestEntry: any; + let closestBlock: FeeRatePercentile | null = null; let closestDifference: number = Infinity; - data.forEach((item) => { - const difference = Math.abs(item.timestamp - timestamp); + for (const block of feeRatePercentileBlocks) { + const difference = Math.abs(block.timestamp - timestamp); if (difference < closestDifference) { closestDifference = difference; - closestEntry = item; + closestBlock = block; } - }); - switch (closestEntry) { - case feeRate < closestEntry.avgFee_10: + } + if (!closestBlock) { + throw new Error("No fee rate data found"); + } + // Find the closest fee rate percentile + switch(true) { + case feeRate < closestBlock.avgFee_0: + return 0; + case feeRate < closestBlock.avgFee_10: return 10; - case feeRate < closestEntry.avgFee_25: + case feeRate < closestBlock.avgFee_25: return 25; - case feeRate < closestEntry.avgFee_50: + case feeRate < closestBlock.avgFee_50: return 50; - case feeRate < closestEntry.avgFee_75: + case feeRate < closestBlock.avgFee_75: return 75; - case feeRate < closestEntry.avgFee_90: + case feeRate < closestBlock.avgFee_90: return 90; - case feeRate < closestEntry.avgFee_100: + case feeRate < closestBlock.avgFee_100: return 100; default: - return 0; + throw new Error("Invalid fee rate"); } default: throw new Error("Invalid client type"); diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index eca79ea7..68586b2b 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -6,4 +6,7 @@ - [x] Cover test cases for `privacy.ts`. - [x] Replace `any` types from `feescore.ts` - [x] Cover test-cases for `feescore.ts` -- [ ] Add link to blog post +- [ ] Implement `Transaction` type on the `getAddressTransaction` func. +- [ ] Take a decision on doubt mentioned in `getFeeRatePercentileForTransaction` +- [ ] Implement Caching or Ticker based mechanism to reduce network latency for percentile of feerate for a given block +- [ ] Write Blog on health and Add link to blog post diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/feescore.test.ts index c50edaee..29067231 100644 --- a/packages/caravan-health/src/feescore.test.ts +++ b/packages/caravan-health/src/feescore.test.ts @@ -33,7 +33,7 @@ describe("Fees Score Functions", () => { const score: number = +( await relativeFeesScore(transactions, mockClient) ).toFixed(3); - expect(score).toBe(1); + expect(score).toBe(0.9); }); }); @@ -91,7 +91,7 @@ describe("Fees Score Functions", () => { const score: number = +( await feesScore(transaction, utxos, mockClient) ).toFixed(3); - expect(score).toBe(0.685); + expect(score).toBe(0.65); }); }); }); diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 0020a7d5..fa0256a8 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -22,7 +22,8 @@ function getFeeRateForTransaction(transaction: Transaction): number { Utility function that helps to obtain the percentile of the fees paid by user in tx block Expected Range : [0, 0.75] --> 10% tile : 1 +-> 0% tile : 1 +-> 10% tile : 0.9 -> 25% tile : 0.75 -> 50% tile : 0.5 -> 75% tile : 0.25 @@ -39,8 +40,10 @@ async function getFeeRatePercentileScore( feeRate, ); switch (percentile) { - case 10: + case 0: return 1; + case 10: + return 0.9; case 25: return 0.75; case 50: @@ -50,7 +53,7 @@ async function getFeeRatePercentileScore( case 90: return 0.1; case 100: - return 0.05; + return 0; default: throw new Error("Invalid percentile"); } diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 1d04efea..56ff61e0 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -147,7 +147,16 @@ async function isReusedAddress( client: BlockchainClient, ): Promise { let txs: Transaction[] = await client.getAddressTransactions(address); - if (txs.length > 1) { + let countSend = 0; + let countReceive = 0; + for (const tx of txs) { + if(tx.isSend===true){ + countSend++; + }else{ + countReceive++; + } + } + if (countSend > 1 || countReceive > 1) { return true; } return false; From dd77fa3369a98380374643062572479c552f9d63 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 26 Jul 2024 02:50:25 +0530 Subject: [PATCH 20/92] Added Transaction[] as return type in clients fxn Signed-off-by: Harshil-Jani --- package-lock.json | 17 +++- packages/caravan-clients/package.json | 1 + packages/caravan-clients/src/client.ts | 98 +++++++++++++++++++-- packages/caravan-health/src/privacy.test.ts | 2 - packages/caravan-health/src/types.ts | 1 - 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 511ffb15..cc1c98d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2722,7 +2722,7 @@ "link": true }, "node_modules/@caravan/health": { - "resolved": "packages/health", + "resolved": "packages/caravan-health", "link": true }, "node_modules/@caravan/multisig": { @@ -25151,6 +25151,7 @@ "@babel/core": "^7.23.7", "@babel/preset-env": "^7.23.8", "@caravan/eslint-config": "*", + "@caravan/health": "*", "@caravan/typescript-config": "*", "babel-jest": "^29.7.0", "eslint": "^8.56.0", @@ -26011,6 +26012,19 @@ "webidl-conversions": "^4.0.2" } }, + "packages/caravan-health": { + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@caravan/clients": "*" + }, + "devDependencies": { + "prettier": "^3.2.5" + }, + "engines": { + "node": ">=20" + } + }, "packages/caravan-psbt": { "name": "@caravan/psbt", "version": "1.4.1", @@ -28463,6 +28477,7 @@ }, "packages/health": { "version": "1.0.0", + "extraneous": true, "license": "MIT", "engines": { "node": ">=20" diff --git a/packages/caravan-clients/package.json b/packages/caravan-clients/package.json index e39000ba..0d946e37 100644 --- a/packages/caravan-clients/package.json +++ b/packages/caravan-clients/package.json @@ -40,6 +40,7 @@ "@babel/preset-env": "^7.23.8", "@caravan/eslint-config": "*", "@caravan/typescript-config": "*", + "@caravan/health": "*", "babel-jest": "^29.7.0", "eslint": "^8.56.0", "jest": "^29.7.0", diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 13bf5b51..09b75b2d 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -18,6 +18,7 @@ import { bitcoindWalletInfo, } from "./wallet"; import BigNumber from "bignumber.js"; +import { Transaction } from "@caravan/health"; export class BlockchainClientError extends Error { constructor(message) { @@ -185,17 +186,98 @@ export class BlockchainClient extends ClientBase { } } - public async getAddressTransactions(address: string): Promise { + public async bitcoindRawTxData(txid: string): Promise { + return await callBitcoind( + this.bitcoindParams.url, + this.bitcoindParams.auth, + "decoderawtransaction", + [txid], + ); + } + + public async getAddressTransactions(address: string): Promise { try { if (this.type === ClientType.PRIVATE) { - return await callBitcoind( + const data = await callBitcoind( this.bitcoindParams.url, this.bitcoindParams.auth, "listtransactions", - [this.bitcoindParams.walletName, 1000], + [this.bitcoindParams.walletName], ); + + const txs: Transaction[] = []; + for (const tx of data) { + if (tx.address === address) { + let isTxSend = tx.category === "send" ? true : false; + const rawTxData = await this.bitcoindRawTxData(tx.txid); + const transaction: Transaction = { + txid: tx.txid, + vin: [], + vout: [], + size: rawTxData.size, + weight: rawTxData.weight, + fee: tx.fee, + isSend: isTxSend, + amount: tx.amount, + blocktime: tx.blocktime, + }; + for (const input of rawTxData.vin) { + transaction.vin.push({ + txid: input.txid, + vout: input.vout, + sequence: input.sequence, + }); + } + for (const output of rawTxData.vout) { + transaction.vout.push({ + scriptPubkeyHex: output.scriptPubKey.hex, + scriptPubkeyAddress: output.scriptPubKey.addresses[0], + value: output.value, + }); + } + txs.push(transaction); + } + } + return txs; } - return await this.Get(`/address/${address}/txs`); + + // For Mempool and Blockstream + const data = await this.Get(`/address/${address}/txs`); + const txs: Transaction[] = []; + for (const tx of data.txs) { + const transaction: Transaction = { + txid: tx.txid, + vin: [], + vout: [], + size: tx.size, + weight: tx.weight, + fee: tx.fee, + isSend: false, // TODO : Need to implement this. + amount: 0, + blocktime: tx.status.block_time, + }; + + for (const input of tx.vin) { + transaction.vin.push({ + txid: input.txid, + vout: input.vout, + sequence: input.sequence, + }); + } + + let total_amount = 0; + for (const output of tx.vout) { + total_amount += output.value; + transaction.vout.push({ + scriptPubkeyHex: output.scriptpubkey, + scriptPubkeyAddress: output.scriptpubkey_address, + value: output.value, + }); + } + transaction.amount = total_amount; + txs.push(transaction); + } + return txs; } catch (error: any) { throw new Error( `Failed to get transactions for address ${address}: ${error.message}`, @@ -354,7 +436,7 @@ export class BlockchainClient extends ClientBase { try { switch (this.type) { case ClientType.PRIVATE: - // DOUBT : I don't think bitcoind or even blockstream gives this info. + // DOUBT : I don't think bitcoind or even blockstream gives this info. // Maybe we should compare it only against MEMPOOL with given timestamp and feerate. case ClientType.BLOCKSTREAM: // DOUBT : Same as above @@ -362,7 +444,7 @@ export class BlockchainClient extends ClientBase { let data = await this.Get(`v1/mining/blocks/fee-rates/all`); let feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { - let feeRatePercentile : FeeRatePercentile = { + let feeRatePercentile: FeeRatePercentile = { avgHeight: block?.avgHeight, timestamp: block?.timestamp, avgFee_0: block?.avgFee_0, @@ -372,7 +454,7 @@ export class BlockchainClient extends ClientBase { avgFee_75: block?.avgFee_75, avgFee_90: block?.avgFee_90, avgFee_100: block?.avgFee_100, - } + }; feeRatePercentileBlocks.push(feeRatePercentile); } // Find the closest entry by timestamp @@ -390,7 +472,7 @@ export class BlockchainClient extends ClientBase { throw new Error("No fee rate data found"); } // Find the closest fee rate percentile - switch(true) { + switch (true) { case feeRate < closestBlock.avgFee_0: return 0; case feeRate < closestBlock.avgFee_10: diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index fe9c7858..ea2438aa 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -29,13 +29,11 @@ describe("Privacy Score Functions", () => { { txid: "input1", vout: 0, - witness: [], sequence: 0, }, { txid: "input2", vout: 0, - witness: [], sequence: 0, }, ], // 2 inputs diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index 8405d745..bee5f24c 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -21,7 +21,6 @@ export interface Transaction { interface Input { txid: string; vout: number; - witness: string[]; sequence: number; } From ffd741c454aaa521c2f151ad877fe1d814619c0f Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sun, 28 Jul 2024 16:29:39 +0530 Subject: [PATCH 21/92] Add dependency on @caravan-bitcoin Signed-off-by: Harshil-Jani --- packages/caravan-health/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/caravan-health/package.json b/packages/caravan-health/package.json index d2a7552b..472525b0 100644 --- a/packages/caravan-health/package.json +++ b/packages/caravan-health/package.json @@ -34,7 +34,8 @@ "test:debug": "node --inspect-brk ../../node_modules/.bin/jest --runInBand" }, "dependencies": { - "@caravan/clients": "*" + "@caravan/clients": "*", + "@caravan/bitcoin": "*" }, "devDependencies": { "prettier": "^3.2.5" From 49b6886c6645f1d0f514dab4bc02afae4199bcee Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sun, 28 Jul 2024 16:30:05 +0530 Subject: [PATCH 22/92] Update logic for address reuse Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 56ff61e0..3ed625a4 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -147,16 +147,13 @@ async function isReusedAddress( client: BlockchainClient, ): Promise { let txs: Transaction[] = await client.getAddressTransactions(address); - let countSend = 0; let countReceive = 0; for (const tx of txs) { - if(tx.isSend===true){ - countSend++; - }else{ + if(tx.isSend===false){ countReceive++; } } - if (countSend > 1 || countReceive > 1) { + if (countReceive > 1) { return true; } return false; From c39eb85b946ac23e54db7be20a2882489781eb96 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sun, 28 Jul 2024 17:18:56 +0530 Subject: [PATCH 23/92] Moving Transaction type into clients Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 25 +++++++++++++++++++- packages/caravan-clients/src/index.ts | 2 +- packages/caravan-health/src/feescore.test.ts | 4 +--- packages/caravan-health/src/feescore.ts | 14 +++++------ packages/caravan-health/src/privacy.test.ts | 3 +-- packages/caravan-health/src/privacy.ts | 4 ++-- packages/caravan-health/src/types.ts | 25 -------------------- 7 files changed, 36 insertions(+), 41 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 09b75b2d..fa0187e8 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -18,7 +18,6 @@ import { bitcoindWalletInfo, } from "./wallet"; import BigNumber from "bignumber.js"; -import { Transaction } from "@caravan/health"; export class BlockchainClientError extends Error { constructor(message) { @@ -37,6 +36,30 @@ export interface UTXO { }; } +export interface Transaction { + txid: string; + vin: Input[]; + vout: Output[]; + size: number; + weight: number; + fee: number; + isSend: boolean; + amount: number; + blocktime: number; +} + +interface Input { + txid: string; + vout: number; + sequence: number; +} + +interface Output { + scriptPubkeyHex: string; + scriptPubkeyAddress: string; + value: number; +} + export enum ClientType { PRIVATE = "private", BLOCKSTREAM = "blockstream", diff --git a/packages/caravan-clients/src/index.ts b/packages/caravan-clients/src/index.ts index 3774161a..2d74f8c7 100644 --- a/packages/caravan-clients/src/index.ts +++ b/packages/caravan-clients/src/index.ts @@ -1,3 +1,3 @@ export { bitcoindImportDescriptors } from "./wallet"; export { BlockchainClient, ClientType } from "./client"; -export type { UTXO } from "./client"; +export type { UTXO, Transaction } from "./client"; diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/feescore.test.ts index 29067231..240e260f 100644 --- a/packages/caravan-health/src/feescore.test.ts +++ b/packages/caravan-health/src/feescore.test.ts @@ -1,8 +1,6 @@ import { feesScore, feesToAmountRatio, relativeFeesScore } from "./feescore"; import { BlockchainClient } from "@caravan/clients"; -import { Transaction } from "./types"; -import { AddressUtxos } from "./types"; -import { MultisigAddressType, Network } from "@caravan/bitcoin"; +import { Transaction } from "@caravan/clients"; describe("Fees Score Functions", () => { let mockClient: BlockchainClient; diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index fa0256a8..c3d0fb8b 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,6 +1,6 @@ -import { BlockchainClient } from "@caravan/clients"; +import { BlockchainClient, Transaction } from "@caravan/clients"; import { utxoSetLengthScore } from "./privacy"; -import { Transaction, AddressUtxos } from "./types"; +import { AddressUtxos } from "./types"; /* Utility function that helps to obtain the fee rate of the transaction @@ -33,11 +33,11 @@ Expected Range : [0, 0.75] async function getFeeRatePercentileScore( timestamp: number, feeRate: number, - client: BlockchainClient, + client: BlockchainClient ) { let percentile: number = await client.getFeeRatePercentileForTransaction( timestamp, - feeRate, + feeRate ); switch (percentile) { case 0: @@ -75,7 +75,7 @@ Expected Range : [0, 1] */ export async function relativeFeesScore( transactions: Transaction[], - client: BlockchainClient, + client: BlockchainClient ): Promise { let sumRFS: number = 0; let numberOfSendTx: number = 0; @@ -86,7 +86,7 @@ export async function relativeFeesScore( let RFS: number = await getFeeRatePercentileScore( tx.blocktime, feeRate, - client, + client ); sumRFS += RFS; } @@ -136,7 +136,7 @@ Expected Range : [0, 1] export async function feesScore( transactions: Transaction[], utxos: AddressUtxos, - client: BlockchainClient, + client: BlockchainClient ): Promise { let RFS: number = await relativeFeesScore(transactions, client); let FAR: number = feesToAmountRatio(transactions); diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index ea2438aa..123b2671 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -7,8 +7,7 @@ import { utxoValueWeightageFactor, privacyScore, } from "./privacy"; -import { BlockchainClient } from "@caravan/clients"; -import { Transaction } from "./types"; +import { BlockchainClient, Transaction } from "@caravan/clients"; import { AddressUtxos } from "./types"; import { MultisigAddressType, Network } from "@caravan/bitcoin"; diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 3ed625a4..ee168007 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,5 +1,5 @@ -import { BlockchainClient } from "@caravan/clients"; -import { Transaction, AddressUtxos } from "./types"; +import { BlockchainClient, Transaction } from "@caravan/clients"; +import { AddressUtxos } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; /* diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index bee5f24c..2d855ecd 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -4,28 +4,3 @@ import { UTXO } from "@caravan/clients"; export interface AddressUtxos { [address: string]: UTXO[]; } - -// Expected Transaction object which should be built in order to consume @caravan-health functionalities -export interface Transaction { - txid: string; - vin: Input[]; - vout: Output[]; - size: number; - weight: number; - fee: number; - isSend: boolean; - amount: number; - blocktime: number; -} - -interface Input { - txid: string; - vout: number; - sequence: number; -} - -interface Output { - scriptPubkeyHex: string; - scriptPubkeyAddress: string; - value: number; -} From f5e6567672c2a85bd8454ae80487fc13d071cb94 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sun, 28 Jul 2024 18:41:58 +0530 Subject: [PATCH 24/92] Performance improvement for feeRate percentile history Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 52 ++---------------- packages/caravan-clients/src/index.ts | 2 +- packages/caravan-health/src/feescore.ts | 70 ++++++++++++++++++++----- 3 files changed, 62 insertions(+), 62 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index fa0187e8..b0e865f5 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -451,19 +451,8 @@ export class BlockchainClient extends ClientBase { } } - // TODO : Implement Caching or Ticker based mechanism to reduce network latency - public async getFeeRatePercentileForTransaction( - timestamp: number, - feeRate: number, - ): Promise { + public async getBlockFeeRatePercentileHistory(): Promise { try { - switch (this.type) { - case ClientType.PRIVATE: - // DOUBT : I don't think bitcoind or even blockstream gives this info. - // Maybe we should compare it only against MEMPOOL with given timestamp and feerate. - case ClientType.BLOCKSTREAM: - // DOUBT : Same as above - case ClientType.MEMPOOL: let data = await this.Get(`v1/mining/blocks/fee-rates/all`); let feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { @@ -480,44 +469,9 @@ export class BlockchainClient extends ClientBase { }; feeRatePercentileBlocks.push(feeRatePercentile); } - // Find the closest entry by timestamp - let closestBlock: FeeRatePercentile | null = null; - let closestDifference: number = Infinity; - - for (const block of feeRatePercentileBlocks) { - const difference = Math.abs(block.timestamp - timestamp); - if (difference < closestDifference) { - closestDifference = difference; - closestBlock = block; - } - } - if (!closestBlock) { - throw new Error("No fee rate data found"); - } - // Find the closest fee rate percentile - switch (true) { - case feeRate < closestBlock.avgFee_0: - return 0; - case feeRate < closestBlock.avgFee_10: - return 10; - case feeRate < closestBlock.avgFee_25: - return 25; - case feeRate < closestBlock.avgFee_50: - return 50; - case feeRate < closestBlock.avgFee_75: - return 75; - case feeRate < closestBlock.avgFee_90: - return 90; - case feeRate < closestBlock.avgFee_100: - return 100; - default: - throw new Error("Invalid fee rate"); - } - default: - throw new Error("Invalid client type"); - } + return feeRatePercentileBlocks; } catch (error: any) { - throw new Error(`Failed to get fee estimate: ${error.message}`); + throw new Error(`Failed to get feerate percentile block: ${error.message}`); } } diff --git a/packages/caravan-clients/src/index.ts b/packages/caravan-clients/src/index.ts index 2d74f8c7..4dc628fb 100644 --- a/packages/caravan-clients/src/index.ts +++ b/packages/caravan-clients/src/index.ts @@ -1,3 +1,3 @@ export { bitcoindImportDescriptors } from "./wallet"; export { BlockchainClient, ClientType } from "./client"; -export type { UTXO, Transaction } from "./client"; +export type { UTXO, Transaction, FeeRatePercentile } from "./client"; diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index c3d0fb8b..5ea933f2 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,4 +1,8 @@ -import { BlockchainClient, Transaction } from "@caravan/clients"; +import { + BlockchainClient, + FeeRatePercentile, + Transaction, +} from "@caravan/clients"; import { utxoSetLengthScore } from "./privacy"; import { AddressUtxos } from "./types"; @@ -30,14 +34,15 @@ Expected Range : [0, 0.75] -> 90% tile : 0.1 -> 100% tile : 0.05 */ -async function getFeeRatePercentileScore( +function getFeeRatePercentileScore( timestamp: number, feeRate: number, - client: BlockchainClient -) { - let percentile: number = await client.getFeeRatePercentileForTransaction( + feeRatePercentileHistory: FeeRatePercentile[] +): number { + let percentile: number = getPercentile( timestamp, - feeRate + feeRate, + feeRatePercentileHistory ); switch (percentile) { case 0: @@ -59,6 +64,46 @@ async function getFeeRatePercentileScore( } } +function getPercentile( + timestamp: number, + feeRate: number, + feeRatePercentileHistory: FeeRatePercentile[] +): number { + // Find the closest entry by timestamp + let closestBlock: FeeRatePercentile | null = null; + let closestDifference: number = Infinity; + + for (const block of feeRatePercentileHistory) { + const difference = Math.abs(block.timestamp - timestamp); + if (difference < closestDifference) { + closestDifference = difference; + closestBlock = block; + } + } + if (!closestBlock) { + throw new Error("No fee rate data found"); + } + // Find the closest fee rate percentile + switch (true) { + case feeRate < closestBlock.avgFee_0: + return 0; + case feeRate < closestBlock.avgFee_10: + return 10; + case feeRate < closestBlock.avgFee_25: + return 25; + case feeRate < closestBlock.avgFee_50: + return 50; + case feeRate < closestBlock.avgFee_75: + return 75; + case feeRate < closestBlock.avgFee_90: + return 90; + case feeRate < closestBlock.avgFee_100: + return 100; + default: + throw new Error("Invalid fee rate"); + } +} + /* R.F.S can be associated with all the transactions and we can give a measure if any transaction was done at expensive fees or nominal fees. @@ -73,20 +118,20 @@ Expected Range : [0, 1] -> Good : (0.6, 0.8] -> Very Good : (0.8, 1] */ -export async function relativeFeesScore( +export function relativeFeesScore( transactions: Transaction[], - client: BlockchainClient -): Promise { + feeRatePercentileHistory: FeeRatePercentile[] +): number { let sumRFS: number = 0; let numberOfSendTx: number = 0; for (const tx of transactions) { if (tx.isSend === true) { numberOfSendTx++; let feeRate: number = getFeeRateForTransaction(tx); - let RFS: number = await getFeeRatePercentileScore( + let RFS: number = getFeeRatePercentileScore( tx.blocktime, feeRate, - client + feeRatePercentileHistory ); sumRFS += RFS; } @@ -138,7 +183,8 @@ export async function feesScore( utxos: AddressUtxos, client: BlockchainClient ): Promise { - let RFS: number = await relativeFeesScore(transactions, client); + let feeRatePercentileHistory: FeeRatePercentile[] = await client.getBlockFeeRatePercentileHistory(); + let RFS: number = relativeFeesScore(transactions, feeRatePercentileHistory); let FAR: number = feesToAmountRatio(transactions); let W: number = utxoSetLengthScore(utxos); return 0.35 * RFS + 0.35 * FAR + 0.3 * W; From f71c17a9251ed12a5e3975c1fdd2af0daaee4bc7 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 21:08:33 +0530 Subject: [PATCH 25/92] Update fees test cases Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.test.ts | 69 +++++++------------- packages/caravan-health/src/feescore.ts | 16 ++--- 2 files changed, 30 insertions(+), 55 deletions(-) diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/feescore.test.ts index 240e260f..102d0001 100644 --- a/packages/caravan-health/src/feescore.test.ts +++ b/packages/caravan-health/src/feescore.test.ts @@ -1,6 +1,6 @@ import { feesScore, feesToAmountRatio, relativeFeesScore } from "./feescore"; import { BlockchainClient } from "@caravan/clients"; -import { Transaction } from "@caravan/clients"; +import { Transaction, FeeRatePercentile } from "@caravan/clients"; describe("Fees Score Functions", () => { let mockClient: BlockchainClient; @@ -9,29 +9,42 @@ describe("Fees Score Functions", () => { mockClient = { getAddressStatus: jest.fn(), getAddressTransactions: jest.fn().mockResolvedValue([{ txid: "tx1" }]), - getFeeRatePercentileForTransaction: jest.fn().mockResolvedValue(10), } as unknown as BlockchainClient; }); describe("relativeFeesScore", () => { - it("Relative fees score for transaction", async () => { + it("Relative fees score for transaction", () => { const transactions: Transaction[] = [ { vin: [], vout: [], txid: "tx1", size: 0, - weight: 0, - fee: 0, + weight: 1, + fee: 1, isSend: true, amount: 0, - blocktime: 0, + blocktime: 1234, + }, + ]; + const feeRatePercentileHistory: FeeRatePercentile[] = [ + { + avgHeight: 0, + timestamp: 1234, + avgFee_0: 0, + avgFee_10: 0, + avgFee_25: 0.5, + avgFee_50: 1, + avgFee_75: 0, + avgFee_90: 0, + avgFee_100: 0, }, ]; - const score: number = +( - await relativeFeesScore(transactions, mockClient) + const score: number = +relativeFeesScore( + transactions, + feeRatePercentileHistory ).toFixed(3); - expect(score).toBe(0.9); + expect(score).toBe(0.5); }); }); @@ -54,42 +67,4 @@ describe("Fees Score Functions", () => { expect(ratio).toBe(0.1); }); }); - - describe("feesScore", () => { - it("Fees score for transaction", async () => { - const transaction: Transaction[] = [ - { - vin: [], - vout: [], - txid: "tx1", - size: 0, - weight: 0, - fee: 1, - isSend: true, - amount: 10, - blocktime: 0, - }, - ]; - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - const score: number = +( - await feesScore(transaction, utxos, mockClient) - ).toFixed(3); - expect(score).toBe(0.65); - }); - }); }); diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 5ea933f2..d0e6c51a 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -75,7 +75,7 @@ function getPercentile( for (const block of feeRatePercentileHistory) { const difference = Math.abs(block.timestamp - timestamp); - if (difference < closestDifference) { + if (difference <= closestDifference) { closestDifference = difference; closestBlock = block; } @@ -85,19 +85,19 @@ function getPercentile( } // Find the closest fee rate percentile switch (true) { - case feeRate < closestBlock.avgFee_0: + case feeRate <= closestBlock.avgFee_0: return 0; - case feeRate < closestBlock.avgFee_10: + case feeRate <= closestBlock.avgFee_10: return 10; - case feeRate < closestBlock.avgFee_25: + case feeRate <= closestBlock.avgFee_25: return 25; - case feeRate < closestBlock.avgFee_50: + case feeRate <= closestBlock.avgFee_50: return 50; - case feeRate < closestBlock.avgFee_75: + case feeRate <= closestBlock.avgFee_75: return 75; - case feeRate < closestBlock.avgFee_90: + case feeRate <= closestBlock.avgFee_90: return 90; - case feeRate < closestBlock.avgFee_100: + case feeRate <= closestBlock.avgFee_100: return 100; default: throw new Error("Invalid fee rate"); From cdb7bfd49b3c100bca53d583ae8258d07f0734f8 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 21:30:30 +0530 Subject: [PATCH 26/92] Decouple feeScore from the client Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index d0e6c51a..a81ccb1d 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,5 +1,4 @@ import { - BlockchainClient, FeeRatePercentile, Transaction, } from "@caravan/clients"; @@ -181,9 +180,8 @@ Expected Range : [0, 1] export async function feesScore( transactions: Transaction[], utxos: AddressUtxos, - client: BlockchainClient + feeRatePercentileHistory: FeeRatePercentile[] ): Promise { - let feeRatePercentileHistory: FeeRatePercentile[] = await client.getBlockFeeRatePercentileHistory(); let RFS: number = relativeFeesScore(transactions, feeRatePercentileHistory); let FAR: number = feesToAmountRatio(transactions); let W: number = utxoSetLengthScore(utxos); From e647debfc40bba73904928d400cb9041086ac0e1 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 21:39:50 +0530 Subject: [PATCH 27/92] Move tx topology as a separate function Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 8 ++--- packages/caravan-health/src/privacy.ts | 39 ++++++++++++--------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 123b2671..d97188e7 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,5 +1,5 @@ import { - privacyScoreByTxTopology, + scoreForTxTopology, addressReuseFactor, addressTypeFactor, utxoSpreadFactor, @@ -50,7 +50,7 @@ describe("Privacy Score Functions", () => { blocktime: 0, }; const score: number = +( - await privacyScoreByTxTopology(transaction, mockClient) + await scoreForTxTopology(transaction, mockClient) ).toFixed(3); expect(score).toBe(0.818); }); @@ -118,7 +118,7 @@ describe("Privacy Score Functions", () => { const factor: number = +addressTypeFactor( transactions, walletAddressType, - network, + network ).toFixed(3); expect(factor).toBe(1); }); @@ -249,7 +249,7 @@ describe("Privacy Score Functions", () => { utxos, walletAddressType, mockClient, - network, + network ) ).toFixed(3); expect(score).toBe(0.005); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index ee168007..2791430c 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -40,7 +40,7 @@ The deterministic scores or their formula for each spend type are as follows function spendTypeScores( spendType: SpendType, numberOfInputs: number, - numberOfOutputs: number, + numberOfOutputs: number ): number { switch (spendType) { case SpendType.SweepSpend: @@ -82,20 +82,32 @@ Expected Range : [0, 0.75] -> Very Good : (0.6, 0.75] */ export async function privacyScoreByTxTopology( + transactions: Transaction[], + client: BlockchainClient +): Promise { + let privacyScore = 0; + for (let tx of transactions) { + let topologyScore = await scoreForTxTopology(tx, client); + privacyScore += topologyScore; + } + return privacyScore / transactions.length; +} + +export async function scoreForTxTopology( transaction: Transaction, - client: BlockchainClient, + client: BlockchainClient ): Promise { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; const spendType: SpendType = determineSpendType( numberOfInputs, - numberOfOutputs, + numberOfOutputs ); const score: number = spendTypeScores( spendType, numberOfInputs, - numberOfOutputs, + numberOfOutputs ); if (spendType === SpendType.Consolidation) { @@ -124,7 +136,7 @@ Expected Range : [0,1] */ export async function addressReuseFactor( utxos: AddressUtxos, - client: BlockchainClient, + client: BlockchainClient ): Promise { let reusedAmount: number = 0; let totalAmount: number = 0; @@ -144,12 +156,12 @@ export async function addressReuseFactor( async function isReusedAddress( address: string, - client: BlockchainClient, + client: BlockchainClient ): Promise { let txs: Transaction[] = await client.getAddressTransactions(address); let countReceive = 0; for (const tx of txs) { - if(tx.isSend===false){ + if (tx.isSend === false) { countReceive++; } } @@ -174,7 +186,7 @@ Expected Range : (0,1] export function addressTypeFactor( transactions: Transaction[], walletAddressType: MultisigAddressType, - network: Network, + network: Network ): number { const addressCounts: Record = { P2WSH: 0, @@ -194,7 +206,7 @@ export function addressTypeFactor( const totalAddresses = Object.values(addressCounts).reduce( (a, b) => a + b, - 0, + 0 ); const walletTypeCount = addressCounts[walletAddressType]; @@ -301,14 +313,9 @@ export async function privacyScore( utxos: AddressUtxos, walletAddressType: MultisigAddressType, client: BlockchainClient, - network: Network, + network: Network ): Promise { - let privacyScore = 0; - for (let tx of transactions) { - let topologyScore = await privacyScoreByTxTopology(tx, client); - privacyScore += topologyScore; - } - privacyScore = privacyScore / transactions.length; + let privacyScore = await privacyScoreByTxTopology(transactions, client); // Adjusting the privacy score based on the address reuse factor let addressReusedFactor = await addressReuseFactor(utxos, client); From 87c7425772d5cfb318d2345e458294c0c6671bdc Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 21:42:19 +0530 Subject: [PATCH 28/92] Prettier Formatting Signed-off-by: Harshil-Jani --- packages/caravan-health/.prettierrc | 4 ++++ packages/caravan-health/src/feescore.test.ts | 2 +- packages/caravan-health/src/feescore.ts | 17 +++++++---------- packages/caravan-health/src/privacy.test.ts | 4 ++-- packages/caravan-health/src/privacy.ts | 20 ++++++++++---------- 5 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 packages/caravan-health/.prettierrc diff --git a/packages/caravan-health/.prettierrc b/packages/caravan-health/.prettierrc new file mode 100644 index 00000000..222861c3 --- /dev/null +++ b/packages/caravan-health/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 2, + "useTabs": false +} diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/feescore.test.ts index 102d0001..3ede8a61 100644 --- a/packages/caravan-health/src/feescore.test.ts +++ b/packages/caravan-health/src/feescore.test.ts @@ -42,7 +42,7 @@ describe("Fees Score Functions", () => { ]; const score: number = +relativeFeesScore( transactions, - feeRatePercentileHistory + feeRatePercentileHistory, ).toFixed(3); expect(score).toBe(0.5); }); diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index a81ccb1d..61faf352 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -1,7 +1,4 @@ -import { - FeeRatePercentile, - Transaction, -} from "@caravan/clients"; +import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { utxoSetLengthScore } from "./privacy"; import { AddressUtxos } from "./types"; @@ -36,12 +33,12 @@ Expected Range : [0, 0.75] function getFeeRatePercentileScore( timestamp: number, feeRate: number, - feeRatePercentileHistory: FeeRatePercentile[] + feeRatePercentileHistory: FeeRatePercentile[], ): number { let percentile: number = getPercentile( timestamp, feeRate, - feeRatePercentileHistory + feeRatePercentileHistory, ); switch (percentile) { case 0: @@ -66,7 +63,7 @@ function getFeeRatePercentileScore( function getPercentile( timestamp: number, feeRate: number, - feeRatePercentileHistory: FeeRatePercentile[] + feeRatePercentileHistory: FeeRatePercentile[], ): number { // Find the closest entry by timestamp let closestBlock: FeeRatePercentile | null = null; @@ -119,7 +116,7 @@ Expected Range : [0, 1] */ export function relativeFeesScore( transactions: Transaction[], - feeRatePercentileHistory: FeeRatePercentile[] + feeRatePercentileHistory: FeeRatePercentile[], ): number { let sumRFS: number = 0; let numberOfSendTx: number = 0; @@ -130,7 +127,7 @@ export function relativeFeesScore( let RFS: number = getFeeRatePercentileScore( tx.blocktime, feeRate, - feeRatePercentileHistory + feeRatePercentileHistory, ); sumRFS += RFS; } @@ -180,7 +177,7 @@ Expected Range : [0, 1] export async function feesScore( transactions: Transaction[], utxos: AddressUtxos, - feeRatePercentileHistory: FeeRatePercentile[] + feeRatePercentileHistory: FeeRatePercentile[], ): Promise { let RFS: number = relativeFeesScore(transactions, feeRatePercentileHistory); let FAR: number = feesToAmountRatio(transactions); diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index d97188e7..ac6c24b2 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -118,7 +118,7 @@ describe("Privacy Score Functions", () => { const factor: number = +addressTypeFactor( transactions, walletAddressType, - network + network, ).toFixed(3); expect(factor).toBe(1); }); @@ -249,7 +249,7 @@ describe("Privacy Score Functions", () => { utxos, walletAddressType, mockClient, - network + network, ) ).toFixed(3); expect(score).toBe(0.005); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 2791430c..03ad8068 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -40,7 +40,7 @@ The deterministic scores or their formula for each spend type are as follows function spendTypeScores( spendType: SpendType, numberOfInputs: number, - numberOfOutputs: number + numberOfOutputs: number, ): number { switch (spendType) { case SpendType.SweepSpend: @@ -83,7 +83,7 @@ Expected Range : [0, 0.75] */ export async function privacyScoreByTxTopology( transactions: Transaction[], - client: BlockchainClient + client: BlockchainClient, ): Promise { let privacyScore = 0; for (let tx of transactions) { @@ -95,19 +95,19 @@ export async function privacyScoreByTxTopology( export async function scoreForTxTopology( transaction: Transaction, - client: BlockchainClient + client: BlockchainClient, ): Promise { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; const spendType: SpendType = determineSpendType( numberOfInputs, - numberOfOutputs + numberOfOutputs, ); const score: number = spendTypeScores( spendType, numberOfInputs, - numberOfOutputs + numberOfOutputs, ); if (spendType === SpendType.Consolidation) { @@ -136,7 +136,7 @@ Expected Range : [0,1] */ export async function addressReuseFactor( utxos: AddressUtxos, - client: BlockchainClient + client: BlockchainClient, ): Promise { let reusedAmount: number = 0; let totalAmount: number = 0; @@ -156,7 +156,7 @@ export async function addressReuseFactor( async function isReusedAddress( address: string, - client: BlockchainClient + client: BlockchainClient, ): Promise { let txs: Transaction[] = await client.getAddressTransactions(address); let countReceive = 0; @@ -186,7 +186,7 @@ Expected Range : (0,1] export function addressTypeFactor( transactions: Transaction[], walletAddressType: MultisigAddressType, - network: Network + network: Network, ): number { const addressCounts: Record = { P2WSH: 0, @@ -206,7 +206,7 @@ export function addressTypeFactor( const totalAddresses = Object.values(addressCounts).reduce( (a, b) => a + b, - 0 + 0, ); const walletTypeCount = addressCounts[walletAddressType]; @@ -313,7 +313,7 @@ export async function privacyScore( utxos: AddressUtxos, walletAddressType: MultisigAddressType, client: BlockchainClient, - network: Network + network: Network, ): Promise { let privacyScore = await privacyScoreByTxTopology(transactions, client); From a493057ad91c570204b3575bc217829cfea640ec Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 22:39:03 +0530 Subject: [PATCH 29/92] Added the changeset Signed-off-by: Harshil-Jani --- .changeset/eighty-planets-help.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/eighty-planets-help.md diff --git a/.changeset/eighty-planets-help.md b/.changeset/eighty-planets-help.md new file mode 100644 index 00000000..a9eba11a --- /dev/null +++ b/.changeset/eighty-planets-help.md @@ -0,0 +1,6 @@ +--- +"@caravan/bitcoin": minor +"@caravan/clients": minor +--- + +Added support for @caravan-health From d33a99d71d617ab633c63c7a609a91547726718e Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 23:07:58 +0530 Subject: [PATCH 30/92] Adding README Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 66 +++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 68586b2b..6ff9e8d3 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -1,12 +1,54 @@ -# Health - -# TODO - -- [x] Write logic for `isAddressReused` function in `privacy.ts`. -- [x] Cover test cases for `privacy.ts`. -- [x] Replace `any` types from `feescore.ts` -- [x] Cover test-cases for `feescore.ts` -- [ ] Implement `Transaction` type on the `getAddressTransaction` func. -- [ ] Take a decision on doubt mentioned in `getFeeRatePercentileForTransaction` -- [ ] Implement Caching or Ticker based mechanism to reduce network latency for percentile of feerate for a given block -- [ ] Write Blog on health and Add link to blog post +# Caravan-Health + +The `caravan-health` package is designed to help users maintain the health of their bitcoin wallets. Wallet health is determined by various factors including financial privacy, transaction fees, and the avoidance of dust outputs. This README will guide you through understanding wallet health goals, scoring metrics, and how to use the caravan-health package to achieve optimal wallet health. + +# Defining Wallet Health Goals +Different users have diverse needs and preferences which impact their wallet health goals. Some users prioritize financial privacy, others focus on minimizing transaction fees, and some want a healthy wallet without delving into the technical details of UTXOs and transactions. The caravan-health package aims to research metrics that help label scores for wallet health and provide suggestions for improvement. + +# Wallet Health Goals: +- Protect financial privacy +- Minimize long-term and short-term fee rates +- Avoid creating dust outputs +- Determine when to consolidate and when to conserve UTXOs +- Manage spending frequency and allow simultaneous payments + +--- + +# Scoring Metrics for health analysis + +## Privacy Metrics + +1. Reuse Factor (R.F) + +Measures the extent to which addresses are reused. Lower reuse factor improves privacy. + +2. Address Type Factor (A.T.F) + +Assesses privacy based on the diversity of address types used in transactions. + +3. UTXO Spread Factor (U.S.F) + +Evaluates the spread of UTXO values to gauge privacy. Higher spread indicates better privacy. + +4. Weightage on Number of UTXOs (W) + +Considers the number of UTXOs in the wallet. + +5. UTXO Value Weightage Factor (U.V.W.F) + +Combines UTXO spread and weightage on number of UTXOs. Adjusts privacy score based on UTXO value dispersion and quantity. + +# Fees Metrics + +1. Relative Fee Score (R.F.S) + +Measures the fee rate compared to historical data. Higher score indicates lower fee rates. + +2. Fee-to-Amount Percent Score (F.A.P.S) + +Ratio of fees paid to the transaction amount. Lower percentage signifies better fee efficiency. + +3. Weightage on Number of UTXOs (W) + +Considers the number of UTXOs. + From 68e072ea8ed20c0052c25a03cc82cf8cd858918a Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 23:17:56 +0530 Subject: [PATCH 31/92] Add isSend logic for mempool and blockstream using prevouts address comparision Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index b0e865f5..b8ef8e16 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -275,12 +275,15 @@ export class BlockchainClient extends ClientBase { size: tx.size, weight: tx.weight, fee: tx.fee, - isSend: false, // TODO : Need to implement this. + isSend: false, amount: 0, blocktime: tx.status.block_time, }; for (const input of tx.vin) { + if(input.prevout.scriptpubkey_address === address) { + transaction.isSend = true; + } transaction.vin.push({ txid: input.txid, vout: input.vout, From bc977e32fdc62b43bab5513f137957483cdaf1f3 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 23:29:56 +0530 Subject: [PATCH 32/92] Add waste metric to the feescore file Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.ts | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/feescore.ts index 61faf352..c2535c91 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/feescore.ts @@ -157,6 +157,42 @@ export function feesToAmountRatio(transactions: Transaction[]): number { return sumFeesToAmountRatio / numberOfSendTx; } +/* +Consider the waste score of the transaction which gives an idea of not spending a +particular output now (assuming fees are currently high), given that we may be able +to consolidate it later when fees are low. + +waste score = consolidation factor + cost of transaction +waste score = weight (feerate - L) + change + excess + +weight: total weight of the input set +feerate: the transaction's target feerate +L: the long-term feerate estimate which the wallet might need to pay to redeem remaining UTXOs +change: the cost of creating and spending a change output +excess: the amount by which we exceed our selection target when creating a changeless transaction, +mutually exclusive with cost of change + +“excess” is if we don't make a change output and instead add the difference to the fees. +If there is a change in output then the excess should be 0. +“change” includes the fees paid on this transaction's change output plus the fees that +will need to be paid to spend it later. Thus the quantity cost of transaction is always a positive number. + +Now depending on the feerate in the long term the consolidation factor can be positive or a negative quantity. + + feerate (current) < L (long-term feerate) –-> Consolidate now (-ve) + feerate (current) > L (long-term feerate) –-> Wait for later when feerate go low (+ve) +*/ +export function wasteMetric( + transaction: Transaction, // Amount that UTXO holds + amount: number, // Amount to be spent in the transaction + L: number, // Long term estimated fee-rate +): number { + let weight: number = transaction.weight; + let feeRate: number = getFeeRateForTransaction(transaction); + let costOfTx: number = Math.abs(amount - transaction.amount); + return weight * (feeRate - L) + costOfTx; +} + /* 35% Weightage of fees score depends on Percentile of fees paid 35% Weightage of fees score depends fees paid with respect to amount spend From 2f2b58f4d8669ac4560439249b27710a6bf18272 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 29 Jul 2024 23:33:56 +0530 Subject: [PATCH 33/92] Add test case for waste metric Signed-off-by: Harshil-Jani --- packages/caravan-health/src/feescore.test.ts | 38 +++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/feescore.test.ts index 3ede8a61..fc23a609 100644 --- a/packages/caravan-health/src/feescore.test.ts +++ b/packages/caravan-health/src/feescore.test.ts @@ -1,4 +1,9 @@ -import { feesScore, feesToAmountRatio, relativeFeesScore } from "./feescore"; +import { + feesScore, + feesToAmountRatio, + relativeFeesScore, + wasteMetric, +} from "./feescore"; import { BlockchainClient } from "@caravan/clients"; import { Transaction, FeeRatePercentile } from "@caravan/clients"; @@ -67,4 +72,35 @@ describe("Fees Score Functions", () => { expect(ratio).toBe(0.1); }); }); + + describe("wasteMetric Function", () => { + it("should calculate the correct waste metric value", () => { + const transaction: Transaction = { + vin: [], + vout: [], + txid: "tx1", + size: 1000, + weight: 500, + fee: 2, + isSend: true, + amount: 50, + blocktime: 1234, + }; + + const amount = 30; + const L = 30; + + const result = wasteMetric(transaction, amount, L); + + const expectedWeight = transaction.weight; + const feeRate = transaction.fee / transaction.weight; + const costOfTx = Math.abs(amount - transaction.amount); + const expectedWasteMetric = expectedWeight * (feeRate - L) + costOfTx; + + expect(result).toBe(expectedWasteMetric); + }); + }); }); +function getFeeRateForTransaction(transaction: Transaction) { + throw new Error("Function not implemented."); +} From 64d8a18792c425e3fc7a124da4ebc17bd7c3181f Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 30 Jul 2024 02:36:05 +0530 Subject: [PATCH 34/92] remove @caravan/health from @caravan/clients Signed-off-by: Harshil-Jani --- packages/caravan-clients/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/caravan-clients/package.json b/packages/caravan-clients/package.json index 0d946e37..e39000ba 100644 --- a/packages/caravan-clients/package.json +++ b/packages/caravan-clients/package.json @@ -40,7 +40,6 @@ "@babel/preset-env": "^7.23.8", "@caravan/eslint-config": "*", "@caravan/typescript-config": "*", - "@caravan/health": "*", "babel-jest": "^29.7.0", "eslint": "^8.56.0", "jest": "^29.7.0", From 6d7f28948f22a23d97bb740d602a25847883c363 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 31 Jul 2024 03:16:34 +0530 Subject: [PATCH 35/92] Move interface to types.ts Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 47 +------------------------- packages/caravan-clients/src/index.ts | 2 +- packages/caravan-clients/src/types.ts | 45 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 47 deletions(-) create mode 100644 packages/caravan-clients/src/types.ts diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index b8ef8e16..75f31124 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -18,6 +18,7 @@ import { bitcoindWalletInfo, } from "./wallet"; import BigNumber from "bignumber.js"; +import { FeeRatePercentile, Transaction, UTXO } from "./types"; export class BlockchainClientError extends Error { constructor(message) { @@ -26,58 +27,12 @@ export class BlockchainClientError extends Error { } } -export interface UTXO { - txid: string; - vout: number; - value: number; - status: { - confirmed: boolean; - block_time: number; - }; -} - -export interface Transaction { - txid: string; - vin: Input[]; - vout: Output[]; - size: number; - weight: number; - fee: number; - isSend: boolean; - amount: number; - blocktime: number; -} - -interface Input { - txid: string; - vout: number; - sequence: number; -} - -interface Output { - scriptPubkeyHex: string; - scriptPubkeyAddress: string; - value: number; -} - export enum ClientType { PRIVATE = "private", BLOCKSTREAM = "blockstream", MEMPOOL = "mempool", } -export interface FeeRatePercentile { - avgHeight: number; - timestamp: number; - avgFee_0: number; - avgFee_10: number; - avgFee_25: number; - avgFee_50: number; - avgFee_75: number; - avgFee_90: number; - avgFee_100: number; -} - const delay = () => { return new Promise((resolve) => setTimeout(resolve, 500)); }; diff --git a/packages/caravan-clients/src/index.ts b/packages/caravan-clients/src/index.ts index 4dc628fb..a89cd4be 100644 --- a/packages/caravan-clients/src/index.ts +++ b/packages/caravan-clients/src/index.ts @@ -1,3 +1,3 @@ export { bitcoindImportDescriptors } from "./wallet"; export { BlockchainClient, ClientType } from "./client"; -export type { UTXO, Transaction, FeeRatePercentile } from "./client"; +export type { UTXO, Transaction, FeeRatePercentile } from "./types"; diff --git a/packages/caravan-clients/src/types.ts b/packages/caravan-clients/src/types.ts new file mode 100644 index 00000000..edfd0633 --- /dev/null +++ b/packages/caravan-clients/src/types.ts @@ -0,0 +1,45 @@ +export interface UTXO { + txid: string; + vout: number; + value: number; + status: { + confirmed: boolean; + block_time: number; + }; +} + +export interface Transaction { + txid: string; + vin: Input[]; + vout: Output[]; + size: number; + weight: number; + fee: number; + isSend: boolean; + amount: number; + blocktime: number; +} + +interface Input { + txid: string; + vout: number; + sequence: number; +} + +interface Output { + scriptPubkeyHex: string; + scriptPubkeyAddress: string; + value: number; +} + +export interface FeeRatePercentile { + avgHeight: number; + timestamp: number; + avgFee_0: number; + avgFee_10: number; + avgFee_25: number; + avgFee_50: number; + avgFee_75: number; + avgFee_90: number; + avgFee_100: number; +} From 76509c9dd19dd269483643aa6f43f3f19ab39d80 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 31 Jul 2024 03:17:34 +0530 Subject: [PATCH 36/92] rolling release version of @caravan-health as 1.0.0-beta Signed-off-by: Harshil-Jani --- packages/caravan-health/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/package.json b/packages/caravan-health/package.json index 472525b0..ef9a8f14 100644 --- a/packages/caravan-health/package.json +++ b/packages/caravan-health/package.json @@ -1,6 +1,6 @@ { "name": "@caravan/health", - "version": "1.0.0", + "version": "1.0.0-beta", "author": "Harshil Jani", "description": "The core logic for analysing wallet health for privacy concerns and nature of spending fees.", "private": true, From 69eb402b0461230cd8979a320a22b5b514a883ce Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 7 Aug 2024 04:08:52 +0530 Subject: [PATCH 37/92] improving grammar and variable naming Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.test.ts | 2 +- packages/caravan-clients/src/client.ts | 6 +- packages/caravan-clients/src/types.ts | 2 +- packages/caravan-health/README.md | 9 +- packages/caravan-health/src/index.ts | 2 +- packages/caravan-health/src/privacy.test.ts | 26 +++--- packages/caravan-health/src/privacy.ts | 88 +++++++++---------- .../src/{feescore.test.ts => waste.test.ts} | 16 +--- .../src/{feescore.ts => waste.ts} | 44 +++------- 9 files changed, 79 insertions(+), 116 deletions(-) rename packages/caravan-health/src/{feescore.test.ts => waste.test.ts} (83%) rename packages/caravan-health/src/{feescore.ts => waste.ts} (82%) diff --git a/packages/caravan-clients/src/client.test.ts b/packages/caravan-clients/src/client.test.ts index 896a78ad..f47d7c9c 100644 --- a/packages/caravan-clients/src/client.test.ts +++ b/packages/caravan-clients/src/client.test.ts @@ -3,12 +3,12 @@ import { BlockchainClient, ClientType, ClientBase, - UTXO, BlockchainClientError, } from "./client"; import * as bitcoind from "./bitcoind"; import * as wallet from "./wallet"; import BigNumber from "bignumber.js"; +import { UTXO } from "./types"; import axios from "axios"; jest.mock("axios"); diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 75f31124..ba1b9f39 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -201,7 +201,7 @@ export class BlockchainClient extends ClientBase { }; for (const input of rawTxData.vin) { transaction.vin.push({ - txid: input.txid, + prevTxId: input.txid, vout: input.vout, sequence: input.sequence, }); @@ -240,7 +240,7 @@ export class BlockchainClient extends ClientBase { transaction.isSend = true; } transaction.vin.push({ - txid: input.txid, + prevTxId: input.txid, vout: input.vout, sequence: input.sequence, }); @@ -411,7 +411,7 @@ export class BlockchainClient extends ClientBase { public async getBlockFeeRatePercentileHistory(): Promise { try { - let data = await this.Get(`v1/mining/blocks/fee-rates/all`); + let data = await this.Get(`/v1/mining/blocks/fee-rates/all`); let feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { let feeRatePercentile: FeeRatePercentile = { diff --git a/packages/caravan-clients/src/types.ts b/packages/caravan-clients/src/types.ts index edfd0633..7a00f08e 100644 --- a/packages/caravan-clients/src/types.ts +++ b/packages/caravan-clients/src/types.ts @@ -21,7 +21,7 @@ export interface Transaction { } interface Input { - txid: string; + prevTxId: string; vout: number; sequence: number; } diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 6ff9e8d3..7fe05927 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -3,7 +3,7 @@ The `caravan-health` package is designed to help users maintain the health of their bitcoin wallets. Wallet health is determined by various factors including financial privacy, transaction fees, and the avoidance of dust outputs. This README will guide you through understanding wallet health goals, scoring metrics, and how to use the caravan-health package to achieve optimal wallet health. # Defining Wallet Health Goals -Different users have diverse needs and preferences which impact their wallet health goals. Some users prioritize financial privacy, others focus on minimizing transaction fees, and some want a healthy wallet without delving into the technical details of UTXOs and transactions. The caravan-health package aims to research metrics that help label scores for wallet health and provide suggestions for improvement. +Different users have diverse needs and preferences which impact their wallet health goals. Some users prioritize financial privacy, others focus on minimizing transaction fees, and some want a healthy wallet without delving into the technical details of UTXOs and transactions. The caravan-health package aims to highlight metrics for wallet health and provide suggestions for improvement. # Wallet Health Goals: - Protect financial privacy @@ -34,15 +34,16 @@ Evaluates the spread of UTXO values to gauge privacy. Higher spread indicates be Considers the number of UTXOs in the wallet. -5. UTXO Value Weightage Factor (U.V.W.F) +5. UTXO Value Dispersion Factor (U.V.D.F) Combines UTXO spread and weightage on number of UTXOs. Adjusts privacy score based on UTXO value dispersion and quantity. -# Fees Metrics +# Waste Metrics 1. Relative Fee Score (R.F.S) -Measures the fee rate compared to historical data. Higher score indicates lower fee rates. +Measures the fee rate compared to historical data. It can be associated with all the transactions and we can give a measure +if any transaction was done at expensive fees or nominal fees. 2. Fee-to-Amount Percent Score (F.A.P.S) diff --git a/packages/caravan-health/src/index.ts b/packages/caravan-health/src/index.ts index 302971de..3b6a425e 100644 --- a/packages/caravan-health/src/index.ts +++ b/packages/caravan-health/src/index.ts @@ -1,3 +1,3 @@ export * from "./privacy"; export * from "./types"; -export * from "./feescore"; +export * from "./waste"; diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index ac6c24b2..817e3435 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,11 +1,11 @@ import { - scoreForTxTopology, + getTopologyScore, addressReuseFactor, addressTypeFactor, utxoSpreadFactor, utxoSetLengthScore, - utxoValueWeightageFactor, - privacyScore, + utxoValueDispersionFactor, + getWalletPrivacyScore, } from "./privacy"; import { BlockchainClient, Transaction } from "@caravan/clients"; import { AddressUtxos } from "./types"; @@ -26,12 +26,12 @@ describe("Privacy Score Functions", () => { const transaction: Transaction = { vin: [ { - txid: "input1", + prevTxId: "input1", vout: 0, sequence: 0, }, { - txid: "input2", + prevTxId: "input2", vout: 0, sequence: 0, }, @@ -50,7 +50,7 @@ describe("Privacy Score Functions", () => { blocktime: 0, }; const score: number = +( - await scoreForTxTopology(transaction, mockClient) + await getTopologyScore(transaction, mockClient) ).toFixed(3); expect(score).toBe(0.818); }); @@ -91,13 +91,12 @@ describe("Privacy Score Functions", () => { describe("addressTypeFactor", () => { it("P2PKH address", () => { - const transactions = [ + const transactions: Transaction[] = [ { vin: [ { - txid: "input1", + prevTxId: "input1", vout: 0, - witness: [], sequence: 0, }, ], @@ -194,20 +193,19 @@ describe("Privacy Score Functions", () => { }, ], }; - const factor: number = +utxoValueWeightageFactor(utxos).toFixed(3); + const factor: number = +utxoValueDispersionFactor(utxos).toFixed(3); expect(factor).toBe(0.05); }); }); describe("privacyScore", () => { it("Privacy score", async () => { - const transactions = [ + const transactions: Transaction[] = [ { vin: [ { - txid: "input1", + prevTxId: "input1", vout: 0, - witness: [], sequence: 0, }, ], @@ -244,7 +242,7 @@ describe("Privacy Score Functions", () => { const walletAddressType: MultisigAddressType = "P2PKH"; const network: Network = Network["MAINNET"]; const score: number = +( - await privacyScore( + await getWalletPrivacyScore( transactions, utxos, walletAddressType, diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 03ad8068..d30aec03 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -19,31 +19,43 @@ const DENIABILITY_FACTOR = 1.5; The p_score is calculated by evaluating the likelihood of self-payments, the involvement of change outputs and the type of transaction based on number of inputs and outputs. -We have 5 categories of transaction type -- Sweep Spend -- Simple Spend -- UTXO Fragmentation (any transaction with more than the standard 2 outputs) -- Consolidation -- CoinJoin +We have 5 categories of transaction type each with their own impact on privacy score +- Perfect Spend (1 input, 1 output) +- Simple Spend (1 input, 2 outputs) +- UTXO Fragmentation (1 input, more than 2 standard outputs) +- Consolidation (more than 1 input, 1 output) +- CoinJoin or Mixing (more than 1 input, more than 1 output) */ enum SpendType { - SweepSpend = "SweepSpend", + PerfectSpend = "PerfectSpend", SimpleSpend = "SimpleSpend", UTXOFragmentation = "UTXOFragmentation", Consolidation = "Consolidation", MixingOrCoinJoin = "MixingOrCoinJoin", } +function determineSpendType(inputs: number, outputs: number): SpendType { + if (inputs === 1) { + if (outputs === 1) return SpendType.PerfectSpend; + if (outputs === 2) return SpendType.SimpleSpend; + return SpendType.UTXOFragmentation; + } else { + if (outputs === 1) return SpendType.Consolidation; + return SpendType.MixingOrCoinJoin; + } +} + /* The deterministic scores or their formula for each spend type are as follows +Refer to the link mentioned to understand the mathematical derivation of the scores. */ -function spendTypeScores( +function getSpendTypeScore( spendType: SpendType, numberOfInputs: number, numberOfOutputs: number, ): number { switch (spendType) { - case SpendType.SweepSpend: + case SpendType.PerfectSpend: return 1 / 2; case SpendType.SimpleSpend: return 4 / 9; @@ -59,20 +71,8 @@ function spendTypeScores( } } -function determineSpendType(inputs: number, outputs: number): SpendType { - if (inputs === 1) { - if (outputs === 1) return SpendType.SweepSpend; - if (outputs === 2) return SpendType.SimpleSpend; - return SpendType.UTXOFragmentation; - } else { - if (outputs === 1) return SpendType.Consolidation; - return SpendType.MixingOrCoinJoin; - } -} - /* -The transaction topology refers to the type of transaction based on -number of inputs and outputs. +The transaction topology score evaluates privacy metrics based on the number of inputs and outputs. Expected Range : [0, 0.75] -> Very Poor : [0, 0.15] @@ -81,19 +81,19 @@ Expected Range : [0, 0.75] -> Good : (0.45, 0.6] -> Very Good : (0.6, 0.75] */ -export async function privacyScoreByTxTopology( +export async function getMeanTopologyScore( transactions: Transaction[], client: BlockchainClient, ): Promise { let privacyScore = 0; for (let tx of transactions) { - let topologyScore = await scoreForTxTopology(tx, client); + let topologyScore = await getTopologyScore(tx, client); privacyScore += topologyScore; } return privacyScore / transactions.length; } -export async function scoreForTxTopology( +export async function getTopologyScore( transaction: Transaction, client: BlockchainClient, ): Promise { @@ -104,7 +104,7 @@ export async function scoreForTxTopology( numberOfInputs, numberOfOutputs, ); - const score: number = spendTypeScores( + const score: number = getSpendTypeScore( spendType, numberOfInputs, numberOfOutputs, @@ -113,8 +113,8 @@ export async function scoreForTxTopology( if (spendType === SpendType.Consolidation) { return score; } - for (let op of transaction.vout) { - let address = op.scriptPubkeyAddress; + for (let output of transaction.vout) { + let address = output.scriptPubkeyAddress; let isResued = await isReusedAddress(address, client); if (isResued === true) { return score; @@ -124,7 +124,7 @@ export async function scoreForTxTopology( } /* -In order to score for address reuse we can check the amount being hold by reused UTXOs +In order to score for address reuse we can check the amount being held by reused addresses with respect to the total amount Expected Range : [0,1] @@ -164,9 +164,7 @@ async function isReusedAddress( if (tx.isSend === false) { countReceive++; } - } - if (countReceive > 1) { - return true; + if (countReceive > 1) return true; } return false; } @@ -247,7 +245,7 @@ export function utxoSpreadFactor(utxos: AddressUtxos): number { } /* -The weightage is ad-hoc to normalize the privacy score based on the number of UTXOs in the set. +The score is ad-hoc to normalize the privacy score based on the number of UTXOs in the set. Expected Range : [0,1] - 0 for UTXO set length >= 50 @@ -262,24 +260,24 @@ export function utxoSetLengthScore(utxos: AddressUtxos): number { const addressUtxos = utxos[address]; utxoSetLength += addressUtxos.length; } - let weight: number; + let score: number; if (utxoSetLength >= 50) { - weight = 0; + score = 0; } else if (utxoSetLength >= 25 && utxoSetLength <= 49) { - weight = 0.25; + score = 0.25; } else if (utxoSetLength >= 15 && utxoSetLength <= 24) { - weight = 0.5; + score = 0.5; } else if (utxoSetLength >= 5 && utxoSetLength <= 14) { - weight = 0.75; + score = 0.75; } else { - weight = 1; + score = 1; } - return weight; + return score; } /* UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. -It signifies the combined effect of how well spreaded the UTXO Set is and how many number of UTXOs are there. +It signifies the combined effect of how much variance is there in the UTXO Set values is and how many number of UTXOs are there. Expected Range : [-0.15,0.15] -> Very Poor : [-0.1, -0.05) @@ -288,7 +286,7 @@ Expected Range : [-0.15,0.15] -> Good : [0.05, 0.1) -> Very Good : [0.1 ,0.15] */ -export function utxoValueWeightageFactor(utxos: AddressUtxos): number { +export function utxoValueDispersionFactor(utxos: AddressUtxos): number { let W: number = utxoSetLengthScore(utxos); let USF: number = utxoSpreadFactor(utxos); return (USF + W) * 0.15 - 0.15; @@ -308,14 +306,14 @@ Expected Range : [0, 1] -> Good : (0.6, 0.8] -> Very Good : (0.8, 1] */ -export async function privacyScore( +export async function getWalletPrivacyScore( transactions: Transaction[], utxos: AddressUtxos, walletAddressType: MultisigAddressType, client: BlockchainClient, network: Network, ): Promise { - let privacyScore = await privacyScoreByTxTopology(transactions, client); + let privacyScore = await getMeanTopologyScore(transactions, client); // Adjusting the privacy score based on the address reuse factor let addressReusedFactor = await addressReuseFactor(utxos, client); @@ -329,7 +327,7 @@ export async function privacyScore( (1 - addressTypeFactor(transactions, walletAddressType, network)); // Adjusting the privacy score based on the UTXO set length and value weightage factor - privacyScore = privacyScore + 0.1 * utxoValueWeightageFactor(utxos); + privacyScore = privacyScore + 0.1 * utxoValueDispersionFactor(utxos); return privacyScore; } diff --git a/packages/caravan-health/src/feescore.test.ts b/packages/caravan-health/src/waste.test.ts similarity index 83% rename from packages/caravan-health/src/feescore.test.ts rename to packages/caravan-health/src/waste.test.ts index fc23a609..f16073fc 100644 --- a/packages/caravan-health/src/feescore.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -1,22 +1,11 @@ import { - feesScore, feesToAmountRatio, relativeFeesScore, wasteMetric, -} from "./feescore"; -import { BlockchainClient } from "@caravan/clients"; +} from "./waste"; import { Transaction, FeeRatePercentile } from "@caravan/clients"; describe("Fees Score Functions", () => { - let mockClient: BlockchainClient; - - beforeEach(() => { - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest.fn().mockResolvedValue([{ txid: "tx1" }]), - } as unknown as BlockchainClient; - }); - describe("relativeFeesScore", () => { it("Relative fees score for transaction", () => { const transactions: Transaction[] = [ @@ -101,6 +90,3 @@ describe("Fees Score Functions", () => { }); }); }); -function getFeeRateForTransaction(transaction: Transaction) { - throw new Error("Function not implemented."); -} diff --git a/packages/caravan-health/src/feescore.ts b/packages/caravan-health/src/waste.ts similarity index 82% rename from packages/caravan-health/src/feescore.ts rename to packages/caravan-health/src/waste.ts index c2535c91..22614a66 100644 --- a/packages/caravan-health/src/feescore.ts +++ b/packages/caravan-health/src/waste.ts @@ -40,24 +40,7 @@ function getFeeRatePercentileScore( feeRate, feeRatePercentileHistory, ); - switch (percentile) { - case 0: - return 1; - case 10: - return 0.9; - case 25: - return 0.75; - case 50: - return 0.5; - case 75: - return 0.25; - case 90: - return 0.1; - case 100: - return 0; - default: - throw new Error("Invalid percentile"); - } + return 1-(percentile/100); } function getPercentile( @@ -136,14 +119,13 @@ export function relativeFeesScore( } /* -Measure of how much the wallet is burning in fees is that we take the ratio of -amount being paid and the fees consumed. +The measure of how much the wallet is affected by fees is determined by the ratio of the amount paid to the fees incurred. Mastercard charges 0.6% cross-border fee for international transactions in US dollars, but if the transaction is in any other currency the fee goes up to 1%. Source : https://www.clearlypayments.com/blog/what-are-cross-border-fees-in-credit-card-payments/ -This ratio is a measure of our fees spending against the fiat charges we pay. +This ratio is a measure of transaction fees compared with market rates for fiat processing fees. */ export function feesToAmountRatio(transactions: Transaction[]): number { let sumFeesToAmountRatio: number = 0; @@ -158,16 +140,15 @@ export function feesToAmountRatio(transactions: Transaction[]): number { } /* -Consider the waste score of the transaction which gives an idea of not spending a -particular output now (assuming fees are currently high), given that we may be able -to consolidate it later when fees are low. +Consider the waste score of the transaction which can inform whether or not it is economical to spend a +particular output now (assuming fees are currently high) or wait to consolidate it later when fees are low. waste score = consolidation factor + cost of transaction -waste score = weight (feerate - L) + change + excess +waste score = weight (fee rate - L) + change + excess -weight: total weight of the input set -feerate: the transaction's target feerate -L: the long-term feerate estimate which the wallet might need to pay to redeem remaining UTXOs +weight: Transaction weight units +feeRate: the transaction's target fee rate +L: the long-term fee rate estimate which the wallet might need to pay to redeem remaining UTXOs change: the cost of creating and spending a change output excess: the amount by which we exceed our selection target when creating a changeless transaction, mutually exclusive with cost of change @@ -177,10 +158,9 @@ If there is a change in output then the excess should be 0. “change” includes the fees paid on this transaction's change output plus the fees that will need to be paid to spend it later. Thus the quantity cost of transaction is always a positive number. -Now depending on the feerate in the long term the consolidation factor can be positive or a negative quantity. - - feerate (current) < L (long-term feerate) –-> Consolidate now (-ve) - feerate (current) > L (long-term feerate) –-> Wait for later when feerate go low (+ve) +Depending on the fee rate in the long term, the consolidation factor can either be positive or negative. + fee rate (current) < L (long-term fee rate) –-> Consolidate now (-ve) + fee rate (current) > L (long-term fee rate) –-> Wait for later when fee rate go low (+ve) */ export function wasteMetric( transaction: Transaction, // Amount that UTXO holds From b72beea15dde70bdede41b87346493c63b5e01eb Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 7 Aug 2024 23:23:45 +0530 Subject: [PATCH 38/92] More descriptive changeset Signed-off-by: Harshil-Jani --- .changeset/eighty-planets-help.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.changeset/eighty-planets-help.md b/.changeset/eighty-planets-help.md index a9eba11a..6a0ec073 100644 --- a/.changeset/eighty-planets-help.md +++ b/.changeset/eighty-planets-help.md @@ -3,4 +3,12 @@ "@caravan/clients": minor --- -Added support for @caravan-health +@caravan/client +We are exposing a new method `getAddressTransactions` which will fetch all the transaction for a given address and format it as per needs. To facilitate the change, we had moved the interfaces in the new file `types.ts`. + +Another change was about getting the block fee-rate percentile history from mempool as a client. + +@caravan/bitcoin +The new function that has the capability to detect the address type (i.e P2SH, P2PKH, P2WSH or P2TR) was added. + +Overall, The changes were to support the new library within caravan called @caravan/health. \ No newline at end of file From 021f4613dd9c22f4fb138aee8ade16e8fb566dbf Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 8 Aug 2024 00:12:58 +0530 Subject: [PATCH 39/92] Adding call to bitcoind in bitcoind.js file Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/bitcoind.js | 14 ++++++++++++++ packages/caravan-clients/src/client.ts | 12 ++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/caravan-clients/src/bitcoind.js b/packages/caravan-clients/src/bitcoind.js index 37f2beba..8c7cb063 100644 --- a/packages/caravan-clients/src/bitcoind.js +++ b/packages/caravan-clients/src/bitcoind.js @@ -143,3 +143,17 @@ export function bitcoindImportMulti({ url, auth, addresses, label, rescan }) { } return callBitcoind(...params); } + +export async function bitcoindRawTxData(txid){ + try{ + return await callBitcoind( + this.bitcoindParams.url, + this.bitcoindParams.auth, + "decoderawtransaction", + [txid], + ); + } + catch(e){ + return e; + } +} diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index ba1b9f39..89d7e87a 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -10,6 +10,7 @@ import { bitcoindSendRawTransaction, isWalletAddressNotFoundError, callBitcoind, + bitcoindRawTxData, } from "./bitcoind"; import { bitcoindGetAddressStatus, @@ -164,15 +165,6 @@ export class BlockchainClient extends ClientBase { } } - public async bitcoindRawTxData(txid: string): Promise { - return await callBitcoind( - this.bitcoindParams.url, - this.bitcoindParams.auth, - "decoderawtransaction", - [txid], - ); - } - public async getAddressTransactions(address: string): Promise { try { if (this.type === ClientType.PRIVATE) { @@ -187,7 +179,7 @@ export class BlockchainClient extends ClientBase { for (const tx of data) { if (tx.address === address) { let isTxSend = tx.category === "send" ? true : false; - const rawTxData = await this.bitcoindRawTxData(tx.txid); + const rawTxData = await bitcoindRawTxData(tx.txid); const transaction: Transaction = { txid: tx.txid, vin: [], From b1a282807429505d449293f74a7500e6a08c4d0c Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 8 Aug 2024 02:53:32 +0530 Subject: [PATCH 40/92] Descriptive Test cases added Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 7 +- packages/caravan-health/src/privacy.test.ts | 602 +++++++++++++------- packages/caravan-health/src/waste.test.ts | 146 +++-- 3 files changed, 483 insertions(+), 272 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 7fe05927..5f6ba03f 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -3,9 +3,11 @@ The `caravan-health` package is designed to help users maintain the health of their bitcoin wallets. Wallet health is determined by various factors including financial privacy, transaction fees, and the avoidance of dust outputs. This README will guide you through understanding wallet health goals, scoring metrics, and how to use the caravan-health package to achieve optimal wallet health. # Defining Wallet Health Goals + Different users have diverse needs and preferences which impact their wallet health goals. Some users prioritize financial privacy, others focus on minimizing transaction fees, and some want a healthy wallet without delving into the technical details of UTXOs and transactions. The caravan-health package aims to highlight metrics for wallet health and provide suggestions for improvement. # Wallet Health Goals: + - Protect financial privacy - Minimize long-term and short-term fee rates - Avoid creating dust outputs @@ -42,7 +44,7 @@ Combines UTXO spread and weightage on number of UTXOs. Adjusts privacy score bas 1. Relative Fee Score (R.F.S) -Measures the fee rate compared to historical data. It can be associated with all the transactions and we can give a measure +Measures the fee rate compared to historical data. It can be associated with all the transactions and we can give a measure if any transaction was done at expensive fees or nominal fees. 2. Fee-to-Amount Percent Score (F.A.P.S) @@ -53,3 +55,6 @@ Ratio of fees paid to the transaction amount. Lower percentage signifies better Considers the number of UTXOs. +## TODOs + +- [] Extent the test cases for privacy and waste metrics to cover every possible case. diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 817e3435..82716a59 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -6,251 +6,467 @@ import { utxoSetLengthScore, utxoValueDispersionFactor, getWalletPrivacyScore, + getMeanTopologyScore, } from "./privacy"; import { BlockchainClient, Transaction } from "@caravan/clients"; import { AddressUtxos } from "./types"; import { MultisigAddressType, Network } from "@caravan/bitcoin"; -describe("Privacy Score Functions", () => { +describe("Privacy Score Metrics", () => { let mockClient: BlockchainClient; + let mockTransactionsAddressNotReused: Transaction[] = [ + { + vin: [ + { + prevTxId: "abcd", + vout: 0, + sequence: 0, + }, + ], + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + ]; + let mockTransactionsAddressReused: Transaction[] = [ + { + vin: [ + { + prevTxId: "abcd", + vout: 0, + sequence: 0, + }, + ], + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + { + vin: [ + { + prevTxId: "abcd", + vout: 0, + sequence: 0, + }, + ], + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + ]; + + it("Perfect Spend Transaction without reused address for calculating transaction topology score", async () => { + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest + .fn() + .mockResolvedValue(mockTransactionsAddressNotReused), + } as unknown as BlockchainClient; + const transaction: Transaction = { + vin: [ + { + prevTxId: "input1", + vout: 0, + sequence: 0, + }, + ], // 1 Input + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output + txid: "123", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }; + const score: number = +( + await getTopologyScore(transaction, mockClient) + ).toFixed(3); + expect(score).toBe(0.75); + }); - beforeEach(() => { + it("Perfect Spend Transaction with reused address for calculating transaction topology score", async () => { mockClient = { getAddressStatus: jest.fn(), - getAddressTransactions: jest.fn().mockResolvedValue([{ txid: "tx1" }]), + getAddressTransactions: jest + .fn() + .mockResolvedValue(mockTransactionsAddressReused), } as unknown as BlockchainClient; + const transaction: Transaction = { + vin: [ + { + prevTxId: "input1", + vout: 0, + sequence: 0, + }, + ], // 1 Input + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output + txid: "123", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }; + const score: number = +( + await getTopologyScore(transaction, mockClient) + ).toFixed(3); + expect(score).toBe(0.5); }); - describe("privacyScoreByTxTopology", () => { - it("CoinJoin with reused address", async () => { - const transaction: Transaction = { + it("Calculating mean transaction topology score for multiple trnasactions", async () => { + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest + .fn() + .mockResolvedValue(mockTransactionsAddressNotReused), + } as unknown as BlockchainClient; + const transactions: Transaction[] = [ + { + // Perfect Spend (No Reused Address) - 0.75 vin: [ { prevTxId: "input1", vout: 0, sequence: 0, }, + ], // 1 Input + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output + txid: "123", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + { + // Simple Spend (No Reused Address) - 0.66 + vin: [ { - prevTxId: "input2", + prevTxId: "input1", vout: 0, sequence: 0, }, - ], // 2 inputs + ], // 1 Input vout: [ { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - ], // 3 Outputs - txid: "", + ], // 2 Outputs + txid: "123", size: 0, weight: 0, fee: 0, isSend: false, amount: 0, blocktime: 0, - }; - const score: number = +( - await getTopologyScore(transaction, mockClient) - ).toFixed(3); - expect(score).toBe(0.818); - }); - }); - - describe("addressReuseFactor", () => { - it("UTXOs having same addresses", async () => { - const utxos: AddressUtxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx3", - vout: 0, - value: 3, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - const factor: number = +( - await addressReuseFactor(utxos, mockClient) - ).toFixed(3); - expect(factor).toBe(0); - }); + }, + ]; + const score: number = +( + await getMeanTopologyScore(transactions, mockClient) + ).toFixed(3); + expect(score).toBe(0.708); }); - describe("addressTypeFactor", () => { - it("P2PKH address", () => { - const transactions: Transaction[] = [ + it("Address Reuse Factor accounts for Unspent coins that are on reused address with respect to total amount in wallet", async () => { + const utxos: AddressUtxos = { + address1: [ { - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], - vout: [ - { scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }, - ], - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, }, - ]; - const walletAddressType: MultisigAddressType = "P2PKH"; - const network: Network = Network["MAINNET"]; - const factor: number = +addressTypeFactor( - transactions, - walletAddressType, - network, - ).toFixed(3); - expect(factor).toBe(1); - }); + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx3", + vout: 0, + value: 3, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + + // No address was reused + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest + .fn() + .mockResolvedValue(mockTransactionsAddressNotReused), + } as unknown as BlockchainClient; + const factor: number = +( + await addressReuseFactor(utxos, mockClient) + ).toFixed(3); + expect(factor).toBe(0); + + // All addresses were reused + mockClient = { + getAddressStatus: jest.fn(), + getAddressTransactions: jest + .fn() + .mockResolvedValue(mockTransactionsAddressReused), + } as unknown as BlockchainClient; + const factor2: number = +( + await addressReuseFactor(utxos, mockClient) + ).toFixed(3); + expect(factor2).toBe(1); }); - describe("utxoSpreadFactor", () => { - it("UTXOs spread across multiple addresses", () => { - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ + it("P2PKH wallet address type being checked for all transactions", () => { + const transactions: Transaction[] = [ + { + vin: [ { - txid: "tx2", + prevTxId: "input1", vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, + sequence: 0, }, ], - }; - const factor: number = +utxoSpreadFactor(utxos).toFixed(3); - expect(factor).toBe(0.333); - }); + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], // Address starting with 1 + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + ]; + const walletAddressType: MultisigAddressType = "P2PKH"; + const network: Network = Network["MAINNET"]; + const factor: number = +addressTypeFactor( + transactions, + walletAddressType, + network, + ).toFixed(3); + expect(factor).toBe(1); }); - describe("utxoSetLengthScore", () => { - it("UTXO set length", () => { - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - const score: number = +utxoSetLengthScore(utxos).toFixed(3); - expect(score).toBe(1); - }); + it("UTXOs spread factor across multiple addresses (assess for how similar the amount values are for each UTXO)", () => { + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + // Value 1 and 2 is so close to each other so the scoring is bad. + const factor: number = +utxoSpreadFactor(utxos).toFixed(3); + expect(factor).toBe(0.333); + + const utxos2 = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 200, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + // Value 1 and 200 is so far from each other so the scoring is good. + const factor2: number = +utxoSpreadFactor(utxos2).toFixed(3); + expect(factor2).toBe(0.99); }); - describe("utxoValueWeightageFactor", () => { - it("UTXO value weightage", () => { - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - const factor: number = +utxoValueDispersionFactor(utxos).toFixed(3); - expect(factor).toBe(0.05); - }); + it("Gives a score on the basis of number of UTXOs present in the wallet", () => { + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], // 1 UTXO only for address1 + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx3", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx4", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx5", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx6", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx7", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], // 6 UTXOs for address 2 + }; + // 7 UTXOs in total - which will give 0.75 as score + const score: number = +utxoSetLengthScore(utxos).toFixed(3); + expect(score).toBe(0.75); }); - describe("privacyScore", () => { - it("Privacy score", async () => { - const transactions: Transaction[] = [ + it("UTXO value dispersion accounts for number of coins in the wallet and how dispersed they are in amount values", () => { + const utxos = { + address1: [ { - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], - vout: [ - { scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }, - ], - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, }, - ]; - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ + ], // 1 UTXO only for address1 + address2: [ + { + txid: "tx2", + vout: 0, + value: 0.02, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx3", + vout: 0, + value: 0.2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx4", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx5", + vout: 0, + value: 20, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx6", + vout: 0, + value: 200, + status: { confirmed: true, block_time: 0 }, + }, + { + txid: "tx7", + vout: 0, + value: 2000, + status: { confirmed: true, block_time: 0 }, + }, + ], // 6 UTXOs for address 2 + }; + const factor: number = +utxoValueDispersionFactor(utxos).toFixed(3); + expect(factor).toBe(0.112); + }); + + it("Overall Privacy Score taking into consideration all parameters for UTXO and Transaction History", async () => { + const transactions: Transaction[] = [ + { + vin: [ { - txid: "tx2", + prevTxId: "input1", vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, + sequence: 0, }, ], - }; - const walletAddressType: MultisigAddressType = "P2PKH"; - const network: Network = Network["MAINNET"]; - const score: number = +( - await getWalletPrivacyScore( - transactions, - utxos, - walletAddressType, - mockClient, - network, - ) - ).toFixed(3); - expect(score).toBe(0.005); - }); + vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], + txid: "", + size: 0, + weight: 0, + fee: 0, + isSend: false, + amount: 0, + blocktime: 0, + }, + ]; + const utxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 1, + status: { confirmed: true, block_time: 0 }, + }, + ], + address2: [ + { + txid: "tx2", + vout: 0, + value: 2, + status: { confirmed: true, block_time: 0 }, + }, + ], + }; + const walletAddressType: MultisigAddressType = "P2PKH"; + const network: Network = Network["MAINNET"]; + const score: number = +( + await getWalletPrivacyScore( + transactions, + utxos, + walletAddressType, + mockClient, + network, + ) + ).toFixed(3); + expect(score).toBe(0.005); }); }); diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index f16073fc..87e1ea88 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -1,92 +1,82 @@ -import { - feesToAmountRatio, - relativeFeesScore, - wasteMetric, -} from "./waste"; +import { feesToAmountRatio, relativeFeesScore, wasteMetric } from "./waste"; import { Transaction, FeeRatePercentile } from "@caravan/clients"; -describe("Fees Score Functions", () => { - describe("relativeFeesScore", () => { - it("Relative fees score for transaction", () => { - const transactions: Transaction[] = [ - { - vin: [], - vout: [], - txid: "tx1", - size: 0, - weight: 1, - fee: 1, - isSend: true, - amount: 0, - blocktime: 1234, - }, - ]; - const feeRatePercentileHistory: FeeRatePercentile[] = [ - { - avgHeight: 0, - timestamp: 1234, - avgFee_0: 0, - avgFee_10: 0, - avgFee_25: 0.5, - avgFee_50: 1, - avgFee_75: 0, - avgFee_90: 0, - avgFee_100: 0, - }, - ]; - const score: number = +relativeFeesScore( - transactions, - feeRatePercentileHistory, - ).toFixed(3); - expect(score).toBe(0.5); - }); - }); - - describe("feesToAmountRatio", () => { - it("Fees to amount ratio for transaction", async () => { - const transaction: Transaction[] = [ - { - vin: [], - vout: [], - txid: "tx1", - size: 0, - weight: 0, - fee: 1, - isSend: true, - amount: 10, - blocktime: 0, - }, - ]; - const ratio: number = +feesToAmountRatio(transaction).toFixed(3); - expect(ratio).toBe(0.1); - }); +describe("Waste metrics that accounts for nature of fees spending for a wallet", () => { + it("Relative fees score for transaction with respect to international fiat payment charges", () => { + const transactions: Transaction[] = [ + { + vin: [], + vout: [], + txid: "tx1", + size: 0, + weight: 1, + fee: 1, + isSend: true, + amount: 0, + blocktime: 1234, + }, + ]; + const feeRatePercentileHistory: FeeRatePercentile[] = [ + { + avgHeight: 0, + timestamp: 1234, + avgFee_0: 0, + avgFee_10: 0, + avgFee_25: 0.5, + avgFee_50: 1, + avgFee_75: 0, + avgFee_90: 0, + avgFee_100: 0, + }, + ]; + const score: number = +relativeFeesScore( + transactions, + feeRatePercentileHistory, + ).toFixed(3); + expect(score).toBe(0.5); }); - describe("wasteMetric Function", () => { - it("should calculate the correct waste metric value", () => { - const transaction: Transaction = { + it("Fees paid for total amount spent as ratio for a transaction", async () => { + const transaction: Transaction[] = [ + { vin: [], vout: [], txid: "tx1", - size: 1000, - weight: 500, - fee: 2, + size: 0, + weight: 0, + fee: 1, isSend: true, - amount: 50, - blocktime: 1234, - }; + amount: 10, + blocktime: 0, + }, + ]; + const ratio: number = +feesToAmountRatio(transaction).toFixed(3); + expect(ratio).toBe(0.1); + }); + + it("waste score metric that determines the cost of keeping or spending the UTXO at given point of time", () => { + const transaction: Transaction = { + vin: [], + vout: [], + txid: "tx1", + size: 1000, + weight: 500, + fee: 2, + isSend: true, + amount: 50, + blocktime: 1234, + }; - const amount = 30; - const L = 30; + const amount = 30; + const L = 30; - const result = wasteMetric(transaction, amount, L); + const result = wasteMetric(transaction, amount, L); - const expectedWeight = transaction.weight; - const feeRate = transaction.fee / transaction.weight; - const costOfTx = Math.abs(amount - transaction.amount); - const expectedWasteMetric = expectedWeight * (feeRate - L) + costOfTx; + const expectedWeight = transaction.weight; + const feeRate = transaction.fee / transaction.weight; + const costOfTx = Math.abs(amount - transaction.amount); + const expectedWasteMetric = expectedWeight * (feeRate - L) + costOfTx; - expect(result).toBe(expectedWasteMetric); - }); + expect(result).toBe(expectedWasteMetric); }); }); From 12b78bd967cad8135fde1cbf26e98c46dda3bb17 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 9 Aug 2024 03:38:06 +0530 Subject: [PATCH 41/92] Replacing Weightage with UTXO Mass Factor Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 6 +++--- packages/caravan-health/src/privacy.ts | 10 +++++----- packages/caravan-health/src/waste.ts | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 5f6ba03f..0e832255 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -32,13 +32,13 @@ Assesses privacy based on the diversity of address types used in transactions. Evaluates the spread of UTXO values to gauge privacy. Higher spread indicates better privacy. -4. Weightage on Number of UTXOs (W) +4. UTXO Mass Factor score which accounts for Number of UTXOs present in a wallet (U.M.F) Considers the number of UTXOs in the wallet. 5. UTXO Value Dispersion Factor (U.V.D.F) -Combines UTXO spread and weightage on number of UTXOs. Adjusts privacy score based on UTXO value dispersion and quantity. +Combines the scores of UTXO spread and UTXO mass. # Waste Metrics @@ -51,7 +51,7 @@ if any transaction was done at expensive fees or nominal fees. Ratio of fees paid to the transaction amount. Lower percentage signifies better fee efficiency. -3. Weightage on Number of UTXOs (W) +3. UTXO Mass Factor on Number of UTXOs (UMF) Considers the number of UTXOs. diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index d30aec03..4426148a 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -276,7 +276,7 @@ export function utxoSetLengthScore(utxos: AddressUtxos): number { } /* -UTXO Value Weightage Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. +UTXO Value Dispersion Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. It signifies the combined effect of how much variance is there in the UTXO Set values is and how many number of UTXOs are there. Expected Range : [-0.15,0.15] @@ -287,9 +287,9 @@ Expected Range : [-0.15,0.15] -> Very Good : [0.1 ,0.15] */ export function utxoValueDispersionFactor(utxos: AddressUtxos): number { - let W: number = utxoSetLengthScore(utxos); + let UMF: number = utxoSetLengthScore(utxos); let USF: number = utxoSpreadFactor(utxos); - return (USF + W) * 0.15 - 0.15; + return (USF + UMF) * 0.15 - 0.15; } /* @@ -297,7 +297,7 @@ The privacy score is a combination of all the factors calculated above. - Privacy Score based on Inputs and Outputs (i.e Tx Topology) - Address Reuse Factor (R.F) - Address Type Factor (A.T.F) -- UTXO Value Weightage Factor (U.V.W.F) +- UTXO Value Dispersion Factor (U.V.D.F) Expected Range : [0, 1] -> Very Poor : [0, 0.2] @@ -326,7 +326,7 @@ export async function getWalletPrivacyScore( privacyScore * (1 - addressTypeFactor(transactions, walletAddressType, network)); - // Adjusting the privacy score based on the UTXO set length and value weightage factor + // Adjusting the privacy score based on the UTXO set length and value dispersion factor privacyScore = privacyScore + 0.1 * utxoValueDispersionFactor(utxos); return privacyScore; diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 22614a66..92e73402 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -178,9 +178,9 @@ export function wasteMetric( 35% Weightage of fees score depends fees paid with respect to amount spend 30% Weightage of fees score depends on the number of UTXOs present in the wallet. -Q : What role does W plays in the fees score? +Q : What role does UMF plays in the fees score? Assume the wallet is being consolidated, Thus number of UTXO will decrease and thus -W (Weightage of number of UTXO) will increase and this justifies that, consolidation +UMF (UTXO Mass Factor) will increase and this justifies that, consolidation increases the fees health since you don’t overpay them in long run. Expected Range : [0, 1] @@ -197,6 +197,6 @@ export async function feesScore( ): Promise { let RFS: number = relativeFeesScore(transactions, feeRatePercentileHistory); let FAR: number = feesToAmountRatio(transactions); - let W: number = utxoSetLengthScore(utxos); - return 0.35 * RFS + 0.35 * FAR + 0.3 * W; + let UMS: number = utxoSetLengthScore(utxos); + return 0.35 * RFS + 0.35 * FAR + 0.3 * UMS; } From 99ca020ed84efe0c74c1b0eaf14150712f23d379 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 9 Aug 2024 03:49:09 +0530 Subject: [PATCH 42/92] L to estimateLongTermFeeRate Signed-off-by: Harshil-Jani --- packages/caravan-health/src/waste.test.ts | 12 +++++++++--- packages/caravan-health/src/waste.ts | 17 ++++++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 87e1ea88..10ddca5f 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -68,14 +68,20 @@ describe("Waste metrics that accounts for nature of fees spending for a wallet", }; const amount = 30; - const L = 30; + // Reference on estimatedLongTermFeeRate : https://bitcoincore.reviews/17331#l-164 + const estimatedLongTermFeeRate = 30; - const result = wasteMetric(transaction, amount, L); + const result = wasteMetric( + transaction, + amount, + estimatedLongTermFeeRate, + ); const expectedWeight = transaction.weight; const feeRate = transaction.fee / transaction.weight; const costOfTx = Math.abs(amount - transaction.amount); - const expectedWasteMetric = expectedWeight * (feeRate - L) + costOfTx; + const expectedWasteMetric = + expectedWeight * (feeRate - estimatedLongTermFeeRate) + costOfTx; expect(result).toBe(expectedWasteMetric); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 92e73402..56b168af 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -40,7 +40,7 @@ function getFeeRatePercentileScore( feeRate, feeRatePercentileHistory, ); - return 1-(percentile/100); + return 1 - percentile / 100; } function getPercentile( @@ -144,11 +144,14 @@ Consider the waste score of the transaction which can inform whether or not it i particular output now (assuming fees are currently high) or wait to consolidate it later when fees are low. waste score = consolidation factor + cost of transaction -waste score = weight (fee rate - L) + change + excess +waste score = weight (fee rate - estimatedLongTermFeeRate) + change + excess + +// Reference on estimatedLongTermFeeRate : https://bitcoincore.reviews/17331#l-164 +// It is upper bound for spending the UTXO in the future weight: Transaction weight units feeRate: the transaction's target fee rate -L: the long-term fee rate estimate which the wallet might need to pay to redeem remaining UTXOs +estimatedLongTermFeeRate: the long-term fee rate estimate which the wallet might need to pay to redeem remaining UTXOs change: the cost of creating and spending a change output excess: the amount by which we exceed our selection target when creating a changeless transaction, mutually exclusive with cost of change @@ -159,18 +162,18 @@ If there is a change in output then the excess should be 0. will need to be paid to spend it later. Thus the quantity cost of transaction is always a positive number. Depending on the fee rate in the long term, the consolidation factor can either be positive or negative. - fee rate (current) < L (long-term fee rate) –-> Consolidate now (-ve) - fee rate (current) > L (long-term fee rate) –-> Wait for later when fee rate go low (+ve) + fee rate (current) < estimatedLongTermFeeRate (long-term fee rate) –-> Consolidate now (-ve) + fee rate (current) > estimatedLongTermFeeRate (long-term fee rate) –-> Wait for later when fee rate go low (+ve) */ export function wasteMetric( transaction: Transaction, // Amount that UTXO holds amount: number, // Amount to be spent in the transaction - L: number, // Long term estimated fee-rate + estimatedLongTermFeeRate: number, // Long term estimated fee-rate ): number { let weight: number = transaction.weight; let feeRate: number = getFeeRateForTransaction(transaction); let costOfTx: number = Math.abs(amount - transaction.amount); - return weight * (feeRate - L) + costOfTx; + return weight * (feeRate - estimatedLongTermFeeRate) + costOfTx; } /* From 0a2f9ff113aea59a6c85cb8357acdf5205fac120 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sat, 10 Aug 2024 03:00:12 +0530 Subject: [PATCH 43/92] Improve docs Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 3 ++- packages/caravan-health/src/privacy.ts | 8 +------- packages/caravan-health/src/waste.test.ts | 10 +++------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 0e832255..e94130d5 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -57,4 +57,5 @@ Considers the number of UTXOs. ## TODOs -- [] Extent the test cases for privacy and waste metrics to cover every possible case. +- [] Expand the test cases for privacy and waste metrics to cover every possible case. +- [] Add links to each algorithm and the corresponding explanation in final research document. diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 4426148a..78069c85 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -7,8 +7,6 @@ The methodology for calculating a privacy score (p_score) for Bitcoin transactio on the number of inputs and outputs is the primary point to define wallet health for privacy. The score is further influenced by several factors such as address reuse, address types and UTXO set fingerprints etc. - -More on the algorithms for scoring privacy: */ // A normalizing quantity that increases the score by a certain factor in cases of self-payment. @@ -45,11 +43,7 @@ function determineSpendType(inputs: number, outputs: number): SpendType { } } -/* -The deterministic scores or their formula for each spend type are as follows -Refer to the link mentioned to understand the mathematical derivation of the scores. -*/ -function getSpendTypeScore( +export function getSpendTypeScore( spendType: SpendType, numberOfInputs: number, numberOfOutputs: number, diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 10ddca5f..f4a851ce 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -1,8 +1,8 @@ import { feesToAmountRatio, relativeFeesScore, wasteMetric } from "./waste"; import { Transaction, FeeRatePercentile } from "@caravan/clients"; -describe("Waste metrics that accounts for nature of fees spending for a wallet", () => { - it("Relative fees score for transaction with respect to international fiat payment charges", () => { +describe("Waste metric scoring", () => { + it("calculates fee score based on tx fee rate relative to percentile in the block where a set of send tx were mined", () => { const transactions: Transaction[] = [ { vin: [], @@ -71,11 +71,7 @@ describe("Waste metrics that accounts for nature of fees spending for a wallet", // Reference on estimatedLongTermFeeRate : https://bitcoincore.reviews/17331#l-164 const estimatedLongTermFeeRate = 30; - const result = wasteMetric( - transaction, - amount, - estimatedLongTermFeeRate, - ); + const result = wasteMetric(transaction, amount, estimatedLongTermFeeRate); const expectedWeight = transaction.weight; const feeRate = transaction.fee / transaction.weight; From 626714b55d46e112b9bb73f27aa66cce9e089368 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sat, 10 Aug 2024 03:21:05 +0530 Subject: [PATCH 44/92] remove precision from tests Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 58 ++++++++------------- packages/caravan-health/src/waste.test.ts | 6 +-- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 82716a59..ef9a15ef 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -94,9 +94,7 @@ describe("Privacy Score Metrics", () => { amount: 0, blocktime: 0, }; - const score: number = +( - await getTopologyScore(transaction, mockClient) - ).toFixed(3); + const score: number = await getTopologyScore(transaction, mockClient); expect(score).toBe(0.75); }); @@ -124,9 +122,7 @@ describe("Privacy Score Metrics", () => { amount: 0, blocktime: 0, }; - const score: number = +( - await getTopologyScore(transaction, mockClient) - ).toFixed(3); + const score: number = await getTopologyScore(transaction, mockClient); expect(score).toBe(0.5); }); @@ -178,10 +174,8 @@ describe("Privacy Score Metrics", () => { blocktime: 0, }, ]; - const score: number = +( - await getMeanTopologyScore(transactions, mockClient) - ).toFixed(3); - expect(score).toBe(0.708); + const score: number = await getMeanTopologyScore(transactions, mockClient); + expect(score).toBeCloseTo(0.708); }); it("Address Reuse Factor accounts for Unspent coins that are on reused address with respect to total amount in wallet", async () => { @@ -217,9 +211,7 @@ describe("Privacy Score Metrics", () => { .fn() .mockResolvedValue(mockTransactionsAddressNotReused), } as unknown as BlockchainClient; - const factor: number = +( - await addressReuseFactor(utxos, mockClient) - ).toFixed(3); + const factor: number = await addressReuseFactor(utxos, mockClient); expect(factor).toBe(0); // All addresses were reused @@ -229,9 +221,7 @@ describe("Privacy Score Metrics", () => { .fn() .mockResolvedValue(mockTransactionsAddressReused), } as unknown as BlockchainClient; - const factor2: number = +( - await addressReuseFactor(utxos, mockClient) - ).toFixed(3); + const factor2: number = await addressReuseFactor(utxos, mockClient); expect(factor2).toBe(1); }); @@ -257,11 +247,11 @@ describe("Privacy Score Metrics", () => { ]; const walletAddressType: MultisigAddressType = "P2PKH"; const network: Network = Network["MAINNET"]; - const factor: number = +addressTypeFactor( + const factor: number = addressTypeFactor( transactions, walletAddressType, network, - ).toFixed(3); + ); expect(factor).toBe(1); }); @@ -285,8 +275,8 @@ describe("Privacy Score Metrics", () => { ], }; // Value 1 and 2 is so close to each other so the scoring is bad. - const factor: number = +utxoSpreadFactor(utxos).toFixed(3); - expect(factor).toBe(0.333); + const factor: number = utxoSpreadFactor(utxos); + expect(factor).toBe(1 / 3); const utxos2 = { address1: [ @@ -307,8 +297,8 @@ describe("Privacy Score Metrics", () => { ], }; // Value 1 and 200 is so far from each other so the scoring is good. - const factor2: number = +utxoSpreadFactor(utxos2).toFixed(3); - expect(factor2).toBe(0.99); + const factor2: number = utxoSpreadFactor(utxos2); + expect(factor2).toBeCloseTo(0.99); }); it("Gives a score on the basis of number of UTXOs present in the wallet", () => { @@ -361,7 +351,7 @@ describe("Privacy Score Metrics", () => { ], // 6 UTXOs for address 2 }; // 7 UTXOs in total - which will give 0.75 as score - const score: number = +utxoSetLengthScore(utxos).toFixed(3); + const score: number = utxoSetLengthScore(utxos); expect(score).toBe(0.75); }); @@ -414,8 +404,8 @@ describe("Privacy Score Metrics", () => { }, ], // 6 UTXOs for address 2 }; - const factor: number = +utxoValueDispersionFactor(utxos).toFixed(3); - expect(factor).toBe(0.112); + const factor: number = utxoValueDispersionFactor(utxos); + expect(factor).toBeCloseTo(0.112); }); it("Overall Privacy Score taking into consideration all parameters for UTXO and Transaction History", async () => { @@ -458,15 +448,13 @@ describe("Privacy Score Metrics", () => { }; const walletAddressType: MultisigAddressType = "P2PKH"; const network: Network = Network["MAINNET"]; - const score: number = +( - await getWalletPrivacyScore( - transactions, - utxos, - walletAddressType, - mockClient, - network, - ) - ).toFixed(3); - expect(score).toBe(0.005); + const score: number = await getWalletPrivacyScore( + transactions, + utxos, + walletAddressType, + mockClient, + network, + ); + expect(score).toBeCloseTo(0.005); }); }); diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index f4a851ce..154335f4 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -29,10 +29,10 @@ describe("Waste metric scoring", () => { avgFee_100: 0, }, ]; - const score: number = +relativeFeesScore( + const score: number = relativeFeesScore( transactions, feeRatePercentileHistory, - ).toFixed(3); + ); expect(score).toBe(0.5); }); @@ -50,7 +50,7 @@ describe("Waste metric scoring", () => { blocktime: 0, }, ]; - const ratio: number = +feesToAmountRatio(transaction).toFixed(3); + const ratio: number = feesToAmountRatio(transaction); expect(ratio).toBe(0.1); }); From e3a94bb6f2411853267217aee8c36038f11bcb64 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sat, 10 Aug 2024 04:10:27 +0530 Subject: [PATCH 45/92] syntax improvement Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 47 ++++++++++++++------------ packages/caravan-health/src/privacy.ts | 2 +- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 89d7e87a..7cb845d4 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -178,7 +178,6 @@ export class BlockchainClient extends ClientBase { const txs: Transaction[] = []; for (const tx of data) { if (tx.address === address) { - let isTxSend = tx.category === "send" ? true : false; const rawTxData = await bitcoindRawTxData(tx.txid); const transaction: Transaction = { txid: tx.txid, @@ -187,7 +186,7 @@ export class BlockchainClient extends ClientBase { size: rawTxData.size, weight: rawTxData.weight, fee: tx.fee, - isSend: isTxSend, + isSend: tx.category === "send" ? true : false, amount: tx.amount, blocktime: tx.blocktime, }; @@ -228,7 +227,7 @@ export class BlockchainClient extends ClientBase { }; for (const input of tx.vin) { - if(input.prevout.scriptpubkey_address === address) { + if (input.prevout.scriptpubkey_address === address) { transaction.isSend = true; } transaction.vin.push({ @@ -401,27 +400,31 @@ export class BlockchainClient extends ClientBase { } } - public async getBlockFeeRatePercentileHistory(): Promise { + public async getBlockFeeRatePercentileHistory(): Promise< + FeeRatePercentile[] + > { try { - let data = await this.Get(`/v1/mining/blocks/fee-rates/all`); - let feeRatePercentileBlocks: FeeRatePercentile[] = []; - for (const block of data) { - let feeRatePercentile: FeeRatePercentile = { - avgHeight: block?.avgHeight, - timestamp: block?.timestamp, - avgFee_0: block?.avgFee_0, - avgFee_10: block?.avgFee_10, - avgFee_25: block?.avgFee_25, - avgFee_50: block?.avgFee_50, - avgFee_75: block?.avgFee_75, - avgFee_90: block?.avgFee_90, - avgFee_100: block?.avgFee_100, - }; - feeRatePercentileBlocks.push(feeRatePercentile); - } - return feeRatePercentileBlocks; + let data = await this.Get(`/v1/mining/blocks/fee-rates/all`); + let feeRatePercentileBlocks: FeeRatePercentile[] = []; + for (const block of data) { + let feeRatePercentile: FeeRatePercentile = { + avgHeight: block?.avgHeight, + timestamp: block?.timestamp, + avgFee_0: block?.avgFee_0, + avgFee_10: block?.avgFee_10, + avgFee_25: block?.avgFee_25, + avgFee_50: block?.avgFee_50, + avgFee_75: block?.avgFee_75, + avgFee_90: block?.avgFee_90, + avgFee_100: block?.avgFee_100, + }; + feeRatePercentileBlocks.push(feeRatePercentile); + } + return feeRatePercentileBlocks; } catch (error: any) { - throw new Error(`Failed to get feerate percentile block: ${error.message}`); + throw new Error( + `Failed to get feerate percentile block: ${error.message}`, + ); } } diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 78069c85..6fdbeb44 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -155,7 +155,7 @@ async function isReusedAddress( let txs: Transaction[] = await client.getAddressTransactions(address); let countReceive = 0; for (const tx of txs) { - if (tx.isSend === false) { + if (!tx.isSend) { countReceive++; } if (countReceive > 1) return true; From 2b185fd6d5926ebd77d053ad6c38d45ad26700bf Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 12 Aug 2024 02:22:26 +0530 Subject: [PATCH 46/92] Introducing new class WasteMetric Signed-off-by: Harshil-Jani --- packages/caravan-health/src/index.ts | 1 + packages/caravan-health/src/privacy.test.ts | 4 +- packages/caravan-health/src/privacy.ts | 4 +- packages/caravan-health/src/utils.ts | 82 +++++ packages/caravan-health/src/waste.test.ts | 154 ++++----- packages/caravan-health/src/waste.ts | 346 +++++++++----------- 6 files changed, 316 insertions(+), 275 deletions(-) create mode 100644 packages/caravan-health/src/utils.ts diff --git a/packages/caravan-health/src/index.ts b/packages/caravan-health/src/index.ts index 3b6a425e..3fa0b667 100644 --- a/packages/caravan-health/src/index.ts +++ b/packages/caravan-health/src/index.ts @@ -1,3 +1,4 @@ export * from "./privacy"; export * from "./types"; export * from "./waste"; +export * from "./utils"; diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index ef9a15ef..bcd5586b 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -3,7 +3,7 @@ import { addressReuseFactor, addressTypeFactor, utxoSpreadFactor, - utxoSetLengthScore, + utxoMassFactor, utxoValueDispersionFactor, getWalletPrivacyScore, getMeanTopologyScore, @@ -351,7 +351,7 @@ describe("Privacy Score Metrics", () => { ], // 6 UTXOs for address 2 }; // 7 UTXOs in total - which will give 0.75 as score - const score: number = utxoSetLengthScore(utxos); + const score: number = utxoMassFactor(utxos); expect(score).toBe(0.75); }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 6fdbeb44..83057394 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -248,7 +248,7 @@ Expected Range : [0,1] - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ -export function utxoSetLengthScore(utxos: AddressUtxos): number { +export function utxoMassFactor(utxos: AddressUtxos): number { let utxoSetLength = 0; for (const address in utxos) { const addressUtxos = utxos[address]; @@ -281,7 +281,7 @@ Expected Range : [-0.15,0.15] -> Very Good : [0.1 ,0.15] */ export function utxoValueDispersionFactor(utxos: AddressUtxos): number { - let UMF: number = utxoSetLengthScore(utxos); + let UMF: number = utxoMassFactor(utxos); let USF: number = utxoSpreadFactor(utxos); return (USF + UMF) * 0.15 - 0.15; } diff --git a/packages/caravan-health/src/utils.ts b/packages/caravan-health/src/utils.ts new file mode 100644 index 00000000..f795d6d3 --- /dev/null +++ b/packages/caravan-health/src/utils.ts @@ -0,0 +1,82 @@ +import { FeeRatePercentile, Transaction } from "@caravan/clients"; + +/* +Utility function that helps to obtain the fee rate of the transaction + +Expected Range : [0, 0.75] +-> Very Poor : [0, 0.15] +-> Poor : (0.15, 0.3] +-> Moderate : (0.3, 0.45] +-> Good : (0.45, 0.6] +-> Very Good : (0.6, 0.75] +*/ +export function getFeeRateForTransaction(transaction: Transaction): number { + let fees: number = transaction.fee; + let weight: number = transaction.weight; + return fees / weight; +} + +/* + Utility function that helps to obtain the percentile of the fees paid by user in tx block + + Expected Range : [0, 0.75] + -> 0% tile : 1 + -> 10% tile : 0.9 + -> 25% tile : 0.75 + -> 50% tile : 0.5 + -> 75% tile : 0.25 + -> 90% tile : 0.1 + -> 100% tile : 0.05 + */ +export function getFeeRatePercentileScore( + timestamp: number, + feeRate: number, + feeRatePercentileHistory: FeeRatePercentile[], +): number { + let percentile: number = getPercentile( + timestamp, + feeRate, + feeRatePercentileHistory, + ); + return 1 - percentile / 100; +} + +function getPercentile( + timestamp: number, + feeRate: number, + feeRatePercentileHistory: FeeRatePercentile[], +): number { + // Find the closest entry by timestamp + let closestBlock: FeeRatePercentile | null = null; + let closestDifference: number = Infinity; + + for (const block of feeRatePercentileHistory) { + const difference = Math.abs(block.timestamp - timestamp); + if (difference <= closestDifference) { + closestDifference = difference; + closestBlock = block; + } + } + if (!closestBlock) { + throw new Error("No fee rate data found"); + } + // Find the closest fee rate percentile + switch (true) { + case feeRate <= closestBlock.avgFee_0: + return 0; + case feeRate <= closestBlock.avgFee_10: + return 10; + case feeRate <= closestBlock.avgFee_25: + return 25; + case feeRate <= closestBlock.avgFee_50: + return 50; + case feeRate <= closestBlock.avgFee_75: + return 75; + case feeRate <= closestBlock.avgFee_90: + return 90; + case feeRate <= closestBlock.avgFee_100: + return 100; + default: + throw new Error("Invalid fee rate"); + } +} diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 154335f4..adf0ab8c 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -1,84 +1,88 @@ -import { feesToAmountRatio, relativeFeesScore, wasteMetric } from "./waste"; -import { Transaction, FeeRatePercentile } from "@caravan/clients"; +import { WasteMetric } from "./waste"; -describe("Waste metric scoring", () => { - it("calculates fee score based on tx fee rate relative to percentile in the block where a set of send tx were mined", () => { - const transactions: Transaction[] = [ - { - vin: [], - vout: [], - txid: "tx1", - size: 0, - weight: 1, - fee: 1, - isSend: true, - amount: 0, - blocktime: 1234, - }, - ]; - const feeRatePercentileHistory: FeeRatePercentile[] = [ - { - avgHeight: 0, - timestamp: 1234, - avgFee_0: 0, - avgFee_10: 0, - avgFee_25: 0.5, - avgFee_50: 1, - avgFee_75: 0, - avgFee_90: 0, - avgFee_100: 0, - }, - ]; - const score: number = relativeFeesScore( - transactions, - feeRatePercentileHistory, - ); - expect(score).toBe(0.5); - }); +const transactions = [ + { + vin: [], // List of inputs + vout: [], // List of outputs + txid: "tx1", // Transaction ID + size: 1, // Size of the transaction + weight: 1, // Weight of the transaction + fee: 1, // Fee paid in the transaction + isSend: true, // Transaction is a send transaction + amount: 10, // Amount spent in the transaction + blocktime: 1234, // Blocktime of the block where the transactions were mined + }, + { + vin: [], + vout: [], + txid: "tx2", + size: 0, + weight: 1, + fee: 1, + isSend: false, + amount: 10, + blocktime: 1234, + }, +]; - it("Fees paid for total amount spent as ratio for a transaction", async () => { - const transaction: Transaction[] = [ - { - vin: [], - vout: [], - txid: "tx1", - size: 0, - weight: 0, - fee: 1, - isSend: true, - amount: 10, - blocktime: 0, - }, - ]; - const ratio: number = feesToAmountRatio(transaction); - expect(ratio).toBe(0.1); - }); +const feeRatePercentileHistory = [ + { + avgHeight: 0, // Height of the block where the transactions were mined + timestamp: 1234, // Blocktime of the block where the transactions were mined + avgFee_0: 0.1, // Lowest fee rate in the block was 0.1 sat/vbyte + avgFee_10: 0.2, // 10th percentile fee rate in the block was 0.2 sat/vbyte + avgFee_25: 0.5, // 25th percentile fee rate in the block was 0.5 sat/vbyte + avgFee_50: 1, // Median fee rate in the block was 1 sat/vbyte + avgFee_75: 1.5, // 75th percentile fee rate in the block was 1.5 sat/vbyte + avgFee_90: 2, // 90th percentile fee rate in the block was 2 sat/vbyte + avgFee_100: 2.5, // Highest fee rate in the block was 2.5 sat/vbyte + }, +]; - it("waste score metric that determines the cost of keeping or spending the UTXO at given point of time", () => { - const transaction: Transaction = { - vin: [], - vout: [], - txid: "tx1", - size: 1000, - weight: 500, - fee: 2, - isSend: true, - amount: 50, - blocktime: 1234, - }; +describe("Waste metric scoring", () => { + const wasteMetric = new WasteMetric(); - const amount = 30; - // Reference on estimatedLongTermFeeRate : https://bitcoincore.reviews/17331#l-164 - const estimatedLongTermFeeRate = 30; + describe("Relative Fees Score (R.F.S)", () => { + it("calculates fee score based on tx fee rate relative to percentile in the block where a set of send tx were mined", () => { + const score: number = wasteMetric.relativeFeesScore( + transactions, + feeRatePercentileHistory, + ); + expect(score).toBe(0.5); + }); + }); - const result = wasteMetric(transaction, amount, estimatedLongTermFeeRate); + describe("Fees to Amount Ratio (F.A.R)", () => { + it("Fees paid over total amount spent as ratio for a 'send' type transaction", async () => { + const ratio: number = wasteMetric.feesToAmountRatio(transactions); + expect(ratio).toBe(0.1); + }); + }); - const expectedWeight = transaction.weight; - const feeRate = transaction.fee / transaction.weight; - const costOfTx = Math.abs(amount - transaction.amount); - const expectedWasteMetric = - expectedWeight * (feeRate - estimatedLongTermFeeRate) + costOfTx; + describe("Spend Waste Amount (S.W.A)", () => { + it("determines the cost of keeping or spending the UTXO at given point of time", () => { + // Input UTXO Set : [0.1 BTC, 0.2 BTC, 0.3 BTC, 0.4 BTC] + // Weight : 30 vbytes + // Current Fee Rate : 10 sat/vbyte + // Input Amount Sum : 10000 sats + // Spend Amount : 8000 sats + // Estimated Long Term Fee Rate : 15 sat/vbyte + const weight = 30; // Estimated weight of the spending transaction + const feeRate = 10; // Current Fee-Rate + const inputAmountSum = 10000; // Sum of all inputs in the spending transaction + const spendAmount = 8000; // Amount spent in the transaction + const estimatedLongTermFeeRate = 15; // Estimated long term fee rate - expect(result).toBe(expectedWasteMetric); + const wasteAmount = wasteMetric.spendWasteAmount( + weight, + feeRate, + inputAmountSum, + spendAmount, + estimatedLongTermFeeRate, + ); + expect(wasteAmount).toBe(1850); + // This number is positive this means that in future if we wait for the fee rate to go down, + // we can save 1850 sats + }); }); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 56b168af..769c15cd 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,205 +1,159 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { utxoSetLengthScore } from "./privacy"; +import { utxoMassFactor } from "./privacy"; import { AddressUtxos } from "./types"; - -/* -Utility function that helps to obtain the fee rate of the transaction - -Expected Range : [0, 0.75] --> Very Poor : [0, 0.15] --> Poor : (0.15, 0.3] --> Moderate : (0.3, 0.45] --> Good : (0.45, 0.6] --> Very Good : (0.6, 0.75] -*/ -function getFeeRateForTransaction(transaction: Transaction): number { - let fees: number = transaction.fee; - let weight: number = transaction.weight; - return fees / weight; -} - -/* -Utility function that helps to obtain the percentile of the fees paid by user in tx block - -Expected Range : [0, 0.75] --> 0% tile : 1 --> 10% tile : 0.9 --> 25% tile : 0.75 --> 50% tile : 0.5 --> 75% tile : 0.25 --> 90% tile : 0.1 --> 100% tile : 0.05 -*/ -function getFeeRatePercentileScore( - timestamp: number, - feeRate: number, - feeRatePercentileHistory: FeeRatePercentile[], -): number { - let percentile: number = getPercentile( - timestamp, - feeRate, - feeRatePercentileHistory, - ); - return 1 - percentile / 100; -} - -function getPercentile( - timestamp: number, - feeRate: number, - feeRatePercentileHistory: FeeRatePercentile[], -): number { - // Find the closest entry by timestamp - let closestBlock: FeeRatePercentile | null = null; - let closestDifference: number = Infinity; - - for (const block of feeRatePercentileHistory) { - const difference = Math.abs(block.timestamp - timestamp); - if (difference <= closestDifference) { - closestDifference = difference; - closestBlock = block; +import { getFeeRateForTransaction, getFeeRatePercentileScore } from "./utils"; + +export class WasteMetric { + /* + Name : + Relative Fees Score (R.F.S) + + Definition : + Comparision of the fees paid by the wallet transactions in a block relative to + the fees paid by other transactions in the same block on the same network. + + Calculation : + We take the percentile value of the fees paid by the user in the block of the transaction. + And then we obtain the mean percentile score for all the transaction done in a wallet. + + Expected Range : [0, 1] + -> Very Poor : [0, 0.2] + -> Poor : (0.2, 0.4] + -> Moderate : (0.4, 0.6] + -> Good : (0.6, 0.8] + -> Very Good : (0.8, 1] + */ + relativeFeesScore( + transactions: Transaction[], + feeRatePercentileHistory: FeeRatePercentile[], + ): number { + let sumRFS: number = 0; + let numberOfSendTx: number = 0; + for (const tx of transactions) { + if (tx.isSend === true) { + numberOfSendTx++; + let feeRate: number = getFeeRateForTransaction(tx); + let RFS: number = getFeeRatePercentileScore( + tx.blocktime, + feeRate, + feeRatePercentileHistory, + ); + sumRFS += RFS; + } } + return sumRFS / numberOfSendTx; } - if (!closestBlock) { - throw new Error("No fee rate data found"); - } - // Find the closest fee rate percentile - switch (true) { - case feeRate <= closestBlock.avgFee_0: - return 0; - case feeRate <= closestBlock.avgFee_10: - return 10; - case feeRate <= closestBlock.avgFee_25: - return 25; - case feeRate <= closestBlock.avgFee_50: - return 50; - case feeRate <= closestBlock.avgFee_75: - return 75; - case feeRate <= closestBlock.avgFee_90: - return 90; - case feeRate <= closestBlock.avgFee_100: - return 100; - default: - throw new Error("Invalid fee rate"); - } -} -/* -R.F.S can be associated with all the transactions and we can give a measure -if any transaction was done at expensive fees or nominal fees. - -This can be done by calculating the percentile of the fees paid by the user -in the block of the transaction. - -Expected Range : [0, 1] --> Very Poor : [0, 0.2] --> Poor : (0.2, 0.4] --> Moderate : (0.4, 0.6] --> Good : (0.6, 0.8] --> Very Good : (0.8, 1] -*/ -export function relativeFeesScore( - transactions: Transaction[], - feeRatePercentileHistory: FeeRatePercentile[], -): number { - let sumRFS: number = 0; - let numberOfSendTx: number = 0; - for (const tx of transactions) { - if (tx.isSend === true) { - numberOfSendTx++; - let feeRate: number = getFeeRateForTransaction(tx); - let RFS: number = getFeeRatePercentileScore( - tx.blocktime, - feeRate, - feeRatePercentileHistory, - ); - sumRFS += RFS; - } + /* + Name : + Fees To Amount Ration (F.A.R) + + Definition : + Ratio of the fees paid by the wallet transactions to the amount spent in the transaction. + + Calculation : + We can compare this ratio against the fiat charges for cross-border transactions. + Mastercard charges 0.6% cross-border fee for international transactions in US dollars, + but if the transaction is in any other currency the fee goes up to 1%. + Source : https://www.clearlypayments.com/blog/what-are-cross-border-fees-in-credit-card-payments/ + + Expected Range : [0, 1] + -> Very Poor : [1, 0.01] // More than 1% amount paid as fees. In ratio 1% is 0.01 and so on for other range + -> Poor : (0.01, 0.0075] + -> Moderate : (0.0075, 0.006] + -> Good : (0.006, 0.001] + -> Very Good : (0.001, 0) + */ + feesToAmountRatio(transactions: Transaction[]): number { + let sumFeesToAmountRatio: number = 0; + let numberOfSendTx: number = 0; + transactions.forEach((tx: Transaction) => { + if (tx.isSend === true) { + sumFeesToAmountRatio += tx.fee / tx.amount; + numberOfSendTx++; + } + }); + return sumFeesToAmountRatio / numberOfSendTx; } - return sumRFS / numberOfSendTx; -} - -/* -The measure of how much the wallet is affected by fees is determined by the ratio of the amount paid to the fees incurred. - -Mastercard charges 0.6% cross-border fee for international transactions in US dollars, -but if the transaction is in any other currency the fee goes up to 1%. -Source : https://www.clearlypayments.com/blog/what-are-cross-border-fees-in-credit-card-payments/ - -This ratio is a measure of transaction fees compared with market rates for fiat processing fees. -*/ -export function feesToAmountRatio(transactions: Transaction[]): number { - let sumFeesToAmountRatio: number = 0; - let numberOfSendTx: number = 0; - transactions.forEach((tx: Transaction) => { - if (tx.isSend === true) { - sumFeesToAmountRatio += tx.fee / tx.amount; - numberOfSendTx++; - } - }); - return sumFeesToAmountRatio / numberOfSendTx; -} -/* -Consider the waste score of the transaction which can inform whether or not it is economical to spend a -particular output now (assuming fees are currently high) or wait to consolidate it later when fees are low. - -waste score = consolidation factor + cost of transaction -waste score = weight (fee rate - estimatedLongTermFeeRate) + change + excess - -// Reference on estimatedLongTermFeeRate : https://bitcoincore.reviews/17331#l-164 -// It is upper bound for spending the UTXO in the future - -weight: Transaction weight units -feeRate: the transaction's target fee rate -estimatedLongTermFeeRate: the long-term fee rate estimate which the wallet might need to pay to redeem remaining UTXOs -change: the cost of creating and spending a change output -excess: the amount by which we exceed our selection target when creating a changeless transaction, -mutually exclusive with cost of change - -“excess” is if we don't make a change output and instead add the difference to the fees. -If there is a change in output then the excess should be 0. -“change” includes the fees paid on this transaction's change output plus the fees that -will need to be paid to spend it later. Thus the quantity cost of transaction is always a positive number. - -Depending on the fee rate in the long term, the consolidation factor can either be positive or negative. - fee rate (current) < estimatedLongTermFeeRate (long-term fee rate) –-> Consolidate now (-ve) - fee rate (current) > estimatedLongTermFeeRate (long-term fee rate) –-> Wait for later when fee rate go low (+ve) -*/ -export function wasteMetric( - transaction: Transaction, // Amount that UTXO holds - amount: number, // Amount to be spent in the transaction - estimatedLongTermFeeRate: number, // Long term estimated fee-rate -): number { - let weight: number = transaction.weight; - let feeRate: number = getFeeRateForTransaction(transaction); - let costOfTx: number = Math.abs(amount - transaction.amount); - return weight * (feeRate - estimatedLongTermFeeRate) + costOfTx; -} + /* + Name : + Spend Waste Score (S.W.S) + + Definition : + A score that indicates whether it is economical to spend a particular output now + or wait to consolidate it later when fees could be low. + + Important Terms: + - Weight: + Transaction weight units + - Fee Rate: + The transaction's target fee rate (current fee-rate of the network) + - Estimated Long Term Fee Rate: + The long-term fee rate estimate which the wallet might need to pay + to redeem remaining UTXOs. + Reference : https://bitcoincore.reviews/17331#l-164 + It is the upper bound for spending the UTXO in the future. + - Change: + The cost of creating and spending a change output. It includes the fees paid + on this transaction's change output plus the fees that will need to be paid + to spend it later. + - Excess: + The amount by which we exceed our selection target when creating a changeless transaction, + mutually exclusive with cost of change. It is extra fees paid if we don't make a change output + and instead add the difference to the fees. + - Input Amount : + Sum of amount for each coin in input of the transaction + - Spend Amount : + Exact amount wanted to be spent in the transaction. + + Calculation : + spend waste score = consolidation factor + cost of transaction + spend waste score = weight (fee rate - estimatedLongTermFeeRate) + change + excess + + Observation : + Depending on the fee rate in the long term, the consolidation factor can either be positive or negative. + fee rate (current) < estimatedLongTermFeeRate (long-term fee rate) –-> Consolidate now (-ve) + fee rate (current) > estimatedLongTermFeeRate (long-term fee rate) –-> Wait for later when fee rate go low (+ve) + + */ + spendWasteAmount( + weight: number, // Estimated weight of the transaction + feeRate: number, // Current Fee rate for the transaction + inputAmountSum: number, // Sum of amount for each coin in input of the transaction + spendAmount: number, // Exact Amount wanted to be spent in the transaction + estimatedLongTermFeeRate: number, // Long term estimated fee-rate + ): number { + let costOfTx: number = Math.abs(spendAmount - inputAmountSum); + return weight * (feeRate - estimatedLongTermFeeRate) + costOfTx; + } -/* -35% Weightage of fees score depends on Percentile of fees paid -35% Weightage of fees score depends fees paid with respect to amount spend -30% Weightage of fees score depends on the number of UTXOs present in the wallet. - -Q : What role does UMF plays in the fees score? -Assume the wallet is being consolidated, Thus number of UTXO will decrease and thus -UMF (UTXO Mass Factor) will increase and this justifies that, consolidation -increases the fees health since you don’t overpay them in long run. - -Expected Range : [0, 1] --> Very Poor : [0, 0.2] --> Poor : (0.2, 0.4] --> Moderate : (0.4, 0.6] --> Good : (0.6, 0.8] --> Very Good : (0.8, 1] -*/ -export async function feesScore( - transactions: Transaction[], - utxos: AddressUtxos, - feeRatePercentileHistory: FeeRatePercentile[], -): Promise { - let RFS: number = relativeFeesScore(transactions, feeRatePercentileHistory); - let FAR: number = feesToAmountRatio(transactions); - let UMS: number = utxoSetLengthScore(utxos); - return 0.35 * RFS + 0.35 * FAR + 0.3 * UMS; + /* + Name : + Weighted Waste Score (W.W.S) + + Definition : + A score that indicates the overall waste of the wallet based on the relative fees score, + fees to amount ratio and the UTXO mass factor. + + Calculation : + weighted waste score = 0.35 * RFS + 0.35 * FAR + 0.3 * UMF + + Expected Range : [0, 1] + -> Very Poor : [0, 0.2] + -> Poor : (0.2, 0.4] + -> Moderate : (0.4, 0.6] + -> Good : (0.6, 0.8] + -> Very Good : (0.8, 1] + */ + weightedWasteScore( + transactions: Transaction[], + utxos: AddressUtxos, + feeRatePercentileHistory: FeeRatePercentile[], + ): number { + let RFS = this.relativeFeesScore(transactions, feeRatePercentileHistory); + let FAR = this.feesToAmountRatio(transactions); + let UMF = utxoMassFactor(utxos); + return 0.35 * RFS + 0.35 * FAR + 0.3 * UMF; + } } From 88920977bf6a4148c35a8ed6a5d0f6568183a80b Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 12 Aug 2024 02:30:36 +0530 Subject: [PATCH 47/92] Remove deps from network client Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.ts | 60 +++++++++++++------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 83057394..9e9ccf19 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,7 +1,17 @@ -import { BlockchainClient, Transaction } from "@caravan/clients"; +import { Transaction } from "@caravan/clients"; import { AddressUtxos } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; +export class PrivacyMetric { + // TODO : Will implement this real quick + /* + Name : + Definition : + Calculation : + Expected Range : + */ +} + /* The methodology for calculating a privacy score (p_score) for Bitcoin transactions based on the number of inputs and outputs is the primary point to define wallet health for privacy. @@ -75,22 +85,20 @@ Expected Range : [0, 0.75] -> Good : (0.45, 0.6] -> Very Good : (0.6, 0.75] */ -export async function getMeanTopologyScore( +export function getMeanTopologyScore( transactions: Transaction[], - client: BlockchainClient, -): Promise { +): number { let privacyScore = 0; for (let tx of transactions) { - let topologyScore = await getTopologyScore(tx, client); + let topologyScore = getTopologyScore(tx); privacyScore += topologyScore; } return privacyScore / transactions.length; } -export async function getTopologyScore( - transaction: Transaction, - client: BlockchainClient, -): Promise { +export function getTopologyScore( + transaction: Transaction +): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -109,7 +117,7 @@ export async function getTopologyScore( } for (let output of transaction.vout) { let address = output.scriptPubkeyAddress; - let isResued = await isReusedAddress(address, client); + let isResued = isReusedAddress(address); if (isResued === true) { return score; } @@ -128,10 +136,9 @@ Expected Range : [0,1] -> Good : [0.2, 0.4) -> Very Good : [0 ,0.2) */ -export async function addressReuseFactor( +export function addressReuseFactor( utxos: AddressUtxos, - client: BlockchainClient, -): Promise { +): number { let reusedAmount: number = 0; let totalAmount: number = 0; @@ -139,7 +146,7 @@ export async function addressReuseFactor( const addressUtxos = utxos[address]; for (const utxo of addressUtxos) { totalAmount += utxo.value; - let isReused = await isReusedAddress(address, client); + let isReused = isReusedAddress(address); if (isReused) { reusedAmount += utxo.value; } @@ -148,18 +155,10 @@ export async function addressReuseFactor( return reusedAmount / totalAmount; } -async function isReusedAddress( - address: string, - client: BlockchainClient, -): Promise { - let txs: Transaction[] = await client.getAddressTransactions(address); - let countReceive = 0; - for (const tx of txs) { - if (!tx.isSend) { - countReceive++; - } - if (countReceive > 1) return true; - } +function isReusedAddress( + address: string +): boolean { + // TODO : Implement a function to check if the address is reused return false; } @@ -300,17 +299,16 @@ Expected Range : [0, 1] -> Good : (0.6, 0.8] -> Very Good : (0.8, 1] */ -export async function getWalletPrivacyScore( +export function getWalletPrivacyScore( transactions: Transaction[], utxos: AddressUtxos, walletAddressType: MultisigAddressType, - client: BlockchainClient, network: Network, -): Promise { - let privacyScore = await getMeanTopologyScore(transactions, client); +): number { + let privacyScore = getMeanTopologyScore(transactions); // Adjusting the privacy score based on the address reuse factor - let addressReusedFactor = await addressReuseFactor(utxos, client); + let addressReusedFactor = addressReuseFactor(utxos); privacyScore = privacyScore * (1 - 0.5 * addressReusedFactor) + 0.1 * (1 - addressReusedFactor); From ccad6e7cc89866a3934d2df83f341c98cf142db4 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 12 Aug 2024 20:30:42 +0530 Subject: [PATCH 48/92] Adding PrivacyMetric class Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 894 ++++++++++---------- packages/caravan-health/src/privacy.ts | 509 ++++++----- packages/caravan-health/src/waste.ts | 4 +- 3 files changed, 723 insertions(+), 684 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index bcd5586b..90719bf4 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,460 +1,460 @@ -import { - getTopologyScore, - addressReuseFactor, - addressTypeFactor, - utxoSpreadFactor, - utxoMassFactor, - utxoValueDispersionFactor, - getWalletPrivacyScore, - getMeanTopologyScore, -} from "./privacy"; -import { BlockchainClient, Transaction } from "@caravan/clients"; -import { AddressUtxos } from "./types"; -import { MultisigAddressType, Network } from "@caravan/bitcoin"; +// import { +// getTopologyScore, +// addressReuseFactor, +// addressTypeFactor, +// utxoSpreadFactor, +// utxoMassFactor, +// utxoValueDispersionFactor, +// getWalletPrivacyScore, +// getMeanTopologyScore, +// } from "./privacy"; +// import { BlockchainClient, Transaction } from "@caravan/clients"; +// import { AddressUtxos } from "./types"; +// import { MultisigAddressType, Network } from "@caravan/bitcoin"; -describe("Privacy Score Metrics", () => { - let mockClient: BlockchainClient; - let mockTransactionsAddressNotReused: Transaction[] = [ - { - vin: [ - { - prevTxId: "abcd", - vout: 0, - sequence: 0, - }, - ], - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - ]; - let mockTransactionsAddressReused: Transaction[] = [ - { - vin: [ - { - prevTxId: "abcd", - vout: 0, - sequence: 0, - }, - ], - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - { - vin: [ - { - prevTxId: "abcd", - vout: 0, - sequence: 0, - }, - ], - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - ]; +// describe("Privacy Score Metrics", () => { +// let mockClient: BlockchainClient; +// let mockTransactionsAddressNotReused: Transaction[] = [ +// { +// vin: [ +// { +// prevTxId: "abcd", +// vout: 0, +// sequence: 0, +// }, +// ], +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// ]; +// let mockTransactionsAddressReused: Transaction[] = [ +// { +// vin: [ +// { +// prevTxId: "abcd", +// vout: 0, +// sequence: 0, +// }, +// ], +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// { +// vin: [ +// { +// prevTxId: "abcd", +// vout: 0, +// sequence: 0, +// }, +// ], +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// ]; - it("Perfect Spend Transaction without reused address for calculating transaction topology score", async () => { - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest - .fn() - .mockResolvedValue(mockTransactionsAddressNotReused), - } as unknown as BlockchainClient; - const transaction: Transaction = { - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], // 1 Input - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output - txid: "123", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }; - const score: number = await getTopologyScore(transaction, mockClient); - expect(score).toBe(0.75); - }); +// it("Perfect Spend Transaction without reused address for calculating transaction topology score", async () => { +// mockClient = { +// getAddressStatus: jest.fn(), +// getAddressTransactions: jest +// .fn() +// .mockResolvedValue(mockTransactionsAddressNotReused), +// } as unknown as BlockchainClient; +// const transaction: Transaction = { +// vin: [ +// { +// prevTxId: "input1", +// vout: 0, +// sequence: 0, +// }, +// ], // 1 Input +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output +// txid: "123", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }; +// const score: number = await getTopologyScore(transaction, mockClient); +// expect(score).toBe(0.75); +// }); - it("Perfect Spend Transaction with reused address for calculating transaction topology score", async () => { - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest - .fn() - .mockResolvedValue(mockTransactionsAddressReused), - } as unknown as BlockchainClient; - const transaction: Transaction = { - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], // 1 Input - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output - txid: "123", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }; - const score: number = await getTopologyScore(transaction, mockClient); - expect(score).toBe(0.5); - }); +// it("Perfect Spend Transaction with reused address for calculating transaction topology score", async () => { +// mockClient = { +// getAddressStatus: jest.fn(), +// getAddressTransactions: jest +// .fn() +// .mockResolvedValue(mockTransactionsAddressReused), +// } as unknown as BlockchainClient; +// const transaction: Transaction = { +// vin: [ +// { +// prevTxId: "input1", +// vout: 0, +// sequence: 0, +// }, +// ], // 1 Input +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output +// txid: "123", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }; +// const score: number = await getTopologyScore(transaction, mockClient); +// expect(score).toBe(0.5); +// }); - it("Calculating mean transaction topology score for multiple trnasactions", async () => { - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest - .fn() - .mockResolvedValue(mockTransactionsAddressNotReused), - } as unknown as BlockchainClient; - const transactions: Transaction[] = [ - { - // Perfect Spend (No Reused Address) - 0.75 - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], // 1 Input - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output - txid: "123", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - { - // Simple Spend (No Reused Address) - 0.66 - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], // 1 Input - vout: [ - { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, - ], // 2 Outputs - txid: "123", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - ]; - const score: number = await getMeanTopologyScore(transactions, mockClient); - expect(score).toBeCloseTo(0.708); - }); +// it("Calculating mean transaction topology score for multiple trnasactions", async () => { +// mockClient = { +// getAddressStatus: jest.fn(), +// getAddressTransactions: jest +// .fn() +// .mockResolvedValue(mockTransactionsAddressNotReused), +// } as unknown as BlockchainClient; +// const transactions: Transaction[] = [ +// { +// // Perfect Spend (No Reused Address) - 0.75 +// vin: [ +// { +// prevTxId: "input1", +// vout: 0, +// sequence: 0, +// }, +// ], // 1 Input +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output +// txid: "123", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// { +// // Simple Spend (No Reused Address) - 0.66 +// vin: [ +// { +// prevTxId: "input1", +// vout: 0, +// sequence: 0, +// }, +// ], // 1 Input +// vout: [ +// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, +// ], // 2 Outputs +// txid: "123", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// ]; +// const score: number = await getMeanTopologyScore(transactions, mockClient); +// expect(score).toBeCloseTo(0.708); +// }); - it("Address Reuse Factor accounts for Unspent coins that are on reused address with respect to total amount in wallet", async () => { - const utxos: AddressUtxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx3", - vout: 0, - value: 3, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; +// it("Address Reuse Factor accounts for Unspent coins that are on reused address with respect to total amount in wallet", async () => { +// const utxos: AddressUtxos = { +// address1: [ +// { +// txid: "tx1", +// vout: 0, +// value: 1, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx2", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// address2: [ +// { +// txid: "tx3", +// vout: 0, +// value: 3, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// }; - // No address was reused - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest - .fn() - .mockResolvedValue(mockTransactionsAddressNotReused), - } as unknown as BlockchainClient; - const factor: number = await addressReuseFactor(utxos, mockClient); - expect(factor).toBe(0); +// // No address was reused +// mockClient = { +// getAddressStatus: jest.fn(), +// getAddressTransactions: jest +// .fn() +// .mockResolvedValue(mockTransactionsAddressNotReused), +// } as unknown as BlockchainClient; +// const factor: number = await addressReuseFactor(utxos, mockClient); +// expect(factor).toBe(0); - // All addresses were reused - mockClient = { - getAddressStatus: jest.fn(), - getAddressTransactions: jest - .fn() - .mockResolvedValue(mockTransactionsAddressReused), - } as unknown as BlockchainClient; - const factor2: number = await addressReuseFactor(utxos, mockClient); - expect(factor2).toBe(1); - }); +// // All addresses were reused +// mockClient = { +// getAddressStatus: jest.fn(), +// getAddressTransactions: jest +// .fn() +// .mockResolvedValue(mockTransactionsAddressReused), +// } as unknown as BlockchainClient; +// const factor2: number = await addressReuseFactor(utxos, mockClient); +// expect(factor2).toBe(1); +// }); - it("P2PKH wallet address type being checked for all transactions", () => { - const transactions: Transaction[] = [ - { - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], // Address starting with 1 - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - ]; - const walletAddressType: MultisigAddressType = "P2PKH"; - const network: Network = Network["MAINNET"]; - const factor: number = addressTypeFactor( - transactions, - walletAddressType, - network, - ); - expect(factor).toBe(1); - }); +// it("P2PKH wallet address type being checked for all transactions", () => { +// const transactions: Transaction[] = [ +// { +// vin: [ +// { +// prevTxId: "input1", +// vout: 0, +// sequence: 0, +// }, +// ], +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], // Address starting with 1 +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// ]; +// const walletAddressType: MultisigAddressType = "P2PKH"; +// const network: Network = Network["MAINNET"]; +// const factor: number = addressTypeFactor( +// transactions, +// walletAddressType, +// network, +// ); +// expect(factor).toBe(1); +// }); - it("UTXOs spread factor across multiple addresses (assess for how similar the amount values are for each UTXO)", () => { - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - // Value 1 and 2 is so close to each other so the scoring is bad. - const factor: number = utxoSpreadFactor(utxos); - expect(factor).toBe(1 / 3); +// it("UTXOs spread factor across multiple addresses (assess for how similar the amount values are for each UTXO)", () => { +// const utxos = { +// address1: [ +// { +// txid: "tx1", +// vout: 0, +// value: 1, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// address2: [ +// { +// txid: "tx2", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// }; +// // Value 1 and 2 is so close to each other so the scoring is bad. +// const factor: number = utxoSpreadFactor(utxos); +// expect(factor).toBe(1 / 3); - const utxos2 = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx2", - vout: 0, - value: 200, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - // Value 1 and 200 is so far from each other so the scoring is good. - const factor2: number = utxoSpreadFactor(utxos2); - expect(factor2).toBeCloseTo(0.99); - }); +// const utxos2 = { +// address1: [ +// { +// txid: "tx1", +// vout: 0, +// value: 1, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// address2: [ +// { +// txid: "tx2", +// vout: 0, +// value: 200, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// }; +// // Value 1 and 200 is so far from each other so the scoring is good. +// const factor2: number = utxoSpreadFactor(utxos2); +// expect(factor2).toBeCloseTo(0.99); +// }); - it("Gives a score on the basis of number of UTXOs present in the wallet", () => { - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], // 1 UTXO only for address1 - address2: [ - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx3", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx4", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx5", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx6", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx7", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], // 6 UTXOs for address 2 - }; - // 7 UTXOs in total - which will give 0.75 as score - const score: number = utxoMassFactor(utxos); - expect(score).toBe(0.75); - }); +// it("Gives a score on the basis of number of UTXOs present in the wallet", () => { +// const utxos = { +// address1: [ +// { +// txid: "tx1", +// vout: 0, +// value: 1, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], // 1 UTXO only for address1 +// address2: [ +// { +// txid: "tx2", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx3", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx4", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx5", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx6", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx7", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], // 6 UTXOs for address 2 +// }; +// // 7 UTXOs in total - which will give 0.75 as score +// const score: number = utxoMassFactor(utxos); +// expect(score).toBe(0.75); +// }); - it("UTXO value dispersion accounts for number of coins in the wallet and how dispersed they are in amount values", () => { - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], // 1 UTXO only for address1 - address2: [ - { - txid: "tx2", - vout: 0, - value: 0.02, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx3", - vout: 0, - value: 0.2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx4", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx5", - vout: 0, - value: 20, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx6", - vout: 0, - value: 200, - status: { confirmed: true, block_time: 0 }, - }, - { - txid: "tx7", - vout: 0, - value: 2000, - status: { confirmed: true, block_time: 0 }, - }, - ], // 6 UTXOs for address 2 - }; - const factor: number = utxoValueDispersionFactor(utxos); - expect(factor).toBeCloseTo(0.112); - }); +// it("UTXO value dispersion accounts for number of coins in the wallet and how dispersed they are in amount values", () => { +// const utxos = { +// address1: [ +// { +// txid: "tx1", +// vout: 0, +// value: 1, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], // 1 UTXO only for address1 +// address2: [ +// { +// txid: "tx2", +// vout: 0, +// value: 0.02, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx3", +// vout: 0, +// value: 0.2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx4", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx5", +// vout: 0, +// value: 20, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx6", +// vout: 0, +// value: 200, +// status: { confirmed: true, block_time: 0 }, +// }, +// { +// txid: "tx7", +// vout: 0, +// value: 2000, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], // 6 UTXOs for address 2 +// }; +// const factor: number = utxoValueDispersionFactor(utxos); +// expect(factor).toBeCloseTo(0.112); +// }); - it("Overall Privacy Score taking into consideration all parameters for UTXO and Transaction History", async () => { - const transactions: Transaction[] = [ - { - vin: [ - { - prevTxId: "input1", - vout: 0, - sequence: 0, - }, - ], - vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], - txid: "", - size: 0, - weight: 0, - fee: 0, - isSend: false, - amount: 0, - blocktime: 0, - }, - ]; - const utxos = { - address1: [ - { - txid: "tx1", - vout: 0, - value: 1, - status: { confirmed: true, block_time: 0 }, - }, - ], - address2: [ - { - txid: "tx2", - vout: 0, - value: 2, - status: { confirmed: true, block_time: 0 }, - }, - ], - }; - const walletAddressType: MultisigAddressType = "P2PKH"; - const network: Network = Network["MAINNET"]; - const score: number = await getWalletPrivacyScore( - transactions, - utxos, - walletAddressType, - mockClient, - network, - ); - expect(score).toBeCloseTo(0.005); - }); -}); +// it("Overall Privacy Score taking into consideration all parameters for UTXO and Transaction History", async () => { +// const transactions: Transaction[] = [ +// { +// vin: [ +// { +// prevTxId: "input1", +// vout: 0, +// sequence: 0, +// }, +// ], +// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], +// txid: "", +// size: 0, +// weight: 0, +// fee: 0, +// isSend: false, +// amount: 0, +// blocktime: 0, +// }, +// ]; +// const utxos = { +// address1: [ +// { +// txid: "tx1", +// vout: 0, +// value: 1, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// address2: [ +// { +// txid: "tx2", +// vout: 0, +// value: 2, +// status: { confirmed: true, block_time: 0 }, +// }, +// ], +// }; +// const walletAddressType: MultisigAddressType = "P2PKH"; +// const network: Network = Network["MAINNET"]; +// const score: number = await getWalletPrivacyScore( +// transactions, +// utxos, +// walletAddressType, +// mockClient, +// network, +// ); +// expect(score).toBeCloseTo(0.005); +// }); +// }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 9e9ccf19..feb921e1 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -3,13 +3,275 @@ import { AddressUtxos } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; export class PrivacyMetric { - // TODO : Will implement this real quick /* - Name : + Name : Topology Score + + Definition : + The score is calculated based on the number of inputs and outputs which + influence the topology type of the transaction. + + Calculation : + We have 5 categories of transaction type each with their own impact on privacy score + - Perfect Spend (1 input, 1 output) + - Simple Spend (1 input, 2 outputs) + - UTXO Fragmentation (1 input, more than 2 standard outputs) + - Consolidation (more than 1 input, 1 output) + - CoinJoin or Mixing (more than 1 input, more than 1 output) + */ + getTopologyScore(transaction: Transaction): number { + const numberOfInputs: number = transaction.vin.length; + const numberOfOutputs: number = transaction.vout.length; + + const spendType: SpendType = determineSpendType( + numberOfInputs, + numberOfOutputs, + ); + const score: number = getSpendTypeScore( + spendType, + numberOfInputs, + numberOfOutputs, + ); + + if (spendType === SpendType.Consolidation) { + return score; + } + for (let output of transaction.vout) { + let address = output.scriptPubkeyAddress; + let isResued = isReusedAddress(address); + if (isResued === true) { + return score; + } + } + return score * DENIABILITY_FACTOR; + } + + /* + Name : Mean Transaction Topology Privacy Score (MTPS) + + Definition : + The mean topology is evaluated for entire wallet history based on + the tx toplogy score for each transaction. It signifies how well the + transactions were performed to maintain privacy. + + Calculation : + The mean topology score is calculated by evaluating the topology score for each transaction. + + Expected Range : [0, 0.75] + -> Very Poor : [0, 0.15] + -> Poor : (0.15, 0.3] + -> Moderate : (0.3, 0.45] + -> Good : (0.45, 0.6] + -> Very Good : (0.6, 0.75) + */ + getMeanTopologyScore(transactions: Transaction[]): number { + let privacyScore = 0; + for (let tx of transactions) { + let topologyScore = this.getTopologyScore(tx); + privacyScore += topologyScore; + } + return privacyScore / transactions.length; + } + + /* + Name : Address Reuse Factor (ARF) + + Definition : + The address reuse factor is evaluates the amount being held by reused addresses with respect + to the total amount. It signifies the privacy health of the wallet based on address reuse. + + Calculation : + The factor is calculated by summing the amount held by reused addresses and dividing it + by the total amount. + + Expected Range : [0,1] + -> Very Poor : (0.8, 1] + -> Poor : [0.6, 0.8) + -> Moderate : [0.4, 0.6) + -> Good : [0.2, 0.4) + -> Very Good : [0 ,0.2) + */ + addressReuseFactor(utxos: AddressUtxos): number { + let reusedAmount: number = 0; + let totalAmount: number = 0; + + for (const address in utxos) { + const addressUtxos = utxos[address]; + for (const utxo of addressUtxos) { + totalAmount += utxo.value; + let isReused = isReusedAddress(address); + if (isReused) { + reusedAmount += utxo.value; + } + } + } + return reusedAmount / totalAmount; + } + + /* + Name : Address Type Factor (ATF) + + Definition : + The address type factor evaluates the address type distribution of the wallet. + It signifies the privacy health of the wallet based on the address types used. + + Calculation : + It is calculated as + ATF= 1/(same+1) + where "same" denotes the number of output address types matching the input address type. + A higher "same" value results in a lower ATF, indicating reduced privacy due to less variety in address types. + If all are same or all are different address type then there will be no change in the privacy score. + + Expected Range : (0,1] + -> Very Poor : (0, 0.1] + -> Poor : [0.1, 0.3) + -> Moderate : [0.3, 0.4) + -> Good : [0.4, 0.5) + -> Very Good : [0.5 ,1] + + */ + addressTypeFactor( + transactions: Transaction[], + walletAddressType: MultisigAddressType, + network: Network, + ): number { + const addressCounts: Record = { + P2WSH: 0, + P2SH: 0, + P2PKH: 0, + P2TR: 0, + UNKNOWN: 0, + "P2SH-P2WSH": 0, + }; + + transactions.forEach((tx) => { + tx.vout.forEach((output) => { + const addressType = getAddressType(output.scriptPubkeyAddress, network); + addressCounts[addressType]++; + }); + }); + + const totalAddresses = Object.values(addressCounts).reduce( + (a, b) => a + b, + 0, + ); + const walletTypeCount = addressCounts[walletAddressType]; + + if (walletTypeCount === 0 || totalAddresses === walletTypeCount) { + return 1; + } + return 1 / (walletTypeCount + 1); + } + + /* + Name : UTXO Spread Factor + + Definition : + The spread factor using standard deviation helps in assessing the dispersion of UTXO values. + In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries + to link transactions and deduce the ownership and spending patterns of users. + + Calculation : + The spread factor is calculated by evaluating the standard deviation of UTXO values. + It is calculated as the standard deviation divided by the sum of the standard deviation with 1. + + Expected Range : [0,1) + -> Very Poor : (0, 0.2] + -> Poor : [0.2, 0.4) + -> Moderate : [0.4, 0.6) + -> Good : [0.6, 0.8) + -> Very Good : [0.8 ,1] + */ + utxoSpreadFactor(utxos: AddressUtxos): number { + const amounts: number[] = []; + for (const address in utxos) { + const addressUtxos = utxos[address]; + addressUtxos.forEach((utxo) => { + amounts.push(utxo.value); + }); + } + + const mean: number = + amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; + const variance: number = + amounts.reduce((sum, amount) => sum + Math.pow(amount - mean, 2), 0) / + amounts.length; + const stdDev: number = Math.sqrt(variance); + return stdDev / (stdDev + 1); + } + + /* + Name : UTXO Mass Factor + + Calculation : + The mass factor is calculated based on the number of UTXOs in the set. + + Expected Range : [0,1] + - 0 for UTXO set length >= 50 + - 0.25 for UTXO set length >= 25 and <= 49 + - 0.5 for UTXO set length >= 15 and <= 24 + - 0.75 for UTXO set length >= 5 and <= 14 + - 1 for UTXO set length < 5 + */ + utxoMassFactor(utxos: AddressUtxos): number { + return utxoSetLengthMass(utxos); + } + + /* + Name : UTXO Value Dispersion Factor + + Definition : + The UTXO value dispersion factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. + It signifies the combined effect of how much variance is there in the UTXO Set values is and + how many number of UTXOs are there. + + Calculation : + The U.V.D.F is calculated as a combination of UTXO Spread Factor and UTXO Set Length Weight. + It is calculated as (USF + UMF) * 0.15 - 0.15. + + Expected Range : [-0.15,0.15] + -> Very Poor : [-0.15, -0.1] + -> Poor : (-0.1, -0.075] + -> Moderate : (-0.075, 0) + -> Good : (0, 0.075] + -> Very Good : (0.075, 0.15] + */ + utxoValueDispersionFactor(utxos: AddressUtxos): number { + let UMF: number = this.utxoMassFactor(utxos); + let USF: number = this.utxoSpreadFactor(utxos); + return (USF + UMF) * 0.15 - 0.15; + } + + /* + Name : Weighted Privacy Score + Definition : + The weighted privacy score is a combination of all the factors calculated above. + It signifies the overall privacy health of the wallet based on the address reuse, + address types and UTXO set fingerprints etc. + Calculation : - Expected Range : + The weighted privacy score is calculated by + WPS = (MTPS * (1 - 0.5 * ARF) + 0.1 * (1 - ARF)) * (1 - ATF) + 0.1 * UVDF + + */ + getWalletPrivacyScore( + transactions: Transaction[], + utxos: AddressUtxos, + walletAddressType: MultisigAddressType, + network: Network, + ): number { + let meanTopologyScore = this.getMeanTopologyScore(transactions); + let ARF = this.addressReuseFactor(utxos); + let ATF = this.addressTypeFactor(transactions, walletAddressType, network); + let UVDF = this.utxoValueDispersionFactor(utxos); + + let WPS: number = + (meanTopologyScore * (1 - 0.5 * ARF) + 0.1 * (1 - ARF)) * (1 - ATF) + + 0.1 * UVDF; + + return WPS; + } } /* @@ -75,251 +337,28 @@ export function getSpendTypeScore( } } -/* -The transaction topology score evaluates privacy metrics based on the number of inputs and outputs. - -Expected Range : [0, 0.75] --> Very Poor : [0, 0.15] --> Poor : (0.15, 0.3] --> Moderate : (0.3, 0.45] --> Good : (0.45, 0.6] --> Very Good : (0.6, 0.75] -*/ -export function getMeanTopologyScore( - transactions: Transaction[], -): number { - let privacyScore = 0; - for (let tx of transactions) { - let topologyScore = getTopologyScore(tx); - privacyScore += topologyScore; - } - return privacyScore / transactions.length; -} - -export function getTopologyScore( - transaction: Transaction -): number { - const numberOfInputs: number = transaction.vin.length; - const numberOfOutputs: number = transaction.vout.length; - - const spendType: SpendType = determineSpendType( - numberOfInputs, - numberOfOutputs, - ); - const score: number = getSpendTypeScore( - spendType, - numberOfInputs, - numberOfOutputs, - ); - - if (spendType === SpendType.Consolidation) { - return score; - } - for (let output of transaction.vout) { - let address = output.scriptPubkeyAddress; - let isResued = isReusedAddress(address); - if (isResued === true) { - return score; - } - } - return score * DENIABILITY_FACTOR; -} - -/* -In order to score for address reuse we can check the amount being held by reused addresses -with respect to the total amount - -Expected Range : [0,1] --> Very Poor : (0.8, 1] --> Poor : [0.6, 0.8) --> Moderate : [0.4, 0.6) --> Good : [0.2, 0.4) --> Very Good : [0 ,0.2) -*/ -export function addressReuseFactor( - utxos: AddressUtxos, -): number { - let reusedAmount: number = 0; - let totalAmount: number = 0; - - for (const address in utxos) { - const addressUtxos = utxos[address]; - for (const utxo of addressUtxos) { - totalAmount += utxo.value; - let isReused = isReusedAddress(address); - if (isReused) { - reusedAmount += utxo.value; - } - } - } - return reusedAmount / totalAmount; -} - -function isReusedAddress( - address: string -): boolean { +function isReusedAddress(address: string): boolean { // TODO : Implement a function to check if the address is reused return false; } -/* -If we are making payment to other wallet types then the privacy score should decrease because -the change received will be to an address type matching our wallet and it will lead to a deduction that -we still own that amount. - -Expected Range : (0,1] --> Very Poor : (0, 0.1] --> Poor : [0.1, 0.3) --> Moderate : [0.3, 0.4) --> Good : [0.4, 0.5) --> Very Good : [0.5 ,1] -*/ -export function addressTypeFactor( - transactions: Transaction[], - walletAddressType: MultisigAddressType, - network: Network, -): number { - const addressCounts: Record = { - P2WSH: 0, - P2SH: 0, - P2PKH: 0, - P2TR: 0, - UNKNOWN: 0, - "P2SH-P2WSH": 0, - }; - - transactions.forEach((tx) => { - tx.vout.forEach((output) => { - const addressType = getAddressType(output.scriptPubkeyAddress, network); - addressCounts[addressType]++; - }); - }); - - const totalAddresses = Object.values(addressCounts).reduce( - (a, b) => a + b, - 0, - ); - const walletTypeCount = addressCounts[walletAddressType]; - - if (walletTypeCount === 0 || totalAddresses === walletTypeCount) { - return 1; - } - return 1 / (walletTypeCount + 1); -} - -/* -The spread factor using standard deviation helps in assessing the dispersion of UTXO values. -In Bitcoin privacy, spreading UTXOs reduces traceability by making it harder for adversaries -to link transactions and deduce the ownership and spending patterns of users. - -Expected Range : [0,1) --> Very Poor : (0, 0.2] --> Poor : [0.2, 0.4) --> Moderate : [0.4, 0.6) --> Good : [0.6, 0.8) --> Very Good : [0.8 ,1] -*/ -export function utxoSpreadFactor(utxos: AddressUtxos): number { - const amounts: number[] = []; - for (const address in utxos) { - const addressUtxos = utxos[address]; - addressUtxos.forEach((utxo) => { - amounts.push(utxo.value); - }); - } - - const mean: number = - amounts.reduce((sum, amount) => sum + amount, 0) / amounts.length; - const variance: number = - amounts.reduce((sum, amount) => sum + Math.pow(amount - mean, 2), 0) / - amounts.length; - const stdDev: number = Math.sqrt(variance); - return stdDev / (stdDev + 1); -} - -/* -The score is ad-hoc to normalize the privacy score based on the number of UTXOs in the set. - -Expected Range : [0,1] -- 0 for UTXO set length >= 50 -- 0.25 for UTXO set length >= 25 and <= 49 -- 0.5 for UTXO set length >= 15 and <= 24 -- 0.75 for UTXO set length >= 5 and <= 14 -- 1 for UTXO set length < 5 -*/ -export function utxoMassFactor(utxos: AddressUtxos): number { +export function utxoSetLengthMass(utxos: AddressUtxos): number { let utxoSetLength = 0; for (const address in utxos) { const addressUtxos = utxos[address]; utxoSetLength += addressUtxos.length; } - let score: number; + let utxoMassFactor: number; if (utxoSetLength >= 50) { - score = 0; + utxoMassFactor = 0; } else if (utxoSetLength >= 25 && utxoSetLength <= 49) { - score = 0.25; + utxoMassFactor = 0.25; } else if (utxoSetLength >= 15 && utxoSetLength <= 24) { - score = 0.5; + utxoMassFactor = 0.5; } else if (utxoSetLength >= 5 && utxoSetLength <= 14) { - score = 0.75; + utxoMassFactor = 0.75; } else { - score = 1; + utxoMassFactor = 1; } - return score; -} - -/* -UTXO Value Dispersion Factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. -It signifies the combined effect of how much variance is there in the UTXO Set values is and how many number of UTXOs are there. - -Expected Range : [-0.15,0.15] --> Very Poor : [-0.1, -0.05) --> Poor : [-0.05, 0) --> Moderate : [0, 0.05) --> Good : [0.05, 0.1) --> Very Good : [0.1 ,0.15] -*/ -export function utxoValueDispersionFactor(utxos: AddressUtxos): number { - let UMF: number = utxoMassFactor(utxos); - let USF: number = utxoSpreadFactor(utxos); - return (USF + UMF) * 0.15 - 0.15; -} - -/* -The privacy score is a combination of all the factors calculated above. -- Privacy Score based on Inputs and Outputs (i.e Tx Topology) -- Address Reuse Factor (R.F) -- Address Type Factor (A.T.F) -- UTXO Value Dispersion Factor (U.V.D.F) - -Expected Range : [0, 1] --> Very Poor : [0, 0.2] --> Poor : (0.2, 0.4] --> Moderate : (0.4, 0.6] --> Good : (0.6, 0.8] --> Very Good : (0.8, 1] -*/ -export function getWalletPrivacyScore( - transactions: Transaction[], - utxos: AddressUtxos, - walletAddressType: MultisigAddressType, - network: Network, -): number { - let privacyScore = getMeanTopologyScore(transactions); - - // Adjusting the privacy score based on the address reuse factor - let addressReusedFactor = addressReuseFactor(utxos); - privacyScore = - privacyScore * (1 - 0.5 * addressReusedFactor) + - 0.1 * (1 - addressReusedFactor); - - // Adjusting the privacy score based on the address type factor - privacyScore = - privacyScore * - (1 - addressTypeFactor(transactions, walletAddressType, network)); - - // Adjusting the privacy score based on the UTXO set length and value dispersion factor - privacyScore = privacyScore + 0.1 * utxoValueDispersionFactor(utxos); - - return privacyScore; + return utxoMassFactor; } diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 769c15cd..fb7da4d7 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,5 +1,5 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { utxoMassFactor } from "./privacy"; +import { utxoSetLengthMass} from "./privacy"; import { AddressUtxos } from "./types"; import { getFeeRateForTransaction, getFeeRatePercentileScore } from "./utils"; @@ -153,7 +153,7 @@ export class WasteMetric { ): number { let RFS = this.relativeFeesScore(transactions, feeRatePercentileHistory); let FAR = this.feesToAmountRatio(transactions); - let UMF = utxoMassFactor(utxos); + let UMF = utxoSetLengthMass(utxos); return 0.35 * RFS + 0.35 * FAR + 0.3 * UMF; } } From 41bcab39f9d0e10b84204fc2c80385822692537c Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 13 Aug 2024 02:26:13 +0530 Subject: [PATCH 49/92] Descriptive Test cases and Object Oriented Restructure Signed-off-by: Harshil-Jani --- packages/caravan-health/jest.config.js | 2 +- packages/caravan-health/src/index.ts | 2 +- packages/caravan-health/src/privacy.test.ts | 838 +++++++++----------- packages/caravan-health/src/privacy.ts | 197 ++--- packages/caravan-health/src/types.ts | 19 + packages/caravan-health/src/utils.ts | 82 -- packages/caravan-health/src/wallet.test.ts | 187 +++++ packages/caravan-health/src/wallet.ts | 114 +++ packages/caravan-health/src/waste.test.ts | 59 +- packages/caravan-health/src/waste.ts | 11 +- 10 files changed, 843 insertions(+), 668 deletions(-) delete mode 100644 packages/caravan-health/src/utils.ts create mode 100644 packages/caravan-health/src/wallet.test.ts create mode 100644 packages/caravan-health/src/wallet.ts diff --git a/packages/caravan-health/jest.config.js b/packages/caravan-health/jest.config.js index b6c0f588..0113a7bb 100644 --- a/packages/caravan-health/jest.config.js +++ b/packages/caravan-health/jest.config.js @@ -1,5 +1,5 @@ // jest.config.js module.exports = { preset: "ts-jest", - testEnvironment: "node" + testEnvironment: "jsdom" }; diff --git a/packages/caravan-health/src/index.ts b/packages/caravan-health/src/index.ts index 3fa0b667..0148b0ce 100644 --- a/packages/caravan-health/src/index.ts +++ b/packages/caravan-health/src/index.ts @@ -1,4 +1,4 @@ export * from "./privacy"; export * from "./types"; export * from "./waste"; -export * from "./utils"; +export * from "./wallet"; diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 90719bf4..69d07fbf 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,460 +1,378 @@ -// import { -// getTopologyScore, -// addressReuseFactor, -// addressTypeFactor, -// utxoSpreadFactor, -// utxoMassFactor, -// utxoValueDispersionFactor, -// getWalletPrivacyScore, -// getMeanTopologyScore, -// } from "./privacy"; -// import { BlockchainClient, Transaction } from "@caravan/clients"; -// import { AddressUtxos } from "./types"; -// import { MultisigAddressType, Network } from "@caravan/bitcoin"; - -// describe("Privacy Score Metrics", () => { -// let mockClient: BlockchainClient; -// let mockTransactionsAddressNotReused: Transaction[] = [ -// { -// vin: [ -// { -// prevTxId: "abcd", -// vout: 0, -// sequence: 0, -// }, -// ], -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// ]; -// let mockTransactionsAddressReused: Transaction[] = [ -// { -// vin: [ -// { -// prevTxId: "abcd", -// vout: 0, -// sequence: 0, -// }, -// ], -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// { -// vin: [ -// { -// prevTxId: "abcd", -// vout: 0, -// sequence: 0, -// }, -// ], -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// ]; - -// it("Perfect Spend Transaction without reused address for calculating transaction topology score", async () => { -// mockClient = { -// getAddressStatus: jest.fn(), -// getAddressTransactions: jest -// .fn() -// .mockResolvedValue(mockTransactionsAddressNotReused), -// } as unknown as BlockchainClient; -// const transaction: Transaction = { -// vin: [ -// { -// prevTxId: "input1", -// vout: 0, -// sequence: 0, -// }, -// ], // 1 Input -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output -// txid: "123", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }; -// const score: number = await getTopologyScore(transaction, mockClient); -// expect(score).toBe(0.75); -// }); - -// it("Perfect Spend Transaction with reused address for calculating transaction topology score", async () => { -// mockClient = { -// getAddressStatus: jest.fn(), -// getAddressTransactions: jest -// .fn() -// .mockResolvedValue(mockTransactionsAddressReused), -// } as unknown as BlockchainClient; -// const transaction: Transaction = { -// vin: [ -// { -// prevTxId: "input1", -// vout: 0, -// sequence: 0, -// }, -// ], // 1 Input -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output -// txid: "123", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }; -// const score: number = await getTopologyScore(transaction, mockClient); -// expect(score).toBe(0.5); -// }); - -// it("Calculating mean transaction topology score for multiple trnasactions", async () => { -// mockClient = { -// getAddressStatus: jest.fn(), -// getAddressTransactions: jest -// .fn() -// .mockResolvedValue(mockTransactionsAddressNotReused), -// } as unknown as BlockchainClient; -// const transactions: Transaction[] = [ -// { -// // Perfect Spend (No Reused Address) - 0.75 -// vin: [ -// { -// prevTxId: "input1", -// vout: 0, -// sequence: 0, -// }, -// ], // 1 Input -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }], // 1 Output -// txid: "123", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// { -// // Simple Spend (No Reused Address) - 0.66 -// vin: [ -// { -// prevTxId: "input1", -// vout: 0, -// sequence: 0, -// }, -// ], // 1 Input -// vout: [ -// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// { scriptPubkeyHex: "", scriptPubkeyAddress: "", value: 0 }, -// ], // 2 Outputs -// txid: "123", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// ]; -// const score: number = await getMeanTopologyScore(transactions, mockClient); -// expect(score).toBeCloseTo(0.708); -// }); - -// it("Address Reuse Factor accounts for Unspent coins that are on reused address with respect to total amount in wallet", async () => { -// const utxos: AddressUtxos = { -// address1: [ -// { -// txid: "tx1", -// vout: 0, -// value: 1, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx2", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// address2: [ -// { -// txid: "tx3", -// vout: 0, -// value: 3, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// }; - -// // No address was reused -// mockClient = { -// getAddressStatus: jest.fn(), -// getAddressTransactions: jest -// .fn() -// .mockResolvedValue(mockTransactionsAddressNotReused), -// } as unknown as BlockchainClient; -// const factor: number = await addressReuseFactor(utxos, mockClient); -// expect(factor).toBe(0); - -// // All addresses were reused -// mockClient = { -// getAddressStatus: jest.fn(), -// getAddressTransactions: jest -// .fn() -// .mockResolvedValue(mockTransactionsAddressReused), -// } as unknown as BlockchainClient; -// const factor2: number = await addressReuseFactor(utxos, mockClient); -// expect(factor2).toBe(1); -// }); - -// it("P2PKH wallet address type being checked for all transactions", () => { -// const transactions: Transaction[] = [ -// { -// vin: [ -// { -// prevTxId: "input1", -// vout: 0, -// sequence: 0, -// }, -// ], -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], // Address starting with 1 -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// ]; -// const walletAddressType: MultisigAddressType = "P2PKH"; -// const network: Network = Network["MAINNET"]; -// const factor: number = addressTypeFactor( -// transactions, -// walletAddressType, -// network, -// ); -// expect(factor).toBe(1); -// }); - -// it("UTXOs spread factor across multiple addresses (assess for how similar the amount values are for each UTXO)", () => { -// const utxos = { -// address1: [ -// { -// txid: "tx1", -// vout: 0, -// value: 1, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// address2: [ -// { -// txid: "tx2", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// }; -// // Value 1 and 2 is so close to each other so the scoring is bad. -// const factor: number = utxoSpreadFactor(utxos); -// expect(factor).toBe(1 / 3); - -// const utxos2 = { -// address1: [ -// { -// txid: "tx1", -// vout: 0, -// value: 1, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// address2: [ -// { -// txid: "tx2", -// vout: 0, -// value: 200, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// }; -// // Value 1 and 200 is so far from each other so the scoring is good. -// const factor2: number = utxoSpreadFactor(utxos2); -// expect(factor2).toBeCloseTo(0.99); -// }); - -// it("Gives a score on the basis of number of UTXOs present in the wallet", () => { -// const utxos = { -// address1: [ -// { -// txid: "tx1", -// vout: 0, -// value: 1, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], // 1 UTXO only for address1 -// address2: [ -// { -// txid: "tx2", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx3", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx4", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx5", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx6", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx7", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], // 6 UTXOs for address 2 -// }; -// // 7 UTXOs in total - which will give 0.75 as score -// const score: number = utxoMassFactor(utxos); -// expect(score).toBe(0.75); -// }); - -// it("UTXO value dispersion accounts for number of coins in the wallet and how dispersed they are in amount values", () => { -// const utxos = { -// address1: [ -// { -// txid: "tx1", -// vout: 0, -// value: 1, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], // 1 UTXO only for address1 -// address2: [ -// { -// txid: "tx2", -// vout: 0, -// value: 0.02, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx3", -// vout: 0, -// value: 0.2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx4", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx5", -// vout: 0, -// value: 20, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx6", -// vout: 0, -// value: 200, -// status: { confirmed: true, block_time: 0 }, -// }, -// { -// txid: "tx7", -// vout: 0, -// value: 2000, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], // 6 UTXOs for address 2 -// }; -// const factor: number = utxoValueDispersionFactor(utxos); -// expect(factor).toBeCloseTo(0.112); -// }); - -// it("Overall Privacy Score taking into consideration all parameters for UTXO and Transaction History", async () => { -// const transactions: Transaction[] = [ -// { -// vin: [ -// { -// prevTxId: "input1", -// vout: 0, -// sequence: 0, -// }, -// ], -// vout: [{ scriptPubkeyHex: "", scriptPubkeyAddress: "1234", value: 0 }], -// txid: "", -// size: 0, -// weight: 0, -// fee: 0, -// isSend: false, -// amount: 0, -// blocktime: 0, -// }, -// ]; -// const utxos = { -// address1: [ -// { -// txid: "tx1", -// vout: 0, -// value: 1, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// address2: [ -// { -// txid: "tx2", -// vout: 0, -// value: 2, -// status: { confirmed: true, block_time: 0 }, -// }, -// ], -// }; -// const walletAddressType: MultisigAddressType = "P2PKH"; -// const network: Network = Network["MAINNET"]; -// const score: number = await getWalletPrivacyScore( -// transactions, -// utxos, -// walletAddressType, -// mockClient, -// network, -// ); -// expect(score).toBeCloseTo(0.005); -// }); -// }); +import { Transaction } from "@caravan/clients"; +import { PrivacyMetrics } from "./privacy"; +import { AddressUtxos, SpendType } from "./types"; +import { Network } from "@caravan/bitcoin"; + +const transactions: Transaction[] = [ + // transactions[0] is a perfect spend transaction + { + txid: "txid1", + vin: [ + { + prevTxId: "prevTxId1", + vout: 0, + sequence: 0, + }, + ], + vout: [ + { + scriptPubkeyHex: "scriptPubkeyHex1", + scriptPubkeyAddress: "scriptPubkeyAddress1", + value: 0.1, + }, + ], + size: 0, + weight: 0, + fee: 0, + isSend: true, + amount: 0, + blocktime: 0, + }, + // transactions[1] is a coin join transaction + { + txid: "txid2", + vin: [ + { + prevTxId: "prevTxId2", + vout: 0, + sequence: 0, + }, + { + prevTxId: "prevTxId2", + vout: 0, + sequence: 0, + }, + ], + vout: [ + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + ], + size: 0, + weight: 0, + fee: 0, + isSend: true, + amount: 0, + blocktime: 0, + }, +]; + +const utxos: AddressUtxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 0.1, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx2", + vout: 0, + value: 0.2, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx3", + vout: 0, + value: 0.3, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx4", + vout: 0, + value: 0.4, + status: { + confirmed: true, + block_time: 1234, + }, + }, + ], +}; + +describe("Waste metric scoring", () => { + const privacyMetric = new PrivacyMetrics(); + + describe("Determine Spend Type", () => { + it("Perfect Spend are transactions with 1 input and 1 output", () => { + const spendType: SpendType = privacyMetric.determineSpendType(1, 1); + expect(spendType).toBe(SpendType.PerfectSpend); + }); + + it("Simple Spend are transactions with 1 input and 2 outputs", () => { + const spendType: SpendType = privacyMetric.determineSpendType(1, 2); + expect(spendType).toBe(SpendType.SimpleSpend); + }); + + it("UTXO Fragmentation are transactions with 1 input and more than 2 outputs", () => { + const spendType: SpendType = privacyMetric.determineSpendType(1, 3); + expect(spendType).toBe(SpendType.UTXOFragmentation); + + const spendType2: SpendType = privacyMetric.determineSpendType(1, 4); + expect(spendType2).toBe(SpendType.UTXOFragmentation); + + const spendType3: SpendType = privacyMetric.determineSpendType(1, 5); + expect(spendType3).toBe(SpendType.UTXOFragmentation); + }); + + it("Consolidation transactions have more than 1 inputs and 1 output", () => { + const spendType: SpendType = privacyMetric.determineSpendType(2, 1); + expect(spendType).toBe(SpendType.Consolidation); + + const spendType2: SpendType = privacyMetric.determineSpendType(3, 1); + expect(spendType2).toBe(SpendType.Consolidation); + + const spendType3: SpendType = privacyMetric.determineSpendType(4, 1); + expect(spendType3).toBe(SpendType.Consolidation); + }); + + it("Mixing or CoinJoin transactions have more than 1 inputs and more than 1 outputs", () => { + const spendType: SpendType = privacyMetric.determineSpendType(2, 2); + expect(spendType).toBe(SpendType.MixingOrCoinJoin); + + const spendType2: SpendType = privacyMetric.determineSpendType(2, 3); + expect(spendType2).toBe(SpendType.MixingOrCoinJoin); + + const spendType3: SpendType = privacyMetric.determineSpendType(3, 2); + expect(spendType3).toBe(SpendType.MixingOrCoinJoin); + }); + }); + + describe("Spend Type Score", () => { + /* + score = P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) + + Perfect Spend Transaction + - No. of Input = 1 + - No. of Output = 1 + + P("An output can be a self-payment") = 0.5 + P("An output cannot be a self-payment") = 0.5 + P(“involvement of any change output”) = 0 (when number of output is 1 it will be 0) + + score = 0.5 * (1 - 0) = 0.5 + */ + it("Perfect Spend has a raw score of 0.5 for external wallet payments", () => { + const score: number = privacyMetric.getSpendTypeScore( + SpendType.PerfectSpend, + 1, + 1, + ); + expect(score).toBe(0.5); + }); + /* + score = P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) + + Simple Spend Transaction + - No. of Input = 1 + - No. of Output = 2 + + P("An output can be a self-payment") = 0.33 + P("An output cannot be a self-payment") = 0.67 + + P1 (Party1) -> P2 (Party-2) , P3(Party-3) | No change involved + P1 (Party1) -> P1 (Party1), P1 (Party1) | No change involved + P2 (Party-2) -> P2 (Party-2) ,P1 (Party1) | Yes change is involved + P(“involvement of any change output”) = 0.33 + + + score = 0.67 * (1-0.33) = 0.4489 + */ + it("Simple Spend has a raw score of 0.44 for external wallet payments", () => { + const score: number = privacyMetric.getSpendTypeScore( + SpendType.SimpleSpend, + 1, + 2, + ); + expect(score).toBeCloseTo(0.44); + }); + + /* + UTXO Fragmentation Transaction + ONE to MANY transaction + No. of Input = 1 + No. of Output = 3 or more (MANY) + + score = 0.67 - ( 1 / Number of Outputs ) + + Justification behind the number 0.67 : + We want that the privacy score should increase with more numbers of outputs, + because if in case it was a self spent transaction then producing more outputs means + producing more UTXOs which have privacy benefits for the wallet. + + Now, We are using a multiplication factor of 1.5 for deniability in case of all the self spend transactions. + So the quantity [ X - ( 1 / No. of outputs) ] should be less than 1 + + [ X - ( 1 / No. of outputs) ] <= 1 + + Here the quantity ( 1 / No. of outputs) could be maximum when No. of outputs = 3 + [X - ⅓ ] <= 1 + [X - 0.33] <= 1 + X<=0.67 + + */ + it("UTXO Fragmentation has a raw score of 0.33 for external wallet payments", () => { + const score: number = privacyMetric.getSpendTypeScore( + SpendType.UTXOFragmentation, + 1, + 3, + ); + expect(score).toBeCloseTo(0.33); + }); + + /* + Consolidation Transaction + MANY to ONE transaction + No. of Input = 2 or more (MANY) + No. of Output = 1 + + When the number of inputs are more against the single output then the privacy score should decrease because + it increases the fingerprint or certainty for on-chain analysers that the following transaction was made as + a consolidation and with more number of inputs we tend to expose more UTXOs for a transaction. + + score = 1 / Number of Inputs + */ + it("Consolidation has raw score of ", () => { + const score: number = privacyMetric.getSpendTypeScore( + SpendType.Consolidation, + 2, + 1, + ); + expect(score).toBeCloseTo(0.5); + + const score2: number = privacyMetric.getSpendTypeScore( + SpendType.Consolidation, + 3, + 1, + ); + expect(score2).toBeCloseTo(0.33); + }); + + /* + Mixing or CoinJoin Transaction + MANY to MANY transaction + No. of Input = 2 or more (MANY) + No. of Output = 2 or more (MANY) + + Justification : + Privacy score is directly proportional to More Number of outputs AND less number of inputs in case of coin join. + The explanation for this to happen is that if you try to consolidate + i.e lower number of output and high number of input, the privacy should be decreased and + in case of coin join where there are so many outputs against few inputs it should have increased + privacy since the probability of someone find out if the coin belongs to you or not is very small. + + score = 1/2 * (y2/x)/(1+y2/x) + */ + it("MixingOrCoinJoin has raw score of ", () => { + const score: number = privacyMetric.getSpendTypeScore( + SpendType.MixingOrCoinJoin, + 2, + 2, + ); + expect(score).toBeCloseTo(0.33); + + const score2: number = privacyMetric.getSpendTypeScore( + SpendType.MixingOrCoinJoin, + 2, + 3, + ); + expect(score2).toBeCloseTo(0.409); + + const score3: number = privacyMetric.getSpendTypeScore( + SpendType.MixingOrCoinJoin, + 3, + 2, + ); + expect(score3).toBeCloseTo(0.285); + }); + }); + + describe("Transaction Topology Score", () => { + it("Calculates the transaction topology score based on the spend type", () => { + const score: number = privacyMetric.getTopologyScore(transactions[0]); + expect(score).toBe(0.75); + + const score2: number = privacyMetric.getTopologyScore(transactions[1]); + expect(score2).toBeCloseTo(0.67); + }); + }); + + describe("Mean Topology Score", () => { + it("Calculates the mean topology score for all transactions done by a wallet", () => { + const meanScore: number = + privacyMetric.getMeanTopologyScore(transactions); + expect(meanScore).toBeCloseTo(0.71); + }); + }); + + describe("Address Reuse Factor", () => { + it("Calculates the amount being held by reused addresses with respect to the total amount", () => { + const addressReuseFactor: number = + privacyMetric.addressReuseFactor(utxos); + expect(addressReuseFactor).toBe(0); + }); + }); + + describe("Address Type Factor", () => { + it("Calculates the the address type distribution of the wallet transactions", () => { + const addressTypeFactor: number = privacyMetric.addressTypeFactor( + transactions, + "P2SH", + Network.MAINNET, + ); + expect(addressTypeFactor).toBe(1); + }); + }); + + describe("UTXO Spread Factor", () => { + it("Calculates the standard deviation of UTXO values which helps in assessing the dispersion of UTXO values", () => { + const utxoSpreadFactor: number = privacyMetric.utxoSpreadFactor(utxos); + expect(utxoSpreadFactor).toBeCloseTo(0.1); + }); + }); + + describe("UTXO Value Dispersion Factor", () => { + it("Combines UTXO Spread Factor and UTXO Mass Factor", () => { + const utxoValueDispersionFactor: number = + privacyMetric.utxoValueDispersionFactor(utxos); + expect(utxoValueDispersionFactor).toBeCloseTo(0.015); + }); + }); + + describe("Overall Privacy Score", () => { + it("Calculates the overall privacy score for a wallet", () => { + const privacyScore: number = privacyMetric.getWalletPrivacyScore( + transactions, + utxos, + "P2SH", + Network.MAINNET, + ); + expect(privacyScore).toBeCloseTo(0.0015); + }); + }); +}); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index feb921e1..4ae4af13 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,8 +1,81 @@ import { Transaction } from "@caravan/clients"; -import { AddressUtxos } from "./types"; +import { AddressUtxos, SpendType } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; +import { WalletMetrics} from "./wallet"; + +// Deniability Factor is a normalizing quantity that increases the score by a certain factor in cases of self-payment. +// More about deniability : https://www.truthcoin.info/blog/deniability/ +const DENIABILITY_FACTOR = 1.5; + +export class PrivacyMetrics extends WalletMetrics { + /* + Name : Spend Type Determination + + Definition : + The type of spend transaction is obtained based on the number of inputs and outputs which + influence the topology type of the transaction and has a role in determining the fingerprints + behind privacy for wallets. + + Calculation : + We have 5 categories of transaction type each with their own impact on privacy score + - Perfect Spend (1 input, 1 output) + - Simple Spend (1 input, 2 outputs) + - UTXO Fragmentation (1 input, more than 2 standard outputs) + - Consolidation (more than 1 input, 1 output) + - CoinJoin or Mixing (more than 1 input, more than 1 output) + */ + determineSpendType(inputs: number, outputs: number): SpendType { + if (inputs === 1) { + if (outputs === 1) return SpendType.PerfectSpend; + if (outputs === 2) return SpendType.SimpleSpend; + return SpendType.UTXOFragmentation; + } else { + if (outputs === 1) return SpendType.Consolidation; + return SpendType.MixingOrCoinJoin; + } + } + + /* + Name : Spend Type Score + Definition : + Statistical derivations are used to calculate the score based on the spend type of the transaction. + + Calculation : + - Perfect Spend : P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) + - Simple Spend : P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) + - UTXO Fragmentation : 2/3 - 1/number of outputs + - Consolidation : 1/number of inputs + - Mixing or CoinJoin : (2/3) * (number of outputs^2) / number of inputs * (1 + (number of outputs^2) / number of inputs) + + Expected Range : [0,0.85] + -> Very Poor : [0, 0.15] + -> Poor : (0.15, 0.3] + -> Moderate : (0.3, 0.45] + -> Good : (0.45, 0.6] + -> Very Good : (0.6, 0.85] + */ + getSpendTypeScore( + spendType: SpendType, + numberOfInputs: number, + numberOfOutputs: number, + ): number { + switch (spendType) { + case SpendType.PerfectSpend: + return 1 / 2; + case SpendType.SimpleSpend: + return 4 / 9; + case SpendType.UTXOFragmentation: + return 2 / 3 - 1 / numberOfOutputs; + case SpendType.Consolidation: + return 1 / numberOfInputs; + case SpendType.MixingOrCoinJoin: + let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; + return ((1 / 2) * x) / (1 + x); + default: + throw new Error("Invalid spend type"); + } + } -export class PrivacyMetric { /* Name : Topology Score @@ -22,11 +95,11 @@ export class PrivacyMetric { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; - const spendType: SpendType = determineSpendType( + const spendType: SpendType = this.determineSpendType( numberOfInputs, numberOfOutputs, ); - const score: number = getSpendTypeScore( + const score: number = this.getSpendTypeScore( spendType, numberOfInputs, numberOfOutputs, @@ -37,7 +110,7 @@ export class PrivacyMetric { } for (let output of transaction.vout) { let address = output.scriptPubkeyAddress; - let isResued = isReusedAddress(address); + let isResued = this.isReusedAddress(address); if (isResued === true) { return score; } @@ -98,7 +171,7 @@ export class PrivacyMetric { const addressUtxos = utxos[address]; for (const utxo of addressUtxos) { totalAmount += utxo.value; - let isReused = isReusedAddress(address); + let isReused = this.isReusedAddress(address); if (isReused) { reusedAmount += utxo.value; } @@ -111,7 +184,7 @@ export class PrivacyMetric { Name : Address Type Factor (ATF) Definition : - The address type factor evaluates the address type distribution of the wallet. + The address type factor evaluates the address type distribution of the wallet transactions. It signifies the privacy health of the wallet based on the address types used. Calculation : @@ -199,28 +272,11 @@ export class PrivacyMetric { return stdDev / (stdDev + 1); } - /* - Name : UTXO Mass Factor - - Calculation : - The mass factor is calculated based on the number of UTXOs in the set. - - Expected Range : [0,1] - - 0 for UTXO set length >= 50 - - 0.25 for UTXO set length >= 25 and <= 49 - - 0.5 for UTXO set length >= 15 and <= 24 - - 0.75 for UTXO set length >= 5 and <= 14 - - 1 for UTXO set length < 5 - */ - utxoMassFactor(utxos: AddressUtxos): number { - return utxoSetLengthMass(utxos); - } - /* Name : UTXO Value Dispersion Factor Definition : - The UTXO value dispersion factor is a combination of UTXO Spread Factor and UTXO Set Length Weight. + The UTXO value dispersion factor is a combination of UTXO Spread Factor and UTXO Mass Factor. It signifies the combined effect of how much variance is there in the UTXO Set values is and how many number of UTXOs are there. @@ -272,93 +328,4 @@ export class PrivacyMetric { return WPS; } -} - -/* -The methodology for calculating a privacy score (p_score) for Bitcoin transactions based -on the number of inputs and outputs is the primary point to define wallet health for privacy. -The score is further influenced by several factors such as address reuse, -address types and UTXO set fingerprints etc. -*/ - -// A normalizing quantity that increases the score by a certain factor in cases of self-payment. -// More about deniability : https://www.truthcoin.info/blog/deniability/ -const DENIABILITY_FACTOR = 1.5; - -/* -The p_score is calculated by evaluating the likelihood of self-payments, the involvement of -change outputs and the type of transaction based on number of inputs and outputs. - -We have 5 categories of transaction type each with their own impact on privacy score -- Perfect Spend (1 input, 1 output) -- Simple Spend (1 input, 2 outputs) -- UTXO Fragmentation (1 input, more than 2 standard outputs) -- Consolidation (more than 1 input, 1 output) -- CoinJoin or Mixing (more than 1 input, more than 1 output) -*/ -enum SpendType { - PerfectSpend = "PerfectSpend", - SimpleSpend = "SimpleSpend", - UTXOFragmentation = "UTXOFragmentation", - Consolidation = "Consolidation", - MixingOrCoinJoin = "MixingOrCoinJoin", -} - -function determineSpendType(inputs: number, outputs: number): SpendType { - if (inputs === 1) { - if (outputs === 1) return SpendType.PerfectSpend; - if (outputs === 2) return SpendType.SimpleSpend; - return SpendType.UTXOFragmentation; - } else { - if (outputs === 1) return SpendType.Consolidation; - return SpendType.MixingOrCoinJoin; - } -} - -export function getSpendTypeScore( - spendType: SpendType, - numberOfInputs: number, - numberOfOutputs: number, -): number { - switch (spendType) { - case SpendType.PerfectSpend: - return 1 / 2; - case SpendType.SimpleSpend: - return 4 / 9; - case SpendType.UTXOFragmentation: - return 2 / 3 - 1 / numberOfOutputs; - case SpendType.Consolidation: - return 1 / numberOfInputs; - case SpendType.MixingOrCoinJoin: - let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; - return ((2 / 3) * x) / (1 + x); - default: - throw new Error("Invalid spend type"); - } -} - -function isReusedAddress(address: string): boolean { - // TODO : Implement a function to check if the address is reused - return false; -} - -export function utxoSetLengthMass(utxos: AddressUtxos): number { - let utxoSetLength = 0; - for (const address in utxos) { - const addressUtxos = utxos[address]; - utxoSetLength += addressUtxos.length; - } - let utxoMassFactor: number; - if (utxoSetLength >= 50) { - utxoMassFactor = 0; - } else if (utxoSetLength >= 25 && utxoSetLength <= 49) { - utxoMassFactor = 0.25; - } else if (utxoSetLength >= 15 && utxoSetLength <= 24) { - utxoMassFactor = 0.5; - } else if (utxoSetLength >= 5 && utxoSetLength <= 14) { - utxoMassFactor = 0.75; - } else { - utxoMassFactor = 1; - } - return utxoMassFactor; -} +} diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index 2d855ecd..505ef9d6 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -4,3 +4,22 @@ import { UTXO } from "@caravan/clients"; export interface AddressUtxos { [address: string]: UTXO[]; } + +/* +The p_score is calculated by evaluating the likelihood of self-payments, the involvement of +change outputs and the type of transaction based on number of inputs and outputs. + +We have 5 categories of transaction type each with their own impact on privacy score +- Perfect Spend (1 input, 1 output) +- Simple Spend (1 input, 2 outputs) +- UTXO Fragmentation (1 input, more than 2 standard outputs) +- Consolidation (more than 1 input, 1 output) +- CoinJoin or Mixing (more than 1 input, more than 1 output) +*/ +export enum SpendType { + PerfectSpend = "PerfectSpend", + SimpleSpend = "SimpleSpend", + UTXOFragmentation = "UTXOFragmentation", + Consolidation = "Consolidation", + MixingOrCoinJoin = "MixingOrCoinJoin", +} diff --git a/packages/caravan-health/src/utils.ts b/packages/caravan-health/src/utils.ts deleted file mode 100644 index f795d6d3..00000000 --- a/packages/caravan-health/src/utils.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { FeeRatePercentile, Transaction } from "@caravan/clients"; - -/* -Utility function that helps to obtain the fee rate of the transaction - -Expected Range : [0, 0.75] --> Very Poor : [0, 0.15] --> Poor : (0.15, 0.3] --> Moderate : (0.3, 0.45] --> Good : (0.45, 0.6] --> Very Good : (0.6, 0.75] -*/ -export function getFeeRateForTransaction(transaction: Transaction): number { - let fees: number = transaction.fee; - let weight: number = transaction.weight; - return fees / weight; -} - -/* - Utility function that helps to obtain the percentile of the fees paid by user in tx block - - Expected Range : [0, 0.75] - -> 0% tile : 1 - -> 10% tile : 0.9 - -> 25% tile : 0.75 - -> 50% tile : 0.5 - -> 75% tile : 0.25 - -> 90% tile : 0.1 - -> 100% tile : 0.05 - */ -export function getFeeRatePercentileScore( - timestamp: number, - feeRate: number, - feeRatePercentileHistory: FeeRatePercentile[], -): number { - let percentile: number = getPercentile( - timestamp, - feeRate, - feeRatePercentileHistory, - ); - return 1 - percentile / 100; -} - -function getPercentile( - timestamp: number, - feeRate: number, - feeRatePercentileHistory: FeeRatePercentile[], -): number { - // Find the closest entry by timestamp - let closestBlock: FeeRatePercentile | null = null; - let closestDifference: number = Infinity; - - for (const block of feeRatePercentileHistory) { - const difference = Math.abs(block.timestamp - timestamp); - if (difference <= closestDifference) { - closestDifference = difference; - closestBlock = block; - } - } - if (!closestBlock) { - throw new Error("No fee rate data found"); - } - // Find the closest fee rate percentile - switch (true) { - case feeRate <= closestBlock.avgFee_0: - return 0; - case feeRate <= closestBlock.avgFee_10: - return 10; - case feeRate <= closestBlock.avgFee_25: - return 25; - case feeRate <= closestBlock.avgFee_50: - return 50; - case feeRate <= closestBlock.avgFee_75: - return 75; - case feeRate <= closestBlock.avgFee_90: - return 90; - case feeRate <= closestBlock.avgFee_100: - return 100; - default: - throw new Error("Invalid fee rate"); - } -} diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts new file mode 100644 index 00000000..c34fe82b --- /dev/null +++ b/packages/caravan-health/src/wallet.test.ts @@ -0,0 +1,187 @@ +import { WalletMetrics } from "./wallet"; +import { AddressUtxos } from "./types"; +import { FeeRatePercentile, Transaction } from "@caravan/clients"; + +const transactions: Transaction[] = [ + // transactions[0] is a perfect spend transaction + { + txid: "txid1", + vin: [ + { + prevTxId: "prevTxId1", + vout: 0, + sequence: 0, + }, + ], + vout: [ + { + scriptPubkeyHex: "scriptPubkeyHex1", + scriptPubkeyAddress: "scriptPubkeyAddress1", + value: 0.1, + }, + ], + size: 1, + weight: 1, + fee: 1, + isSend: true, + amount: 1, + blocktime: 1234, + }, + // transactions[1] is a coin join transaction + { + txid: "txid2", + vin: [ + { + prevTxId: "prevTxId2", + vout: 0, + sequence: 0, + }, + { + prevTxId: "prevTxId2", + vout: 0, + sequence: 0, + }, + ], + vout: [ + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + { + scriptPubkeyHex: "scriptPubkeyHex2", + scriptPubkeyAddress: "scriptPubkeyAddress2", + value: 0.2, + }, + ], + size: 0, + weight: 0, + fee: 0, + isSend: true, + amount: 0, + blocktime: 0, + }, +]; + +const utxos: AddressUtxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 0.1, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx2", + vout: 0, + value: 0.2, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx3", + vout: 0, + value: 0.3, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx4", + vout: 0, + value: 0.4, + status: { + confirmed: true, + block_time: 1234, + }, + }, + ], +}; + +describe("Wallet Metrics", () => { + const walletMetrics = new WalletMetrics(); + describe("UTXO Mass Factor", () => { + it("should return 1 for UTXO set length = 4", () => { + expect(walletMetrics.utxoMassFactor(utxos)).toBe(1); + }); + }); + + describe("Fee Rate For Transaction", () => { + it("should return 1 for fee = 1 and weight = 1", () => { + expect(walletMetrics.getFeeRateForTransaction(transactions[0])).toBe(1); + }); + }); + + describe("Fee Rate Percentile Score", () => { + it("should return 0.5 for 50th percentile", () => { + const feeRatePercentileHistory: FeeRatePercentile[] = [ + { + avgHeight: 1234, + timestamp: 1234, + avgFee_0: 0.001, + avgFee_10: 0.01, + avgFee_25: 0.1, + avgFee_50: 1, + avgFee_75: 1.1, + avgFee_90: 1.2, + avgFee_100: 1.3, + }, + ]; + expect( + walletMetrics.getFeeRatePercentileScore( + 1234, + 1, + feeRatePercentileHistory, + ), + ).toBe(0.5); + }); + }); + + describe("Closest Percentile", () => { + it("should return 50 for 0.5 at 1229 timestamp", () => { + const feeRatePercentileHistory: FeeRatePercentile[] = [ + { + avgHeight: 1234, + timestamp: 1234, + avgFee_0: 0.001, + avgFee_10: 0.01, + avgFee_25: 0.1, + avgFee_50: 1, + avgFee_75: 1.1, + avgFee_90: 1.2, + avgFee_100: 1.3, + }, + { + avgHeight: 1230, + timestamp: 1234, + avgFee_0: 0.002, + avgFee_10: 0.02, + avgFee_25: 0.2, + avgFee_50: 1, + avgFee_75: 1.2, + avgFee_90: 1.4, + avgFee_100: 1.8, + }, + ]; + expect( + walletMetrics.getClosestPercentile(1229, 0.5, feeRatePercentileHistory), + ).toBe(50); + }); + }); +}); diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts new file mode 100644 index 00000000..b40859d4 --- /dev/null +++ b/packages/caravan-health/src/wallet.ts @@ -0,0 +1,114 @@ +import { FeeRatePercentile, Transaction } from "@caravan/clients"; +import { AddressUtxos } from "./types"; + +export class WalletMetrics { + /* + Name : UTXO Mass Factor + + Calculation : + The mass factor is calculated based on the number of UTXOs in the set. + + Expected Range : [0,1] + - 0 for UTXO set length >= 50 + - 0.25 for UTXO set length >= 25 and <= 49 + - 0.5 for UTXO set length >= 15 and <= 24 + - 0.75 for UTXO set length >= 5 and <= 14 + - 1 for UTXO set length < 5 + */ + utxoMassFactor(utxos: AddressUtxos): number { + let utxoSetLength = 0; + for (const address in utxos) { + const addressUtxos = utxos[address]; + utxoSetLength += addressUtxos.length; + } + let utxoMassFactor: number; + if (utxoSetLength >= 50) { + utxoMassFactor = 0; + } else if (utxoSetLength >= 25 && utxoSetLength <= 49) { + utxoMassFactor = 0.25; + } else if (utxoSetLength >= 15 && utxoSetLength <= 24) { + utxoMassFactor = 0.5; + } else if (utxoSetLength >= 5 && utxoSetLength <= 14) { + utxoMassFactor = 0.75; + } else { + utxoMassFactor = 1; + } + return utxoMassFactor; + } + + /* + Utility function that helps to obtain the fee rate of the transaction + */ + getFeeRateForTransaction(transaction: Transaction): number { + let fees: number = transaction.fee; + let weight: number = transaction.weight; + return fees / weight; + } + + /* + Utility function that helps to obtain the percentile of the fees paid by user in tx block + */ + getFeeRatePercentileScore( + timestamp: number, + feeRate: number, + feeRatePercentileHistory: FeeRatePercentile[], + ): number { + let percentile: number = this.getClosestPercentile( + timestamp, + feeRate, + feeRatePercentileHistory, + ); + return 1 - percentile / 100; + } + + /* + Utility function that helps to obtain the closest percentile of the fees paid by user in tx block + */ + getClosestPercentile( + timestamp: number, + feeRate: number, + feeRatePercentileHistory: FeeRatePercentile[], + ): number { + // Find the closest entry by timestamp + let closestBlock: FeeRatePercentile | null = null; + let closestDifference: number = Infinity; + + for (const block of feeRatePercentileHistory) { + const difference = Math.abs(block.timestamp - timestamp); + if (difference <= closestDifference) { + closestDifference = difference; + closestBlock = block; + } + } + if (!closestBlock) { + throw new Error("No fee rate data found"); + } + // Find the closest fee rate percentile + switch (true) { + case feeRate <= closestBlock.avgFee_0: + return 0; + case feeRate <= closestBlock.avgFee_10: + return 10; + case feeRate <= closestBlock.avgFee_25: + return 25; + case feeRate <= closestBlock.avgFee_50: + return 50; + case feeRate <= closestBlock.avgFee_75: + return 75; + case feeRate <= closestBlock.avgFee_90: + return 90; + case feeRate <= closestBlock.avgFee_100: + return 100; + default: + throw new Error("Invalid fee rate"); + } + } + + /* + Utility function to check if the given address was used alreadyin past transactions + */ + isReusedAddress(address: string): boolean { + // TODO : Implement a function to check if the address is reused + return false; + } +} diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index adf0ab8c..5f3e2c87 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -1,4 +1,5 @@ -import { WasteMetric } from "./waste"; +import { AddressUtxos } from "./types"; +import { WasteMetrics } from "./waste"; const transactions = [ { @@ -39,8 +40,49 @@ const feeRatePercentileHistory = [ }, ]; +const utxos: AddressUtxos = { + address1: [ + { + txid: "tx1", + vout: 0, + value: 0.1, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx2", + vout: 0, + value: 0.2, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx3", + vout: 0, + value: 0.3, + status: { + confirmed: true, + block_time: 1234, + }, + }, + { + txid: "tx4", + vout: 0, + value: 0.4, + status: { + confirmed: true, + block_time: 1234, + }, + }, + ], +}; + describe("Waste metric scoring", () => { - const wasteMetric = new WasteMetric(); + const wasteMetric = new WasteMetrics(); describe("Relative Fees Score (R.F.S)", () => { it("calculates fee score based on tx fee rate relative to percentile in the block where a set of send tx were mined", () => { @@ -53,7 +95,7 @@ describe("Waste metric scoring", () => { }); describe("Fees to Amount Ratio (F.A.R)", () => { - it("Fees paid over total amount spent as ratio for a 'send' type transaction", async () => { + it("Fees paid over total amount spent as ratio for a 'send' type transaction", () => { const ratio: number = wasteMetric.feesToAmountRatio(transactions); expect(ratio).toBe(0.1); }); @@ -85,4 +127,15 @@ describe("Waste metric scoring", () => { // we can save 1850 sats }); }); + + describe("Weighted Waste Score (W.W.S)", () => { + it("calculates the overall waste of the wallet based on the relative fees score, fees to amount ratio and the UTXO mass factor", () => { + const score: number = wasteMetric.weightedWasteScore( + transactions, + utxos, + feeRatePercentileHistory, + ); + expect(score).toBe(0.21); + }); + }); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index fb7da4d7..41617226 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,9 +1,8 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { utxoSetLengthMass} from "./privacy"; import { AddressUtxos } from "./types"; -import { getFeeRateForTransaction, getFeeRatePercentileScore } from "./utils"; +import { WalletMetrics } from "../dist"; -export class WasteMetric { +export class WasteMetrics extends WalletMetrics { /* Name : Relative Fees Score (R.F.S) @@ -32,8 +31,8 @@ export class WasteMetric { for (const tx of transactions) { if (tx.isSend === true) { numberOfSendTx++; - let feeRate: number = getFeeRateForTransaction(tx); - let RFS: number = getFeeRatePercentileScore( + let feeRate: number = this.getFeeRateForTransaction(tx); + let RFS: number = this.getFeeRatePercentileScore( tx.blocktime, feeRate, feeRatePercentileHistory, @@ -153,7 +152,7 @@ export class WasteMetric { ): number { let RFS = this.relativeFeesScore(transactions, feeRatePercentileHistory); let FAR = this.feesToAmountRatio(transactions); - let UMF = utxoSetLengthMass(utxos); + let UMF = 0; return 0.35 * RFS + 0.35 * FAR + 0.3 * UMF; } } From 22b863b643d4f1e0f7f7f9cfa88c7ad5f4267b52 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 13 Aug 2024 02:55:33 +0530 Subject: [PATCH 50/92] Introduce Address Usage Map Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 1 + packages/caravan-health/src/privacy.test.ts | 23 ++++++++++++++----- packages/caravan-health/src/privacy.ts | 19 ++++++++-------- packages/caravan-health/src/types.ts | 5 +++++ packages/caravan-health/src/wallet.ts | 25 +++++++++++++++++---- 5 files changed, 54 insertions(+), 19 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index e94130d5..a9e51e42 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -59,3 +59,4 @@ Considers the number of UTXOs. - [] Expand the test cases for privacy and waste metrics to cover every possible case. - [] Add links to each algorithm and the corresponding explanation in final research document. +- [] Add test cases for AddressReuseMap diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 69d07fbf..9d6804b6 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -117,6 +117,7 @@ const utxos: AddressUtxos = { describe("Waste metric scoring", () => { const privacyMetric = new PrivacyMetrics(); + const addressUsageMap = privacyMetric.constructAddressUsageMap(transactions); describe("Determine Spend Type", () => { it("Perfect Spend are transactions with 1 input and 1 output", () => { @@ -314,26 +315,36 @@ describe("Waste metric scoring", () => { describe("Transaction Topology Score", () => { it("Calculates the transaction topology score based on the spend type", () => { - const score: number = privacyMetric.getTopologyScore(transactions[0]); + const score: number = privacyMetric.getTopologyScore( + transactions[0], + addressUsageMap, + ); expect(score).toBe(0.75); - const score2: number = privacyMetric.getTopologyScore(transactions[1]); + const score2: number = privacyMetric.getTopologyScore( + transactions[1], + addressUsageMap, + ); expect(score2).toBeCloseTo(0.67); }); }); describe("Mean Topology Score", () => { it("Calculates the mean topology score for all transactions done by a wallet", () => { - const meanScore: number = - privacyMetric.getMeanTopologyScore(transactions); + const meanScore: number = privacyMetric.getMeanTopologyScore( + transactions, + addressUsageMap, + ); expect(meanScore).toBeCloseTo(0.71); }); }); describe("Address Reuse Factor", () => { it("Calculates the amount being held by reused addresses with respect to the total amount", () => { - const addressReuseFactor: number = - privacyMetric.addressReuseFactor(utxos); + const addressReuseFactor: number = privacyMetric.addressReuseFactor( + utxos, + addressUsageMap, + ); expect(addressReuseFactor).toBe(0); }); }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 4ae4af13..b8ca2080 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,5 +1,5 @@ import { Transaction } from "@caravan/clients"; -import { AddressUtxos, SpendType } from "./types"; +import { AddressUtxos, SpendType, AddressUsageMap } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; import { WalletMetrics} from "./wallet"; @@ -91,7 +91,7 @@ export class PrivacyMetrics extends WalletMetrics { - Consolidation (more than 1 input, 1 output) - CoinJoin or Mixing (more than 1 input, more than 1 output) */ - getTopologyScore(transaction: Transaction): number { + getTopologyScore(transaction: Transaction, addressUsageMap: AddressUsageMap): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -110,7 +110,7 @@ export class PrivacyMetrics extends WalletMetrics { } for (let output of transaction.vout) { let address = output.scriptPubkeyAddress; - let isResued = this.isReusedAddress(address); + let isResued = this.isReusedAddress(address, addressUsageMap); if (isResued === true) { return score; } @@ -136,10 +136,10 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : (0.45, 0.6] -> Very Good : (0.6, 0.75) */ - getMeanTopologyScore(transactions: Transaction[]): number { + getMeanTopologyScore(transactions: Transaction[], addressUsageMap: AddressUsageMap): number { let privacyScore = 0; for (let tx of transactions) { - let topologyScore = this.getTopologyScore(tx); + let topologyScore = this.getTopologyScore(tx, addressUsageMap); privacyScore += topologyScore; } return privacyScore / transactions.length; @@ -163,7 +163,7 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : [0.2, 0.4) -> Very Good : [0 ,0.2) */ - addressReuseFactor(utxos: AddressUtxos): number { + addressReuseFactor(utxos: AddressUtxos, addressUsageMap: AddressUsageMap): number { let reusedAmount: number = 0; let totalAmount: number = 0; @@ -171,7 +171,7 @@ export class PrivacyMetrics extends WalletMetrics { const addressUtxos = utxos[address]; for (const utxo of addressUtxos) { totalAmount += utxo.value; - let isReused = this.isReusedAddress(address); + let isReused = this.isReusedAddress(address, addressUsageMap); if (isReused) { reusedAmount += utxo.value; } @@ -317,8 +317,9 @@ export class PrivacyMetrics extends WalletMetrics { walletAddressType: MultisigAddressType, network: Network, ): number { - let meanTopologyScore = this.getMeanTopologyScore(transactions); - let ARF = this.addressReuseFactor(utxos); + let addressUsageMap = this.constructAddressUsageMap(transactions); + let meanTopologyScore = this.getMeanTopologyScore(transactions, addressUsageMap); + let ARF = this.addressReuseFactor(utxos, addressUsageMap); let ATF = this.addressTypeFactor(transactions, walletAddressType, network); let UVDF = this.utxoValueDispersionFactor(utxos); diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index 505ef9d6..bfebc039 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -23,3 +23,8 @@ export enum SpendType { Consolidation = "Consolidation", MixingOrCoinJoin = "MixingOrCoinJoin", } + +// Represents the usage of the address in past transactions +export interface AddressUsageMap { + [address:string]: boolean; +} diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index b40859d4..dd7ec151 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -1,5 +1,5 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { AddressUtxos } from "./types"; +import { AddressUtxos, AddressUsageMap } from "./types"; export class WalletMetrics { /* @@ -104,11 +104,28 @@ export class WalletMetrics { } } + constructAddressUsageMap(transactions: Transaction[]): AddressUsageMap { + let addressUsageMap: AddressUsageMap = {}; + for (const tx of transactions) { + for (const output of tx.vout) { + let address = output.scriptPubkeyAddress; + if (addressUsageMap[address]) { + addressUsageMap[address] = true; + } else { + addressUsageMap[address] = false; + } + } + } + return addressUsageMap; + } + /* - Utility function to check if the given address was used alreadyin past transactions + Utility function to check if the given address was used already in past transactions */ - isReusedAddress(address: string): boolean { - // TODO : Implement a function to check if the address is reused + isReusedAddress(address: string, addressUsageMap: AddressUsageMap): boolean { + if (addressUsageMap[address]) { + return true; + } return false; } } From a4eafb72671db4f1b1a7c7998a46fdba503d199a Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 13 Aug 2024 03:02:52 +0530 Subject: [PATCH 51/92] solve blocktime and block_time naming inconsistency Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 4 ++-- packages/caravan-clients/src/types.ts | 2 +- packages/caravan-health/src/privacy.test.ts | 4 ++-- packages/caravan-health/src/wallet.test.ts | 4 ++-- packages/caravan-health/src/waste.test.ts | 4 ++-- packages/caravan-health/src/waste.ts | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 7cb845d4..0e1e5f12 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -188,7 +188,7 @@ export class BlockchainClient extends ClientBase { fee: tx.fee, isSend: tx.category === "send" ? true : false, amount: tx.amount, - blocktime: tx.blocktime, + block_time: tx.blocktime, }; for (const input of rawTxData.vin) { transaction.vin.push({ @@ -223,7 +223,7 @@ export class BlockchainClient extends ClientBase { fee: tx.fee, isSend: false, amount: 0, - blocktime: tx.status.block_time, + block_time: tx.status.block_time, }; for (const input of tx.vin) { diff --git a/packages/caravan-clients/src/types.ts b/packages/caravan-clients/src/types.ts index 7a00f08e..a52f3142 100644 --- a/packages/caravan-clients/src/types.ts +++ b/packages/caravan-clients/src/types.ts @@ -17,7 +17,7 @@ export interface Transaction { fee: number; isSend: boolean; amount: number; - blocktime: number; + block_time: number; } interface Input { diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 9d6804b6..dca4d6ed 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -26,7 +26,7 @@ const transactions: Transaction[] = [ fee: 0, isSend: true, amount: 0, - blocktime: 0, + block_time: 0, }, // transactions[1] is a coin join transaction { @@ -70,7 +70,7 @@ const transactions: Transaction[] = [ fee: 0, isSend: true, amount: 0, - blocktime: 0, + block_time: 0, }, ]; diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index c34fe82b..21484b6a 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -25,7 +25,7 @@ const transactions: Transaction[] = [ fee: 1, isSend: true, amount: 1, - blocktime: 1234, + block_time: 1234, }, // transactions[1] is a coin join transaction { @@ -69,7 +69,7 @@ const transactions: Transaction[] = [ fee: 0, isSend: true, amount: 0, - blocktime: 0, + block_time: 0, }, ]; diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 5f3e2c87..7069c466 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -11,7 +11,7 @@ const transactions = [ fee: 1, // Fee paid in the transaction isSend: true, // Transaction is a send transaction amount: 10, // Amount spent in the transaction - blocktime: 1234, // Blocktime of the block where the transactions were mined + block_time: 1234, // Blocktime of the block where the transactions were mined }, { vin: [], @@ -22,7 +22,7 @@ const transactions = [ fee: 1, isSend: false, amount: 10, - blocktime: 1234, + block_time: 1234, }, ]; diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 41617226..81ebdfd7 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -33,7 +33,7 @@ export class WasteMetrics extends WalletMetrics { numberOfSendTx++; let feeRate: number = this.getFeeRateForTransaction(tx); let RFS: number = this.getFeeRatePercentileScore( - tx.blocktime, + tx.block_time, feeRate, feeRatePercentileHistory, ); From 97a80938156cad51db68587b2ba28826d150f3f8 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 14 Aug 2024 00:45:44 +0530 Subject: [PATCH 52/92] Cache addressUsageMap in the class Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 13 +++++-------- packages/caravan-health/src/privacy.ts | 21 ++++++++++++--------- packages/caravan-health/src/wallet.test.ts | 2 +- packages/caravan-health/src/wallet.ts | 9 +++++++-- packages/caravan-health/src/waste.test.ts | 2 +- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index dca4d6ed..dd1c58fd 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -115,8 +115,8 @@ const utxos: AddressUtxos = { ], }; -describe("Waste metric scoring", () => { - const privacyMetric = new PrivacyMetrics(); +describe("Privacy metric scoring", () => { + const privacyMetric = new PrivacyMetrics(transactions); const addressUsageMap = privacyMetric.constructAddressUsageMap(transactions); describe("Determine Spend Type", () => { @@ -316,14 +316,12 @@ describe("Waste metric scoring", () => { describe("Transaction Topology Score", () => { it("Calculates the transaction topology score based on the spend type", () => { const score: number = privacyMetric.getTopologyScore( - transactions[0], - addressUsageMap, + transactions[0] ); expect(score).toBe(0.75); const score2: number = privacyMetric.getTopologyScore( - transactions[1], - addressUsageMap, + transactions[1] ); expect(score2).toBeCloseTo(0.67); }); @@ -332,8 +330,7 @@ describe("Waste metric scoring", () => { describe("Mean Topology Score", () => { it("Calculates the mean topology score for all transactions done by a wallet", () => { const meanScore: number = privacyMetric.getMeanTopologyScore( - transactions, - addressUsageMap, + transactions ); expect(meanScore).toBeCloseTo(0.71); }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index b8ca2080..fed3f8fd 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,7 +1,7 @@ import { Transaction } from "@caravan/clients"; import { AddressUtxos, SpendType, AddressUsageMap } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; -import { WalletMetrics} from "./wallet"; +import { WalletMetrics } from "./wallet"; // Deniability Factor is a normalizing quantity that increases the score by a certain factor in cases of self-payment. // More about deniability : https://www.truthcoin.info/blog/deniability/ @@ -91,7 +91,7 @@ export class PrivacyMetrics extends WalletMetrics { - Consolidation (more than 1 input, 1 output) - CoinJoin or Mixing (more than 1 input, more than 1 output) */ - getTopologyScore(transaction: Transaction, addressUsageMap: AddressUsageMap): number { + getTopologyScore(transaction: Transaction): number { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; @@ -110,7 +110,7 @@ export class PrivacyMetrics extends WalletMetrics { } for (let output of transaction.vout) { let address = output.scriptPubkeyAddress; - let isResued = this.isReusedAddress(address, addressUsageMap); + let isResued = this.isReusedAddress(address); if (isResued === true) { return score; } @@ -136,10 +136,10 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : (0.45, 0.6] -> Very Good : (0.6, 0.75) */ - getMeanTopologyScore(transactions: Transaction[], addressUsageMap: AddressUsageMap): number { + getMeanTopologyScore(transactions: Transaction[]): number { let privacyScore = 0; for (let tx of transactions) { - let topologyScore = this.getTopologyScore(tx, addressUsageMap); + let topologyScore = this.getTopologyScore(tx); privacyScore += topologyScore; } return privacyScore / transactions.length; @@ -163,7 +163,10 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : [0.2, 0.4) -> Very Good : [0 ,0.2) */ - addressReuseFactor(utxos: AddressUtxos, addressUsageMap: AddressUsageMap): number { + addressReuseFactor( + utxos: AddressUtxos, + addressUsageMap: AddressUsageMap, + ): number { let reusedAmount: number = 0; let totalAmount: number = 0; @@ -171,7 +174,7 @@ export class PrivacyMetrics extends WalletMetrics { const addressUtxos = utxos[address]; for (const utxo of addressUtxos) { totalAmount += utxo.value; - let isReused = this.isReusedAddress(address, addressUsageMap); + let isReused = this.isReusedAddress(address); if (isReused) { reusedAmount += utxo.value; } @@ -318,7 +321,7 @@ export class PrivacyMetrics extends WalletMetrics { network: Network, ): number { let addressUsageMap = this.constructAddressUsageMap(transactions); - let meanTopologyScore = this.getMeanTopologyScore(transactions, addressUsageMap); + let meanTopologyScore = this.getMeanTopologyScore(transactions); let ARF = this.addressReuseFactor(utxos, addressUsageMap); let ATF = this.addressTypeFactor(transactions, walletAddressType, network); let UVDF = this.utxoValueDispersionFactor(utxos); @@ -329,4 +332,4 @@ export class PrivacyMetrics extends WalletMetrics { return WPS; } -} +} diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index 21484b6a..23323cf3 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -115,7 +115,7 @@ const utxos: AddressUtxos = { }; describe("Wallet Metrics", () => { - const walletMetrics = new WalletMetrics(); + const walletMetrics = new WalletMetrics(transactions); describe("UTXO Mass Factor", () => { it("should return 1 for UTXO set length = 4", () => { expect(walletMetrics.utxoMassFactor(utxos)).toBe(1); diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index dd7ec151..1b1e4b8e 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -2,6 +2,11 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { AddressUtxos, AddressUsageMap } from "./types"; export class WalletMetrics { + private addressUsageMap: AddressUsageMap; + + constructor(transactions: Transaction[]) { + this.addressUsageMap = this.constructAddressUsageMap(transactions); + } /* Name : UTXO Mass Factor @@ -122,8 +127,8 @@ export class WalletMetrics { /* Utility function to check if the given address was used already in past transactions */ - isReusedAddress(address: string, addressUsageMap: AddressUsageMap): boolean { - if (addressUsageMap[address]) { + isReusedAddress(address: string): boolean { + if (this.addressUsageMap[address]) { return true; } return false; diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 7069c466..183a099e 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -82,7 +82,7 @@ const utxos: AddressUtxos = { }; describe("Waste metric scoring", () => { - const wasteMetric = new WasteMetrics(); + const wasteMetric = new WasteMetrics(transactions); describe("Relative Fees Score (R.F.S)", () => { it("calculates fee score based on tx fee rate relative to percentile in the block where a set of send tx were mined", () => { From ac28f304cf2a4ea8b3d92bacc51a7019a0132d3a Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 14 Aug 2024 00:50:19 +0530 Subject: [PATCH 53/92] raise exceptions for other clients if not mempool for feerate history Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 0e1e5f12..bea3c3fa 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -404,6 +404,14 @@ export class BlockchainClient extends ClientBase { FeeRatePercentile[] > { try { + if ( + this.type === ClientType.PRIVATE || + this.type === ClientType.BLOCKSTREAM + ) { + throw new Error( + "Not supported for private clients and blockstream. Currently only supported for mempool", + ); + } let data = await this.Get(`/v1/mining/blocks/fee-rates/all`); let feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { From 1785c8f9eb9f103aed0802ff9e3e02a27c0dc975 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 15 Aug 2024 05:22:35 +0530 Subject: [PATCH 54/92] Adding test cases for clients methods Signed-off-by: Harshil-Jani --- package-lock.json | 5 +- packages/caravan-clients/src/client.test.ts | 197 +++++++++++++++++++- packages/caravan-clients/src/client.ts | 7 +- 3 files changed, 202 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index cc1c98d7..17db4fc3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25151,7 +25151,6 @@ "@babel/core": "^7.23.7", "@babel/preset-env": "^7.23.8", "@caravan/eslint-config": "*", - "@caravan/health": "*", "@caravan/typescript-config": "*", "babel-jest": "^29.7.0", "eslint": "^8.56.0", @@ -26013,9 +26012,11 @@ } }, "packages/caravan-health": { - "version": "1.0.0", + "name": "@caravan/health", + "version": "1.0.0-beta", "license": "MIT", "dependencies": { + "@caravan/bitcoin": "*", "@caravan/clients": "*" }, "devDependencies": { diff --git a/packages/caravan-clients/src/client.test.ts b/packages/caravan-clients/src/client.test.ts index f47d7c9c..c8628114 100644 --- a/packages/caravan-clients/src/client.test.ts +++ b/packages/caravan-clients/src/client.test.ts @@ -9,8 +9,8 @@ import * as bitcoind from "./bitcoind"; import * as wallet from "./wallet"; import BigNumber from "bignumber.js"; import { UTXO } from "./types"; - import axios from "axios"; +import { FeeRatePercentile } from "./types"; jest.mock("axios"); describe("ClientBase", () => { @@ -839,7 +839,11 @@ describe("BlockchainClient", () => { const receive = "receive"; const change = "change"; - await blockchainClient.importDescriptors({ receive, change, rescan: true}); + await blockchainClient.importDescriptors({ + receive, + change, + rescan: true, + }); expect(mockImportDescriptors).toHaveBeenCalledWith({ receive, change, @@ -861,7 +865,11 @@ describe("BlockchainClient", () => { const receive = "receive"; const change = "change"; - await blockchainClient.importDescriptors({ receive, change, rescan: false}); + await blockchainClient.importDescriptors({ + receive, + change, + rescan: false, + }); expect(mockImportDescriptors).toHaveBeenCalledWith({ receive, change, @@ -896,4 +904,187 @@ describe("BlockchainClient", () => { }); }); }); + + describe("getAddressTransactions", () => { + it("should get the all the transactions for a given address in PRIVATE network MAINNET", async () => { + // Mock the response from the API + const mockResponseListTransaction = [ + { + address: + "bcrt1qymrajrm0wq5uvwlmj26lxkpchxge7rsf0qt3tfrvpcwcvxsmrp3qq60fu7", + parent_descs: [ + "wsh(sortedmulti(1,tpubDFnYXDztf7GxeGVpPsgYaqbfE6mCsvVzCGKhtafJU3pbF8r8cuGQgp81puJcjuBdsMhk1oUHdhNbsrPcn8SHjktJ45pzJNhAd1BY3jRdzvj/0/*,tpubDDwMB2bTZPY5Usnyqn7PN1cYmNWNghRxtY968LCA2DRr4HM93JqkLd5uEHXQb2rRLjHrkccguYRxyDkQi71mBuZ7XAfLH29918Gu9vKVmhy/0/*))#dw99d0sw", + ], + category: "receive", + amount: 15.0, + label: "", + vout: 0, + confirmations: 22, + blockhash: + "1ab9eed7ff3b824dfdee22560e8fc826f2bac0ca835c992b8659b1c834721ffa", + blockheight: 1181, + blockindex: 1, + blocktime: 1718291897, + txid: "c24617439089a088adb813b5c14238a9354db2f1f6a2224a36a8d7fe095b793d", + wtxid: + "341610613a8fcde8933322dc20f35f2635f37cc926c11001a446f604effb73a4", + walletconflicts: [], + time: 1718291888, + timereceived: 1718291888, + "bip125-replaceable": "no", + }, + ]; + const mockBitcoindListTransaction = jest.spyOn(bitcoind, "callBitcoind"); + mockBitcoindListTransaction.mockResolvedValue( + mockResponseListTransaction, + ); + + const mockBitcoindRawTxData = { + txid: "c24617439089a088adb813b5c14238a9354db2f1f6a2224a36a8d7fe095b793d", + hash: "341610613a8fcde8933322dc20f35f2635f37cc926c11001a446f604effb73a4", + version: 2, + size: 312, + vsize: 212, + weight: 846, + locktime: 1180, + vin: [ + { + txid: "c628cc1cde5ca9adf470c4837ac99d3745a72d9a57a6cffb40e22508627af554", + vout: 1, + scriptSig: { + asm: "", + hex: "", + }, + txinwitness: [ + "23e8ef69bd66165cb1bc41a4354ecc69ee0d92a1b98fcb528f93dd2ae54ea7033c0fdf24e1419705ace5d1bd3d2aba34cccfabde22ec08ed1728c97e6fb85a7b", + ], + sequence: 4294967293, + }, + { + txid: "0dfe7a6df3c7840df8a6f5f74160bb3545d60aa0924eb0a6574f29e3eddb4354", + vout: 0, + scriptSig: { + asm: "", + hex: "", + }, + txinwitness: [ + "b342d6f9d0e75900d7301e3ddf3c386f1f4103596e92bbd36df88d860d2a8631af177d69a47e619fbe1054690865fada18b3695d3160620d716090ae356ddf53", + ], + sequence: 4294967293, + }, + ], + vout: [ + { + value: 15.0, + n: 0, + scriptPubKey: { + asm: "0 26c7d90f6f7029c63bfb92b5f35838b9919f0e09781715a46c0e1d861a1b1862", + desc: "addr(bcrt1qymrajrm0wq5uvwlmj26lxkpchxge7rsf0qt3tfrvpcwcvxsmrp3qq60fu7)#szq6selt", + hex: "002026c7d90f6f7029c63bfb92b5f35838b9919f0e09781715a46c0e1d861a1b1862", + address: + "bcrt1qymrajrm0wq5uvwlmj26lxkpchxge7rsf0qt3tfrvpcwcvxsmrp3qq60fu7", + type: "witness_v0_scripthash", + }, + }, + { + value: 11.561891, + n: 1, + scriptPubKey: { + asm: "1 5a475ace36c3054538843e859a1485bf3dd438924ec4d2d81abf55feeabe5a56", + desc: "rawtr(5a475ace36c3054538843e859a1485bf3dd438924ec4d2d81abf55feeabe5a56)#am899zm3", + hex: "51205a475ace36c3054538843e859a1485bf3dd438924ec4d2d81abf55feeabe5a56", + address: + "bcrt1ptfr44n3kcvz52wyy86ze59y9hu7agwyjfmzd9kq6ha2la647tftq8dhx2a", + type: "witness_v1_taproot", + }, + }, + ], + }; + + const mockBitcoindGetAddressTransactions = jest.spyOn( + bitcoind, + "bitcoindRawTxData", + ); + mockBitcoindGetAddressTransactions.mockResolvedValue( + mockBitcoindRawTxData, + ); + + const blockchainClient = new BlockchainClient({ + type: ClientType.PRIVATE, + network: Network.MAINNET, + }); + + const address = + "bcrt1qymrajrm0wq5uvwlmj26lxkpchxge7rsf0qt3tfrvpcwcvxsmrp3qq60fu7"; + const transactions = + await blockchainClient.getAddressTransactions(address); + const expectedResponse = [ + { + txid: "c24617439089a088adb813b5c14238a9354db2f1f6a2224a36a8d7fe095b793d", + vin: [ + { + prevTxId: + "c628cc1cde5ca9adf470c4837ac99d3745a72d9a57a6cffb40e22508627af554", + vout: 1, + sequence: 4294967293, + }, + { + prevTxId: + "0dfe7a6df3c7840df8a6f5f74160bb3545d60aa0924eb0a6574f29e3eddb4354", + vout: 0, + sequence: 4294967293, + }, + ], + vout: [ + { + scriptPubkeyHex: + "002026c7d90f6f7029c63bfb92b5f35838b9919f0e09781715a46c0e1d861a1b1862", + scriptPubkeyAddress: + "bcrt1qymrajrm0wq5uvwlmj26lxkpchxge7rsf0qt3tfrvpcwcvxsmrp3qq60fu7", + value: 15, + }, + { + scriptPubkeyHex: + "51205a475ace36c3054538843e859a1485bf3dd438924ec4d2d81abf55feeabe5a56", + scriptPubkeyAddress: + "bcrt1ptfr44n3kcvz52wyy86ze59y9hu7agwyjfmzd9kq6ha2la647tftq8dhx2a", + value: 11.561891, + }, + ], + size: 312, + weight: 846, + fee: undefined, + isSend: false, + amount: 15, + block_time: 1718291897, + }, + ]; + + expect(transactions).toEqual(expectedResponse); + }); + }); + + describe("getBlockFeeRatePercentileHistory", () => { + it("should get the fee rate percentiles for a closest blocks' transactions (MEMPOOL client)", async () => { + // Create a new instance of BlockchainClient + const blockchainClient = new BlockchainClient({ + type: ClientType.MEMPOOL, + network: Network.MAINNET, + }); + + const result = await blockchainClient.getBlockFeeRatePercentileHistory(); + const expectedResponse: FeeRatePercentile = { + avgHeight: 0, + timestamp: 1231006505, + avgFee_0: 0, + avgFee_10: 0, + avgFee_25: 0, + avgFee_50: 0, + avgFee_75: 0, + avgFee_90: 0, + avgFee_100: 0, + }; + expect(result[0]).toEqual(expectedResponse); + }); + }); }); diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index bea3c3fa..22dd229f 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -200,7 +200,7 @@ export class BlockchainClient extends ClientBase { for (const output of rawTxData.vout) { transaction.vout.push({ scriptPubkeyHex: output.scriptPubKey.hex, - scriptPubkeyAddress: output.scriptPubKey.addresses[0], + scriptPubkeyAddress: output.scriptPubKey.address, value: output.value, }); } @@ -412,7 +412,10 @@ export class BlockchainClient extends ClientBase { "Not supported for private clients and blockstream. Currently only supported for mempool", ); } - let data = await this.Get(`/v1/mining/blocks/fee-rates/all`); + const response = await fetch( + "https://mempool.space/api/v1/mining/blocks/fee-rates/all", + ); + const data = await response.json(); let feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { let feeRatePercentile: FeeRatePercentile = { From e66b3214fdab12d6b4084596ddff8034d4506b2f Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 15 Aug 2024 15:48:29 +0530 Subject: [PATCH 55/92] Clean test cases Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.test.ts | 66 ++++++++++++++++----- packages/caravan-clients/src/client.ts | 7 +-- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/packages/caravan-clients/src/client.test.ts b/packages/caravan-clients/src/client.test.ts index c8628114..fe77b71f 100644 --- a/packages/caravan-clients/src/client.test.ts +++ b/packages/caravan-clients/src/client.test.ts @@ -1066,25 +1066,63 @@ describe("BlockchainClient", () => { describe("getBlockFeeRatePercentileHistory", () => { it("should get the fee rate percentiles for a closest blocks' transactions (MEMPOOL client)", async () => { - // Create a new instance of BlockchainClient + // Mock the response from the API + const mockResponse = [ + { + avgHeight: 45, + timestamp: 1231605377, + avgFee_0: 0, + avgFee_10: 0, + avgFee_25: 0, + avgFee_50: 0, + avgFee_75: 0, + avgFee_90: 0, + avgFee_100: 0, + }, + ]; + const mockGet = jest.fn().mockResolvedValue(mockResponse); + // Create a new instance of BlockchainClient with a mock axios instance const blockchainClient = new BlockchainClient({ type: ClientType.MEMPOOL, network: Network.MAINNET, }); + blockchainClient.Get = mockGet; - const result = await blockchainClient.getBlockFeeRatePercentileHistory(); - const expectedResponse: FeeRatePercentile = { - avgHeight: 0, - timestamp: 1231006505, - avgFee_0: 0, - avgFee_10: 0, - avgFee_25: 0, - avgFee_50: 0, - avgFee_75: 0, - avgFee_90: 0, - avgFee_100: 0, - }; - expect(result[0]).toEqual(expectedResponse); + // Call the getTransactionHex method + const feeRateHistory = + await blockchainClient.getBlockFeeRatePercentileHistory(); + + // Verify the mock axios instance was called with the correct URL + expect(mockGet).toHaveBeenCalledWith(`/v1/mining/blocks/fee-rates/all`); + + // Verify the returned transaction hex + expect(feeRateHistory).toEqual(mockResponse); + }); + + it("should throw an error when using BLOCKSTREAM or PRIVATE client", async () => { + const mockError = new Error( + "Not supported for private clients and blockstream. Currently only supported for mempool", + ); + + // Create a new instance of BlockchainClient with a mock axios instance + const blockchainClient = new BlockchainClient({ + type: ClientType.PRIVATE, + network: Network.MAINNET, + }); + + let error; + try { + await blockchainClient.getBlockFeeRatePercentileHistory(); + } catch (err) { + error = err; + } + + // Verify the error message + expect(error).toEqual( + new Error( + `Failed to get feerate percentile block: ${mockError.message}`, + ), + ); }); }); }); diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index 22dd229f..e25952cf 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -412,10 +412,9 @@ export class BlockchainClient extends ClientBase { "Not supported for private clients and blockstream. Currently only supported for mempool", ); } - const response = await fetch( - "https://mempool.space/api/v1/mining/blocks/fee-rates/all", - ); - const data = await response.json(); + + const data = await this.Get(`/v1/mining/blocks/fee-rates/all`); + let feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { let feeRatePercentile: FeeRatePercentile = { From 7329b13ffd0e5fb5c7ae7f65d2722b380904bb1b Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 15 Aug 2024 15:52:26 +0530 Subject: [PATCH 56/92] AddressUsageMap test case Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 1 - packages/caravan-health/src/wallet.test.ts | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index a9e51e42..e94130d5 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -59,4 +59,3 @@ Considers the number of UTXOs. - [] Expand the test cases for privacy and waste metrics to cover every possible case. - [] Add links to each algorithm and the corresponding explanation in final research document. -- [] Add test cases for AddressReuseMap diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index 23323cf3..5674c2fa 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -184,4 +184,18 @@ describe("Wallet Metrics", () => { ).toBe(50); }); }); + + describe("Address Reuse Map", () => { + it("should return a map of all the used or unused addresses", () => { + const addressUsageMap = + walletMetrics.constructAddressUsageMap(transactions); + + const expectedMap = { + scriptPubkeyAddress1: false, + scriptPubkeyAddress2: false, + }; + + expect(addressUsageMap).toEqual(expectedMap); + }); + }); }); From e35af1d20331f4d6075cb3d46cf9a64ec4f09647 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 15 Aug 2024 16:23:35 +0530 Subject: [PATCH 57/92] Improved Documentation Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 117 +++++++++++++++++------------- 1 file changed, 68 insertions(+), 49 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index e94130d5..a33b650b 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -1,61 +1,80 @@ # Caravan-Health -The `caravan-health` package is designed to help users maintain the health of their bitcoin wallets. Wallet health is determined by various factors including financial privacy, transaction fees, and the avoidance of dust outputs. This README will guide you through understanding wallet health goals, scoring metrics, and how to use the caravan-health package to achieve optimal wallet health. +The `@caravan/health` package is a toolkit for analyzing and scoring the privacy and fee spending efficiency of Bitcoin transactions and wallets. Wallet health is determined by various factors including financial privacy, transaction fees, and the avoidance of dust outputs. It provides metrics and algorithms to evaluate various aspects of transaction behavior, UTXO management, and fee strategies. # Defining Wallet Health Goals -Different users have diverse needs and preferences which impact their wallet health goals. Some users prioritize financial privacy, others focus on minimizing transaction fees, and some want a healthy wallet without delving into the technical details of UTXOs and transactions. The caravan-health package aims to highlight metrics for wallet health and provide suggestions for improvement. +Different users have diverse needs and preferences which impact their wallet health goals. Some users prioritize financial privacy, others focus on minimizing transaction fees, and some want a healthy wallet without delving into the technical details of UTXOs and transactions. The `@caravan/health ` package aims to highlight metrics for wallet health and provide suggestions for improvement. -# Wallet Health Goals: - -- Protect financial privacy -- Minimize long-term and short-term fee rates -- Avoid creating dust outputs -- Determine when to consolidate and when to conserve UTXOs -- Manage spending frequency and allow simultaneous payments - ---- - -# Scoring Metrics for health analysis +# Features ## Privacy Metrics -1. Reuse Factor (R.F) - -Measures the extent to which addresses are reused. Lower reuse factor improves privacy. - -2. Address Type Factor (A.T.F) - -Assesses privacy based on the diversity of address types used in transactions. - -3. UTXO Spread Factor (U.S.F) - -Evaluates the spread of UTXO values to gauge privacy. Higher spread indicates better privacy. - -4. UTXO Mass Factor score which accounts for Number of UTXOs present in a wallet (U.M.F) - -Considers the number of UTXOs in the wallet. - -5. UTXO Value Dispersion Factor (U.V.D.F) - -Combines the scores of UTXO spread and UTXO mass. - -# Waste Metrics - -1. Relative Fee Score (R.F.S) - -Measures the fee rate compared to historical data. It can be associated with all the transactions and we can give a measure -if any transaction was done at expensive fees or nominal fees. - -2. Fee-to-Amount Percent Score (F.A.P.S) - -Ratio of fees paid to the transaction amount. Lower percentage signifies better fee efficiency. - -3. UTXO Mass Factor on Number of UTXOs (UMF) - -Considers the number of UTXOs. - -## TODOs +The `PrivacyMetrics` class offers tools to assess the privacy of Bitcoin transactions and wallets: + +- **Spend Type Determination :** Categorizes transactions based on input and output patterns. +- **Topology Score :** Evaluates transaction privacy based on input/output structure. +- **Mean Transaction Topology Score :** Calculates the average privacy score across all wallet transactions. +- **Address Reuse Factor (ARF) :** Measures the extent of address reuse within the wallet. +- **Address Type Factor (ATF) :** Evaluates the diversity of address types used in transactions. +- **UTXO Spread Factor :** Assesses the dispersion of UTXO values to gauge traceability resistance. +- **UTXO Value Dispersion Factor :** Combines UTXO spread and mass factors for a comprehensive view. +- **Weighted Privacy Score :** Provides an overall privacy health score for the wallet. + +## Waste Metrics + +The `WasteMetrics` class focuses on transaction fee efficiency and UTXO management: + +- **Relative Fees Score (RFS) :** Compares transaction fees to others in the same block. +- **Fees To Amount Ratio (FAR) :** Evaluates the proportion of fees to transaction amounts. +- **Spend Waste Score (SWS) :** Determines the economic efficiency of spending UTXOs. +- **Weighted Waste Score (WWS) :** Combines various metrics for an overall efficiency score. + +# Dependencies + +This library depends on the `@caravan/clients` and `@caravan/bitcoin` package for type validations and preparing some of the required data for that type. Make sure to install and import it correctly in your project. + +# Usage + +To use the Caravan Health Library, you'll need to import the necessary classes and types + +```javascript +import { + PrivacyMetrics, + WasteMetrics, + AddressUtxos, + SpendType, +} from "@caravan/health"; +import { Transaction, FeeRatePercentile } from "@caravan/clients"; +import { Network, MultisigAddressType } from "@caravan/bitcoin"; + +const transactions : Transaction[] = []; +const utxos: AddressUtxos = {}; +const walletAddressType : MultisigAddressType = "P2SH"; +const network : Network = "mainnet"; +const feeRatePercentileHistory : FeeRatePercentile[] + +// Initialize classes for health analysis +const privacyMetrics = new PrivacyMetrics(transactions); +const wasteMetrics = new WasteMetrics(transactions); + +// For example use metric that calculates overall privacy score +const privacyScore = privacyMetrics.getWalletPrivacyScore( + transactions, + utxos, + walletAddressType, + network, +); + +// For example use metric that calculates overall waste score +const wasteScore = wasteMetrics.weightedWasteScore( + transactions, + utxos, + feeRatePercentileHistory, +); +``` + +# TODOs - [] Expand the test cases for privacy and waste metrics to cover every possible case. - [] Add links to each algorithm and the corresponding explanation in final research document. From bd7bf1e9f0b94aab1deac983306c822c990be95a Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 16 Aug 2024 02:57:05 +0530 Subject: [PATCH 58/92] Fixing package-lock.json Signed-off-by: Harshil-Jani --- package-lock.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 17db4fc3..ec412959 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28476,14 +28476,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "packages/health": { - "version": "1.0.0", - "extraneous": true, - "license": "MIT", - "engines": { - "node": ">=20" - } - }, "packages/multisig": { "name": "@caravan/multisig", "version": "0.0.0", From cf835e6944a3e14f04dfd15ed5a3c7f604735cbe Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 16 Aug 2024 03:16:51 +0530 Subject: [PATCH 59/92] Storing transactions, utxos in a class Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 24 +++++++------- packages/caravan-health/package.json | 4 +-- packages/caravan-health/src/privacy.test.ts | 28 +++++----------- packages/caravan-health/src/privacy.ts | 36 +++++++++------------ packages/caravan-health/src/wallet.test.ts | 7 ++-- packages/caravan-health/src/wallet.ts | 17 ++++++---- packages/caravan-health/src/waste.test.ts | 9 ++---- packages/caravan-health/src/waste.ts | 24 ++++++-------- 8 files changed, 64 insertions(+), 85 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index a33b650b..b0f75c0b 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -12,23 +12,23 @@ Different users have diverse needs and preferences which impact their wallet hea The `PrivacyMetrics` class offers tools to assess the privacy of Bitcoin transactions and wallets: -- **Spend Type Determination :** Categorizes transactions based on input and output patterns. -- **Topology Score :** Evaluates transaction privacy based on input/output structure. -- **Mean Transaction Topology Score :** Calculates the average privacy score across all wallet transactions. -- **Address Reuse Factor (ARF) :** Measures the extent of address reuse within the wallet. -- **Address Type Factor (ATF) :** Evaluates the diversity of address types used in transactions. -- **UTXO Spread Factor :** Assesses the dispersion of UTXO values to gauge traceability resistance. -- **UTXO Value Dispersion Factor :** Combines UTXO spread and mass factors for a comprehensive view. -- **Weighted Privacy Score :** Provides an overall privacy health score for the wallet. +- **Spend Type Determination :** Categorizes transactions based on input and output patterns. +- **Topology Score :** Evaluates transaction privacy based on input/output structure. +- **Mean Transaction Topology Score :** Calculates the average privacy score across all wallet transactions. +- **Address Reuse Factor (ARF) :** Measures the extent of address reuse within the wallet. +- **Address Type Factor (ATF) :** Evaluates the diversity of address types used in transactions. +- **UTXO Spread Factor :** Assesses the dispersion of UTXO values to gauge traceability resistance. +- **UTXO Value Dispersion Factor :** Combines UTXO spread and mass factors for a comprehensive view. +- **Weighted Privacy Score :** Provides an overall privacy health score for the wallet. ## Waste Metrics The `WasteMetrics` class focuses on transaction fee efficiency and UTXO management: -- **Relative Fees Score (RFS) :** Compares transaction fees to others in the same block. -- **Fees To Amount Ratio (FAR) :** Evaluates the proportion of fees to transaction amounts. -- **Spend Waste Score (SWS) :** Determines the economic efficiency of spending UTXOs. -- **Weighted Waste Score (WWS) :** Combines various metrics for an overall efficiency score. +- **Relative Fees Score (RFS) :** Compares transaction fees to others in the same block. +- **Fees To Amount Ratio (FAR) :** Evaluates the proportion of fees to transaction amounts. +- **Spend Waste Score (SWS) :** Determines the economic efficiency of spending UTXOs. +- **Weighted Waste Score (WWS) :** Combines various metrics for an overall efficiency score. # Dependencies diff --git a/packages/caravan-health/package.json b/packages/caravan-health/package.json index ef9a8f14..bad7c39b 100644 --- a/packages/caravan-health/package.json +++ b/packages/caravan-health/package.json @@ -38,7 +38,7 @@ "@caravan/bitcoin": "*" }, "devDependencies": { - "prettier": "^3.2.5" + "@caravan/eslint-config": "*", + "@caravan/typescript-config": "*" } - } diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index dd1c58fd..a0cb7f67 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -116,8 +116,8 @@ const utxos: AddressUtxos = { }; describe("Privacy metric scoring", () => { - const privacyMetric = new PrivacyMetrics(transactions); - const addressUsageMap = privacyMetric.constructAddressUsageMap(transactions); + const privacyMetric = new PrivacyMetrics(transactions, utxos); + const addressUsageMap = privacyMetric.constructAddressUsageMap(); describe("Determine Spend Type", () => { it("Perfect Spend are transactions with 1 input and 1 output", () => { @@ -315,33 +315,24 @@ describe("Privacy metric scoring", () => { describe("Transaction Topology Score", () => { it("Calculates the transaction topology score based on the spend type", () => { - const score: number = privacyMetric.getTopologyScore( - transactions[0] - ); + const score: number = privacyMetric.getTopologyScore(transactions[0]); expect(score).toBe(0.75); - const score2: number = privacyMetric.getTopologyScore( - transactions[1] - ); + const score2: number = privacyMetric.getTopologyScore(transactions[1]); expect(score2).toBeCloseTo(0.67); }); }); describe("Mean Topology Score", () => { it("Calculates the mean topology score for all transactions done by a wallet", () => { - const meanScore: number = privacyMetric.getMeanTopologyScore( - transactions - ); + const meanScore: number = privacyMetric.getMeanTopologyScore(); expect(meanScore).toBeCloseTo(0.71); }); }); describe("Address Reuse Factor", () => { it("Calculates the amount being held by reused addresses with respect to the total amount", () => { - const addressReuseFactor: number = privacyMetric.addressReuseFactor( - utxos, - addressUsageMap, - ); + const addressReuseFactor: number = privacyMetric.addressReuseFactor(); expect(addressReuseFactor).toBe(0); }); }); @@ -349,7 +340,6 @@ describe("Privacy metric scoring", () => { describe("Address Type Factor", () => { it("Calculates the the address type distribution of the wallet transactions", () => { const addressTypeFactor: number = privacyMetric.addressTypeFactor( - transactions, "P2SH", Network.MAINNET, ); @@ -359,7 +349,7 @@ describe("Privacy metric scoring", () => { describe("UTXO Spread Factor", () => { it("Calculates the standard deviation of UTXO values which helps in assessing the dispersion of UTXO values", () => { - const utxoSpreadFactor: number = privacyMetric.utxoSpreadFactor(utxos); + const utxoSpreadFactor: number = privacyMetric.utxoSpreadFactor(); expect(utxoSpreadFactor).toBeCloseTo(0.1); }); }); @@ -367,7 +357,7 @@ describe("Privacy metric scoring", () => { describe("UTXO Value Dispersion Factor", () => { it("Combines UTXO Spread Factor and UTXO Mass Factor", () => { const utxoValueDispersionFactor: number = - privacyMetric.utxoValueDispersionFactor(utxos); + privacyMetric.utxoValueDispersionFactor(); expect(utxoValueDispersionFactor).toBeCloseTo(0.015); }); }); @@ -375,8 +365,6 @@ describe("Privacy metric scoring", () => { describe("Overall Privacy Score", () => { it("Calculates the overall privacy score for a wallet", () => { const privacyScore: number = privacyMetric.getWalletPrivacyScore( - transactions, - utxos, "P2SH", Network.MAINNET, ); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index fed3f8fd..e15751d1 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,5 +1,5 @@ import { Transaction } from "@caravan/clients"; -import { AddressUtxos, SpendType, AddressUsageMap } from "./types"; +import { SpendType } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; import { WalletMetrics } from "./wallet"; @@ -136,8 +136,9 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : (0.45, 0.6] -> Very Good : (0.6, 0.75) */ - getMeanTopologyScore(transactions: Transaction[]): number { + getMeanTopologyScore(): number { let privacyScore = 0; + const transactions = this.transactions; for (let tx of transactions) { let topologyScore = this.getTopologyScore(tx); privacyScore += topologyScore; @@ -163,13 +164,10 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : [0.2, 0.4) -> Very Good : [0 ,0.2) */ - addressReuseFactor( - utxos: AddressUtxos, - addressUsageMap: AddressUsageMap, - ): number { + addressReuseFactor(): number { let reusedAmount: number = 0; let totalAmount: number = 0; - + const utxos = this.utxos; for (const address in utxos) { const addressUtxos = utxos[address]; for (const utxo of addressUtxos) { @@ -206,7 +204,6 @@ export class PrivacyMetrics extends WalletMetrics { */ addressTypeFactor( - transactions: Transaction[], walletAddressType: MultisigAddressType, network: Network, ): number { @@ -218,7 +215,7 @@ export class PrivacyMetrics extends WalletMetrics { UNKNOWN: 0, "P2SH-P2WSH": 0, }; - + const transactions = this.transactions; transactions.forEach((tx) => { tx.vout.forEach((output) => { const addressType = getAddressType(output.scriptPubkeyAddress, network); @@ -257,8 +254,9 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : [0.6, 0.8) -> Very Good : [0.8 ,1] */ - utxoSpreadFactor(utxos: AddressUtxos): number { + utxoSpreadFactor(): number { const amounts: number[] = []; + const utxos = this.utxos; for (const address in utxos) { const addressUtxos = utxos[address]; addressUtxos.forEach((utxo) => { @@ -294,9 +292,9 @@ export class PrivacyMetrics extends WalletMetrics { -> Good : (0, 0.075] -> Very Good : (0.075, 0.15] */ - utxoValueDispersionFactor(utxos: AddressUtxos): number { - let UMF: number = this.utxoMassFactor(utxos); - let USF: number = this.utxoSpreadFactor(utxos); + utxoValueDispersionFactor(): number { + let UMF: number = this.utxoMassFactor(); + let USF: number = this.utxoSpreadFactor(); return (USF + UMF) * 0.15 - 0.15; } @@ -315,16 +313,14 @@ export class PrivacyMetrics extends WalletMetrics { */ getWalletPrivacyScore( - transactions: Transaction[], - utxos: AddressUtxos, walletAddressType: MultisigAddressType, network: Network, ): number { - let addressUsageMap = this.constructAddressUsageMap(transactions); - let meanTopologyScore = this.getMeanTopologyScore(transactions); - let ARF = this.addressReuseFactor(utxos, addressUsageMap); - let ATF = this.addressTypeFactor(transactions, walletAddressType, network); - let UVDF = this.utxoValueDispersionFactor(utxos); + let addressUsageMap = this.constructAddressUsageMap(); + let meanTopologyScore = this.getMeanTopologyScore(); + let ARF = this.addressReuseFactor(); + let ATF = this.addressTypeFactor(walletAddressType, network); + let UVDF = this.utxoValueDispersionFactor(); let WPS: number = (meanTopologyScore * (1 - 0.5 * ARF) + 0.1 * (1 - ARF)) * (1 - ATF) + diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index 5674c2fa..1d8d7686 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -115,10 +115,10 @@ const utxos: AddressUtxos = { }; describe("Wallet Metrics", () => { - const walletMetrics = new WalletMetrics(transactions); + const walletMetrics = new WalletMetrics(transactions, utxos); describe("UTXO Mass Factor", () => { it("should return 1 for UTXO set length = 4", () => { - expect(walletMetrics.utxoMassFactor(utxos)).toBe(1); + expect(walletMetrics.utxoMassFactor()).toBe(1); }); }); @@ -187,8 +187,7 @@ describe("Wallet Metrics", () => { describe("Address Reuse Map", () => { it("should return a map of all the used or unused addresses", () => { - const addressUsageMap = - walletMetrics.constructAddressUsageMap(transactions); + const addressUsageMap = walletMetrics.constructAddressUsageMap(); const expectedMap = { scriptPubkeyAddress1: false, diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index 1b1e4b8e..3bcd85ce 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -2,10 +2,13 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { AddressUtxos, AddressUsageMap } from "./types"; export class WalletMetrics { - private addressUsageMap: AddressUsageMap; - - constructor(transactions: Transaction[]) { - this.addressUsageMap = this.constructAddressUsageMap(transactions); + public addressUsageMap: AddressUsageMap; + public transactions: Transaction[]; + public utxos: AddressUtxos; + constructor(transactions: Transaction[], utxos: AddressUtxos) { + this.transactions = transactions; + this.utxos = utxos; + this.addressUsageMap = this.constructAddressUsageMap(); } /* Name : UTXO Mass Factor @@ -20,8 +23,9 @@ export class WalletMetrics { - 0.75 for UTXO set length >= 5 and <= 14 - 1 for UTXO set length < 5 */ - utxoMassFactor(utxos: AddressUtxos): number { + utxoMassFactor(): number { let utxoSetLength = 0; + const utxos = this.utxos; for (const address in utxos) { const addressUtxos = utxos[address]; utxoSetLength += addressUtxos.length; @@ -109,8 +113,9 @@ export class WalletMetrics { } } - constructAddressUsageMap(transactions: Transaction[]): AddressUsageMap { + constructAddressUsageMap(): AddressUsageMap { let addressUsageMap: AddressUsageMap = {}; + const transactions = this.transactions; for (const tx of transactions) { for (const output of tx.vout) { let address = output.scriptPubkeyAddress; diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 183a099e..448dd558 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -82,12 +82,11 @@ const utxos: AddressUtxos = { }; describe("Waste metric scoring", () => { - const wasteMetric = new WasteMetrics(transactions); + const wasteMetric = new WasteMetrics(transactions, utxos); describe("Relative Fees Score (R.F.S)", () => { it("calculates fee score based on tx fee rate relative to percentile in the block where a set of send tx were mined", () => { const score: number = wasteMetric.relativeFeesScore( - transactions, feeRatePercentileHistory, ); expect(score).toBe(0.5); @@ -96,7 +95,7 @@ describe("Waste metric scoring", () => { describe("Fees to Amount Ratio (F.A.R)", () => { it("Fees paid over total amount spent as ratio for a 'send' type transaction", () => { - const ratio: number = wasteMetric.feesToAmountRatio(transactions); + const ratio: number = wasteMetric.feesToAmountRatio(); expect(ratio).toBe(0.1); }); }); @@ -131,11 +130,9 @@ describe("Waste metric scoring", () => { describe("Weighted Waste Score (W.W.S)", () => { it("calculates the overall waste of the wallet based on the relative fees score, fees to amount ratio and the UTXO mass factor", () => { const score: number = wasteMetric.weightedWasteScore( - transactions, - utxos, feeRatePercentileHistory, ); - expect(score).toBe(0.21); + expect(score).toBe(0.51); }); }); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 81ebdfd7..0bb52c9a 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,6 +1,5 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { AddressUtxos } from "./types"; -import { WalletMetrics } from "../dist"; +import { WalletMetrics } from "./wallet"; export class WasteMetrics extends WalletMetrics { /* @@ -22,12 +21,10 @@ export class WasteMetrics extends WalletMetrics { -> Good : (0.6, 0.8] -> Very Good : (0.8, 1] */ - relativeFeesScore( - transactions: Transaction[], - feeRatePercentileHistory: FeeRatePercentile[], - ): number { + relativeFeesScore(feeRatePercentileHistory: FeeRatePercentile[]): number { let sumRFS: number = 0; let numberOfSendTx: number = 0; + const transactions = this.transactions; for (const tx of transactions) { if (tx.isSend === true) { numberOfSendTx++; @@ -63,9 +60,10 @@ export class WasteMetrics extends WalletMetrics { -> Good : (0.006, 0.001] -> Very Good : (0.001, 0) */ - feesToAmountRatio(transactions: Transaction[]): number { + feesToAmountRatio(): number { let sumFeesToAmountRatio: number = 0; let numberOfSendTx: number = 0; + const transactions = this.transactions; transactions.forEach((tx: Transaction) => { if (tx.isSend === true) { sumFeesToAmountRatio += tx.fee / tx.amount; @@ -145,14 +143,10 @@ export class WasteMetrics extends WalletMetrics { -> Good : (0.6, 0.8] -> Very Good : (0.8, 1] */ - weightedWasteScore( - transactions: Transaction[], - utxos: AddressUtxos, - feeRatePercentileHistory: FeeRatePercentile[], - ): number { - let RFS = this.relativeFeesScore(transactions, feeRatePercentileHistory); - let FAR = this.feesToAmountRatio(transactions); - let UMF = 0; + weightedWasteScore(feeRatePercentileHistory: FeeRatePercentile[]): number { + let RFS = this.relativeFeesScore(feeRatePercentileHistory); + let FAR = this.feesToAmountRatio(); + let UMF = this.utxoMassFactor(); return 0.35 * RFS + 0.35 * FAR + 0.3 * UMF; } } From 5a5ab61acedf316a1c627775b8a899d1e2e52479 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 16 Aug 2024 03:23:12 +0530 Subject: [PATCH 60/92] Separate out utility functions Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 59 ++++++----------- packages/caravan-health/src/privacy.ts | 73 +-------------------- packages/caravan-health/src/types.ts | 2 +- packages/caravan-health/src/utility.ts | 69 +++++++++++++++++++ 4 files changed, 93 insertions(+), 110 deletions(-) create mode 100644 packages/caravan-health/src/utility.ts diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index a0cb7f67..875007ab 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -2,6 +2,7 @@ import { Transaction } from "@caravan/clients"; import { PrivacyMetrics } from "./privacy"; import { AddressUtxos, SpendType } from "./types"; import { Network } from "@caravan/bitcoin"; +import { determineSpendType, getSpendTypeScore } from "./utility"; const transactions: Transaction[] = [ // transactions[0] is a perfect spend transaction @@ -121,45 +122,45 @@ describe("Privacy metric scoring", () => { describe("Determine Spend Type", () => { it("Perfect Spend are transactions with 1 input and 1 output", () => { - const spendType: SpendType = privacyMetric.determineSpendType(1, 1); + const spendType: SpendType = determineSpendType(1, 1); expect(spendType).toBe(SpendType.PerfectSpend); }); it("Simple Spend are transactions with 1 input and 2 outputs", () => { - const spendType: SpendType = privacyMetric.determineSpendType(1, 2); + const spendType: SpendType = determineSpendType(1, 2); expect(spendType).toBe(SpendType.SimpleSpend); }); it("UTXO Fragmentation are transactions with 1 input and more than 2 outputs", () => { - const spendType: SpendType = privacyMetric.determineSpendType(1, 3); + const spendType: SpendType = determineSpendType(1, 3); expect(spendType).toBe(SpendType.UTXOFragmentation); - const spendType2: SpendType = privacyMetric.determineSpendType(1, 4); + const spendType2: SpendType = determineSpendType(1, 4); expect(spendType2).toBe(SpendType.UTXOFragmentation); - const spendType3: SpendType = privacyMetric.determineSpendType(1, 5); + const spendType3: SpendType = determineSpendType(1, 5); expect(spendType3).toBe(SpendType.UTXOFragmentation); }); it("Consolidation transactions have more than 1 inputs and 1 output", () => { - const spendType: SpendType = privacyMetric.determineSpendType(2, 1); + const spendType: SpendType = determineSpendType(2, 1); expect(spendType).toBe(SpendType.Consolidation); - const spendType2: SpendType = privacyMetric.determineSpendType(3, 1); + const spendType2: SpendType = determineSpendType(3, 1); expect(spendType2).toBe(SpendType.Consolidation); - const spendType3: SpendType = privacyMetric.determineSpendType(4, 1); + const spendType3: SpendType = determineSpendType(4, 1); expect(spendType3).toBe(SpendType.Consolidation); }); it("Mixing or CoinJoin transactions have more than 1 inputs and more than 1 outputs", () => { - const spendType: SpendType = privacyMetric.determineSpendType(2, 2); + const spendType: SpendType = determineSpendType(2, 2); expect(spendType).toBe(SpendType.MixingOrCoinJoin); - const spendType2: SpendType = privacyMetric.determineSpendType(2, 3); + const spendType2: SpendType = determineSpendType(2, 3); expect(spendType2).toBe(SpendType.MixingOrCoinJoin); - const spendType3: SpendType = privacyMetric.determineSpendType(3, 2); + const spendType3: SpendType = determineSpendType(3, 2); expect(spendType3).toBe(SpendType.MixingOrCoinJoin); }); }); @@ -179,11 +180,7 @@ describe("Privacy metric scoring", () => { score = 0.5 * (1 - 0) = 0.5 */ it("Perfect Spend has a raw score of 0.5 for external wallet payments", () => { - const score: number = privacyMetric.getSpendTypeScore( - SpendType.PerfectSpend, - 1, - 1, - ); + const score: number = getSpendTypeScore(SpendType.PerfectSpend, 1, 1); expect(score).toBe(0.5); }); /* @@ -205,11 +202,7 @@ describe("Privacy metric scoring", () => { score = 0.67 * (1-0.33) = 0.4489 */ it("Simple Spend has a raw score of 0.44 for external wallet payments", () => { - const score: number = privacyMetric.getSpendTypeScore( - SpendType.SimpleSpend, - 1, - 2, - ); + const score: number = getSpendTypeScore(SpendType.SimpleSpend, 1, 2); expect(score).toBeCloseTo(0.44); }); @@ -238,7 +231,7 @@ describe("Privacy metric scoring", () => { */ it("UTXO Fragmentation has a raw score of 0.33 for external wallet payments", () => { - const score: number = privacyMetric.getSpendTypeScore( + const score: number = getSpendTypeScore( SpendType.UTXOFragmentation, 1, 3, @@ -259,18 +252,10 @@ describe("Privacy metric scoring", () => { score = 1 / Number of Inputs */ it("Consolidation has raw score of ", () => { - const score: number = privacyMetric.getSpendTypeScore( - SpendType.Consolidation, - 2, - 1, - ); + const score: number = getSpendTypeScore(SpendType.Consolidation, 2, 1); expect(score).toBeCloseTo(0.5); - const score2: number = privacyMetric.getSpendTypeScore( - SpendType.Consolidation, - 3, - 1, - ); + const score2: number = getSpendTypeScore(SpendType.Consolidation, 3, 1); expect(score2).toBeCloseTo(0.33); }); @@ -290,21 +275,17 @@ describe("Privacy metric scoring", () => { score = 1/2 * (y2/x)/(1+y2/x) */ it("MixingOrCoinJoin has raw score of ", () => { - const score: number = privacyMetric.getSpendTypeScore( - SpendType.MixingOrCoinJoin, - 2, - 2, - ); + const score: number = getSpendTypeScore(SpendType.MixingOrCoinJoin, 2, 2); expect(score).toBeCloseTo(0.33); - const score2: number = privacyMetric.getSpendTypeScore( + const score2: number = getSpendTypeScore( SpendType.MixingOrCoinJoin, 2, 3, ); expect(score2).toBeCloseTo(0.409); - const score3: number = privacyMetric.getSpendTypeScore( + const score3: number = getSpendTypeScore( SpendType.MixingOrCoinJoin, 3, 2, diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index e15751d1..4a2b7fc5 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -2,80 +2,13 @@ import { Transaction } from "@caravan/clients"; import { SpendType } from "./types"; import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; import { WalletMetrics } from "./wallet"; +import { determineSpendType, getSpendTypeScore } from "./utility"; // Deniability Factor is a normalizing quantity that increases the score by a certain factor in cases of self-payment. // More about deniability : https://www.truthcoin.info/blog/deniability/ const DENIABILITY_FACTOR = 1.5; export class PrivacyMetrics extends WalletMetrics { - /* - Name : Spend Type Determination - - Definition : - The type of spend transaction is obtained based on the number of inputs and outputs which - influence the topology type of the transaction and has a role in determining the fingerprints - behind privacy for wallets. - - Calculation : - We have 5 categories of transaction type each with their own impact on privacy score - - Perfect Spend (1 input, 1 output) - - Simple Spend (1 input, 2 outputs) - - UTXO Fragmentation (1 input, more than 2 standard outputs) - - Consolidation (more than 1 input, 1 output) - - CoinJoin or Mixing (more than 1 input, more than 1 output) - */ - determineSpendType(inputs: number, outputs: number): SpendType { - if (inputs === 1) { - if (outputs === 1) return SpendType.PerfectSpend; - if (outputs === 2) return SpendType.SimpleSpend; - return SpendType.UTXOFragmentation; - } else { - if (outputs === 1) return SpendType.Consolidation; - return SpendType.MixingOrCoinJoin; - } - } - - /* - Name : Spend Type Score - Definition : - Statistical derivations are used to calculate the score based on the spend type of the transaction. - - Calculation : - - Perfect Spend : P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) - - Simple Spend : P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) - - UTXO Fragmentation : 2/3 - 1/number of outputs - - Consolidation : 1/number of inputs - - Mixing or CoinJoin : (2/3) * (number of outputs^2) / number of inputs * (1 + (number of outputs^2) / number of inputs) - - Expected Range : [0,0.85] - -> Very Poor : [0, 0.15] - -> Poor : (0.15, 0.3] - -> Moderate : (0.3, 0.45] - -> Good : (0.45, 0.6] - -> Very Good : (0.6, 0.85] - */ - getSpendTypeScore( - spendType: SpendType, - numberOfInputs: number, - numberOfOutputs: number, - ): number { - switch (spendType) { - case SpendType.PerfectSpend: - return 1 / 2; - case SpendType.SimpleSpend: - return 4 / 9; - case SpendType.UTXOFragmentation: - return 2 / 3 - 1 / numberOfOutputs; - case SpendType.Consolidation: - return 1 / numberOfInputs; - case SpendType.MixingOrCoinJoin: - let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; - return ((1 / 2) * x) / (1 + x); - default: - throw new Error("Invalid spend type"); - } - } - /* Name : Topology Score @@ -95,11 +28,11 @@ export class PrivacyMetrics extends WalletMetrics { const numberOfInputs: number = transaction.vin.length; const numberOfOutputs: number = transaction.vout.length; - const spendType: SpendType = this.determineSpendType( + const spendType: SpendType = determineSpendType( numberOfInputs, numberOfOutputs, ); - const score: number = this.getSpendTypeScore( + const score: number = getSpendTypeScore( spendType, numberOfInputs, numberOfOutputs, diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index bfebc039..9d86596f 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -26,5 +26,5 @@ export enum SpendType { // Represents the usage of the address in past transactions export interface AddressUsageMap { - [address:string]: boolean; + [address: string]: boolean; } diff --git a/packages/caravan-health/src/utility.ts b/packages/caravan-health/src/utility.ts new file mode 100644 index 00000000..8dddef67 --- /dev/null +++ b/packages/caravan-health/src/utility.ts @@ -0,0 +1,69 @@ +import { SpendType } from "./types"; + +/* + Name : Spend Type Determination + + Definition : + The type of spend transaction is obtained based on the number of inputs and outputs which + influence the topology type of the transaction and has a role in determining the fingerprints + behind privacy for wallets. + + Calculation : + We have 5 categories of transaction type each with their own impact on privacy score + - Perfect Spend (1 input, 1 output) + - Simple Spend (1 input, 2 outputs) + - UTXO Fragmentation (1 input, more than 2 standard outputs) + - Consolidation (more than 1 input, 1 output) + - CoinJoin or Mixing (more than 1 input, more than 1 output) + */ +export function determineSpendType(inputs: number, outputs: number): SpendType { + if (inputs === 1) { + if (outputs === 1) return SpendType.PerfectSpend; + if (outputs === 2) return SpendType.SimpleSpend; + return SpendType.UTXOFragmentation; + } else { + if (outputs === 1) return SpendType.Consolidation; + return SpendType.MixingOrCoinJoin; + } +} + +/* + Name : Spend Type Score + Definition : + Statistical derivations are used to calculate the score based on the spend type of the transaction. + + Calculation : + - Perfect Spend : P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) + - Simple Spend : P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) + - UTXO Fragmentation : 2/3 - 1/number of outputs + - Consolidation : 1/number of inputs + - Mixing or CoinJoin : (2/3) * (number of outputs^2) / number of inputs * (1 + (number of outputs^2) / number of inputs) + + Expected Range : [0,0.85] + -> Very Poor : [0, 0.15] + -> Poor : (0.15, 0.3] + -> Moderate : (0.3, 0.45] + -> Good : (0.45, 0.6] + -> Very Good : (0.6, 0.85] + */ +export function getSpendTypeScore( + spendType: SpendType, + numberOfInputs: number, + numberOfOutputs: number, +): number { + switch (spendType) { + case SpendType.PerfectSpend: + return 1 / 2; + case SpendType.SimpleSpend: + return 4 / 9; + case SpendType.UTXOFragmentation: + return 2 / 3 - 1 / numberOfOutputs; + case SpendType.Consolidation: + return 1 / numberOfInputs; + case SpendType.MixingOrCoinJoin: + let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; + return ((1 / 2) * x) / (1 + x); + default: + throw new Error("Invalid spend type"); + } +} From 2ee031de038f5533e464aa7677640c169da07616 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 16 Aug 2024 03:25:26 +0530 Subject: [PATCH 61/92] Updating documentation Signed-off-by: Harshil-Jani --- package-lock.json | 3 ++- packages/caravan-health/README.md | 31 +++++++++++++------------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index ec412959..1f56c442 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26020,7 +26020,8 @@ "@caravan/clients": "*" }, "devDependencies": { - "prettier": "^3.2.5" + "@caravan/eslint-config": "*", + "@caravan/typescript-config": "*" }, "engines": { "node": ">=20" diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index b0f75c0b..d6baec85 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -12,23 +12,22 @@ Different users have diverse needs and preferences which impact their wallet hea The `PrivacyMetrics` class offers tools to assess the privacy of Bitcoin transactions and wallets: -- **Spend Type Determination :** Categorizes transactions based on input and output patterns. -- **Topology Score :** Evaluates transaction privacy based on input/output structure. -- **Mean Transaction Topology Score :** Calculates the average privacy score across all wallet transactions. -- **Address Reuse Factor (ARF) :** Measures the extent of address reuse within the wallet. -- **Address Type Factor (ATF) :** Evaluates the diversity of address types used in transactions. -- **UTXO Spread Factor :** Assesses the dispersion of UTXO values to gauge traceability resistance. -- **UTXO Value Dispersion Factor :** Combines UTXO spread and mass factors for a comprehensive view. -- **Weighted Privacy Score :** Provides an overall privacy health score for the wallet. +- **Topology Score :** Evaluates transaction privacy based on input/output structure. +- **Mean Transaction Topology Score :** Calculates the average privacy score across all wallet transactions. +- **Address Reuse Factor (ARF) :** Measures the extent of address reuse within the wallet. +- **Address Type Factor (ATF) :** Evaluates the diversity of address types used in transactions. +- **UTXO Spread Factor :** Assesses the dispersion of UTXO values to gauge traceability resistance. +- **UTXO Value Dispersion Factor :** Combines UTXO spread and mass factors for a comprehensive view. +- **Weighted Privacy Score :** Provides an overall privacy health score for the wallet. ## Waste Metrics The `WasteMetrics` class focuses on transaction fee efficiency and UTXO management: -- **Relative Fees Score (RFS) :** Compares transaction fees to others in the same block. -- **Fees To Amount Ratio (FAR) :** Evaluates the proportion of fees to transaction amounts. -- **Spend Waste Score (SWS) :** Determines the economic efficiency of spending UTXOs. -- **Weighted Waste Score (WWS) :** Combines various metrics for an overall efficiency score. +- **Relative Fees Score (RFS) :** Compares transaction fees to others in the same block. +- **Fees To Amount Ratio (FAR) :** Evaluates the proportion of fees to transaction amounts. +- **Spend Waste Score (SWS) :** Determines the economic efficiency of spending UTXOs. +- **Weighted Waste Score (WWS) :** Combines various metrics for an overall efficiency score. # Dependencies @@ -55,21 +54,17 @@ const network : Network = "mainnet"; const feeRatePercentileHistory : FeeRatePercentile[] // Initialize classes for health analysis -const privacyMetrics = new PrivacyMetrics(transactions); -const wasteMetrics = new WasteMetrics(transactions); +const privacyMetrics = new PrivacyMetrics(transactions,utxos); +const wasteMetrics = new WasteMetrics(transactions,utxos); // For example use metric that calculates overall privacy score const privacyScore = privacyMetrics.getWalletPrivacyScore( - transactions, - utxos, walletAddressType, network, ); // For example use metric that calculates overall waste score const wasteScore = wasteMetrics.weightedWasteScore( - transactions, - utxos, feeRatePercentileHistory, ); ``` From 9781cc2972f893709bd058ed21a003f76f8663a2 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 19 Aug 2024 03:00:01 +0530 Subject: [PATCH 62/92] Adding the new UTXO dust method and updating address lookup Map Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 23 +++++++------- packages/caravan-health/src/privacy.test.ts | 4 +-- packages/caravan-health/src/wallet.test.ts | 33 ++++++++++----------- packages/caravan-health/src/wallet.ts | 20 ++++++------- packages/caravan-health/src/waste.ts | 31 +++++++++++++++++++ 5 files changed, 71 insertions(+), 40 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index d6baec85..37576b8a 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -12,22 +12,23 @@ Different users have diverse needs and preferences which impact their wallet hea The `PrivacyMetrics` class offers tools to assess the privacy of Bitcoin transactions and wallets: -- **Topology Score :** Evaluates transaction privacy based on input/output structure. -- **Mean Transaction Topology Score :** Calculates the average privacy score across all wallet transactions. -- **Address Reuse Factor (ARF) :** Measures the extent of address reuse within the wallet. -- **Address Type Factor (ATF) :** Evaluates the diversity of address types used in transactions. -- **UTXO Spread Factor :** Assesses the dispersion of UTXO values to gauge traceability resistance. -- **UTXO Value Dispersion Factor :** Combines UTXO spread and mass factors for a comprehensive view. -- **Weighted Privacy Score :** Provides an overall privacy health score for the wallet. +- **Topology Score:** Evaluates transaction privacy based on input/output structure. +- **Mean Transaction Topology Score:** Calculates the average privacy score across all wallet transactions. +- **Address Reuse Factor (ARF):** Measures the extent of address reuse within the wallet. +- **Address Type Factor (ATF):** Evaluates the diversity of address types used in transactions. +- **UTXO Spread Factor:** Assesses the dispersion of UTXO values to gauge traceability resistance. +- **UTXO Value Dispersion Factor:** Combines UTXO spread and mass factors for a comprehensive view. +- **Weighted Privacy Score:** Provides an overall privacy health score for the wallet. ## Waste Metrics The `WasteMetrics` class focuses on transaction fee efficiency and UTXO management: -- **Relative Fees Score (RFS) :** Compares transaction fees to others in the same block. -- **Fees To Amount Ratio (FAR) :** Evaluates the proportion of fees to transaction amounts. -- **Spend Waste Score (SWS) :** Determines the economic efficiency of spending UTXOs. -- **Weighted Waste Score (WWS) :** Combines various metrics for an overall efficiency score. +- **Relative Fees Score (RFS):** Compares transaction fees to others in the same block. +- **Fees To Amount Ratio (FAR):** Evaluates the proportion of fees to transaction amounts. +- **calculateDustLimits:** Calculates the dust limits for UTXOs based on the current fee rate. +- **Spend Waste Score (SWS):** Determines the economic efficiency of spending UTXOs. +- **Weighted Waste Score (WWS):** Combines various metrics for an overall efficiency score. # Dependencies diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 875007ab..c7195f89 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -300,14 +300,14 @@ describe("Privacy metric scoring", () => { expect(score).toBe(0.75); const score2: number = privacyMetric.getTopologyScore(transactions[1]); - expect(score2).toBeCloseTo(0.67); + expect(score2).toBeCloseTo(0.44); }); }); describe("Mean Topology Score", () => { it("Calculates the mean topology score for all transactions done by a wallet", () => { const meanScore: number = privacyMetric.getMeanTopologyScore(); - expect(meanScore).toBeCloseTo(0.71); + expect(meanScore).toBeCloseTo(0.597); }); }); diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index 1d8d7686..56202a03 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -32,7 +32,7 @@ const transactions: Transaction[] = [ txid: "txid2", vin: [ { - prevTxId: "prevTxId2", + prevTxId: "txid1", vout: 0, sequence: 0, }, @@ -49,18 +49,8 @@ const transactions: Transaction[] = [ value: 0.2, }, { - scriptPubkeyHex: "scriptPubkeyHex2", - scriptPubkeyAddress: "scriptPubkeyAddress2", - value: 0.2, - }, - { - scriptPubkeyHex: "scriptPubkeyHex2", - scriptPubkeyAddress: "scriptPubkeyAddress2", - value: 0.2, - }, - { - scriptPubkeyHex: "scriptPubkeyHex2", - scriptPubkeyAddress: "scriptPubkeyAddress2", + scriptPubkeyHex: "scriptPubkeyHex3", + scriptPubkeyAddress: "scriptPubkeyAddress1", value: 0.2, }, ], @@ -189,12 +179,21 @@ describe("Wallet Metrics", () => { it("should return a map of all the used or unused addresses", () => { const addressUsageMap = walletMetrics.constructAddressUsageMap(); - const expectedMap = { - scriptPubkeyAddress1: false, - scriptPubkeyAddress2: false, - }; + const expectedMap = new Map(); + expectedMap.set("scriptPubkeyAddress1", 2); + expectedMap.set("scriptPubkeyAddress2", 1); expect(addressUsageMap).toEqual(expectedMap); }); }); + + describe("is Address Reused", () => { + it("should return true for reused address", () => { + expect(walletMetrics.isReusedAddress("scriptPubkeyAddress1")).toBe(true); + }); + + it("should return false for unused address", () => { + expect(walletMetrics.isReusedAddress("scriptPubkeyAddress2")).toBe(false); + }); + }); }); diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index 3bcd85ce..fefbcb3b 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -2,7 +2,7 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { AddressUtxos, AddressUsageMap } from "./types"; export class WalletMetrics { - public addressUsageMap: AddressUsageMap; + public addressUsageMap: Map; public transactions: Transaction[]; public utxos: AddressUtxos; constructor(transactions: Transaction[], utxos: AddressUtxos) { @@ -113,16 +113,16 @@ export class WalletMetrics { } } - constructAddressUsageMap(): AddressUsageMap { - let addressUsageMap: AddressUsageMap = {}; + constructAddressUsageMap(): Map { + const addressUsageMap: Map = new Map(); const transactions = this.transactions; for (const tx of transactions) { for (const output of tx.vout) { let address = output.scriptPubkeyAddress; - if (addressUsageMap[address]) { - addressUsageMap[address] = true; + if (addressUsageMap.has(address)) { + addressUsageMap.set(address, addressUsageMap.get(address)! + 1); } else { - addressUsageMap[address] = false; + addressUsageMap.set(address, 1); } } } @@ -133,9 +133,9 @@ export class WalletMetrics { Utility function to check if the given address was used already in past transactions */ isReusedAddress(address: string): boolean { - if (this.addressUsageMap[address]) { - return true; - } - return false; + return ( + this.addressUsageMap.has(address) && + this.addressUsageMap.get(address)! > 1 + ); } } diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 0bb52c9a..4609e89f 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -125,6 +125,37 @@ export class WasteMetrics extends WalletMetrics { return weight * (feeRate - estimatedLongTermFeeRate) + costOfTx; } + /* + Name : calculateDustLimits + Definition : + Dust outputs are uneconomical to spend because the fees to spend them are higher than + the value of the output. So to avoid the situations for having dust outputs in the wallet, + we calculate the dust limits for the wallet. + + Calculation : + lowerLimit - Below which the UTXO will actually behave as a dust output. + upperLimit - Above which the UTXO will be safe and economical to spend. + riskMultiplier - A factor that helps to determine the upper limit. + + The average size of the transaction to move one UTXO value could be 250-400 vBytes. + So, for the lower limit we are taking 250 vBytes as the transaction weight by default. + If your wallet supports weight units, you can change the value accordingly. + + lowerLimit = 250 * feeRate (sats/vByte) + upperLimit = lowerLimit * riskMultiplier + */ + calculateDustLimits( + feeRate: number, + txWeight: number = 250, + riskMultiplier: number = 2, + ): { lowerLimit: number; upperLimit: number } { + // By default, we are taking 250 vBytes as the transaction weight + // and 2 as the risk multiplier since weight could go as high as 400-500 vBytes. + let lowerLimit: number = txWeight * feeRate; + let upperLimit: number = lowerLimit * riskMultiplier; + return { lowerLimit, upperLimit }; + } + /* Name : Weighted Waste Score (W.W.S) From 74d656d4ef851c9efe2bc4fc634bb5a6d7bb3485 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Mon, 19 Aug 2024 03:39:49 +0530 Subject: [PATCH 63/92] Make transactions and UTXO options for ad-hoc cases Signed-off-by: Harshil-Jani --- packages/caravan-health/src/wallet.ts | 19 ++++++++++++------- packages/caravan-health/src/waste.test.ts | 10 ++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index fefbcb3b..a7a68273 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -2,13 +2,18 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { AddressUtxos, AddressUsageMap } from "./types"; export class WalletMetrics { - public addressUsageMap: Map; - public transactions: Transaction[]; - public utxos: AddressUtxos; - constructor(transactions: Transaction[], utxos: AddressUtxos) { - this.transactions = transactions; - this.utxos = utxos; - this.addressUsageMap = this.constructAddressUsageMap(); + public addressUsageMap: Map = new Map(); + public transactions: Transaction[] = []; + public utxos: AddressUtxos = {}; + + constructor(transactions?: Transaction[], utxos?: AddressUtxos) { + if (transactions) { + this.transactions = transactions; + this.addressUsageMap = this.constructAddressUsageMap(); + } + if (utxos) { + this.utxos = utxos; + } } /* Name : UTXO Mass Factor diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 448dd558..1d6ac3e6 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -127,6 +127,16 @@ describe("Waste metric scoring", () => { }); }); + describe("Dust Limits", () => { + it("calculates the lower and upper limit of the dust amount based on the fee rate and transaction weight", () => { + const uninitializedWasteMetric = new WasteMetrics(); + const { lowerLimit, upperLimit } = + uninitializedWasteMetric.calculateDustLimits(10, 300, 1.5); + expect(lowerLimit).toBe(3000); + expect(upperLimit).toBe(4500); + }); + }); + describe("Weighted Waste Score (W.W.S)", () => { it("calculates the overall waste of the wallet based on the relative fees score, fees to amount ratio and the UTXO mass factor", () => { const score: number = wasteMetric.weightedWasteScore( From 0d0ec9fb483272e546e37fa889e95a819c9eab99 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sat, 24 Aug 2024 02:35:24 +0530 Subject: [PATCH 64/92] Consider Script Type for dust limits Signed-off-by: Harshil-Jani --- packages/caravan-health/README.md | 4 +- packages/caravan-health/src/waste.test.ts | 11 +++--- packages/caravan-health/src/waste.ts | 46 +++++++++++++++-------- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 37576b8a..3a8f3c3f 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -26,8 +26,8 @@ The `WasteMetrics` class focuses on transaction fee efficiency and UTXO manageme - **Relative Fees Score (RFS):** Compares transaction fees to others in the same block. - **Fees To Amount Ratio (FAR):** Evaluates the proportion of fees to transaction amounts. -- **calculateDustLimits:** Calculates the dust limits for UTXOs based on the current fee rate. -- **Spend Waste Score (SWS):** Determines the economic efficiency of spending UTXOs. +- **calculateDustLimits:** Calculates the dust limits for UTXOs based on the current fee rate. A utxo is dust if it costs more to send based on the size of the input. +- **Spend Waste Amount (SWA):** Determines the cost of keeping or spending the UTXOs at a given point of time - **Weighted Waste Score (WWS):** Combines various metrics for an overall efficiency score. # Dependencies diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 1d6ac3e6..0764e257 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -101,7 +101,7 @@ describe("Waste metric scoring", () => { }); describe("Spend Waste Amount (S.W.A)", () => { - it("determines the cost of keeping or spending the UTXO at given point of time", () => { + it("determines the cost of keeping or spending the UTXOs at given point of time", () => { // Input UTXO Set : [0.1 BTC, 0.2 BTC, 0.3 BTC, 0.4 BTC] // Weight : 30 vbytes // Current Fee Rate : 10 sat/vbyte @@ -122,8 +122,9 @@ describe("Waste metric scoring", () => { estimatedLongTermFeeRate, ); expect(wasteAmount).toBe(1850); - // This number is positive this means that in future if we wait for the fee rate to go down, - // we can save 1850 sats + // This number is positive this means that in future if we spend the UTXOs now, + // we will be saving 1850 sats in fees. This is because in future the fee rate + // is expected to increase from 10 sat/vbyte to 15 sat/vbyte. }); }); @@ -132,8 +133,8 @@ describe("Waste metric scoring", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = uninitializedWasteMetric.calculateDustLimits(10, 300, 1.5); - expect(lowerLimit).toBe(3000); - expect(upperLimit).toBe(4500); + expect(lowerLimit).toBe(4480); + expect(upperLimit).toBe(6720); }); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 4609e89f..b3e6eea2 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,5 +1,6 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { WalletMetrics } from "./wallet"; +import { MultisigAddressType } from "@caravan/bitcoin"; export class WasteMetrics extends WalletMetrics { /* @@ -75,10 +76,10 @@ export class WasteMetrics extends WalletMetrics { /* Name : - Spend Waste Score (S.W.S) + Spend Waste Amount (S.W.A) Definition : - A score that indicates whether it is economical to spend a particular output now + A quantity that indicates whether it is economical to spend a particular output now or wait to consolidate it later when fees could be low. Important Terms: @@ -105,8 +106,8 @@ export class WasteMetrics extends WalletMetrics { Exact amount wanted to be spent in the transaction. Calculation : - spend waste score = consolidation factor + cost of transaction - spend waste score = weight (fee rate - estimatedLongTermFeeRate) + change + excess + spend waste amount = consolidation factor + cost of transaction + spend waste amount = weight (fee rate - estimatedLongTermFeeRate) + change + excess Observation : Depending on the fee rate in the long term, the consolidation factor can either be positive or negative. @@ -128,29 +129,43 @@ export class WasteMetrics extends WalletMetrics { /* Name : calculateDustLimits Definition : - Dust outputs are uneconomical to spend because the fees to spend them are higher than - the value of the output. So to avoid the situations for having dust outputs in the wallet, - we calculate the dust limits for the wallet. + "Dust" is defined in terms of dustRelayFee, + which has units satoshis-per-kilobyte. + If you'd pay more in fees than the value of the output + to spend something, then we consider it dust. + A typical spendable non-segwit txout is 34 bytes big, and will + need a CTxIn of at least 148 bytes to spend: + so dust is a spendable txout less than + 182*dustRelayFee/1000 (in satoshis). + 546 satoshis at the default rate of 3000 sat/kB. + A typical spendable segwit txout is 31 bytes big, and will + need a CTxIn of at least 67 bytes to spend: + so dust is a spendable txout less than + 98*dustRelayFee/1000 (in satoshis). + 294 satoshis at the default rate of 3000 sat/kB. Calculation : lowerLimit - Below which the UTXO will actually behave as a dust output. upperLimit - Above which the UTXO will be safe and economical to spend. riskMultiplier - A factor that helps to determine the upper limit. - The average size of the transaction to move one UTXO value could be 250-400 vBytes. - So, for the lower limit we are taking 250 vBytes as the transaction weight by default. - If your wallet supports weight units, you can change the value accordingly. - - lowerLimit = 250 * feeRate (sats/vByte) + lowerLimit = txWeight(default=182(34+148)) * feeRate (sats/vByte) upperLimit = lowerLimit * riskMultiplier + */ calculateDustLimits( feeRate: number, - txWeight: number = 250, + // A typical spendable non-segwit txout is 34 bytes big + txWeight: number = 34, riskMultiplier: number = 2, + isWitness: boolean = false, + WITNESS_SCALE_FACTOR = 1, ): { lowerLimit: number; upperLimit: number } { - // By default, we are taking 250 vBytes as the transaction weight - // and 2 as the risk multiplier since weight could go as high as 400-500 vBytes. + if (isWitness === true) { + txWeight += 32 + 4 + 1 + 107 / WITNESS_SCALE_FACTOR + 4; + } else { + txWeight += 32 + 4 + 1 + 107 + 4; + } let lowerLimit: number = txWeight * feeRate; let upperLimit: number = lowerLimit * riskMultiplier; return { lowerLimit, upperLimit }; @@ -174,6 +189,7 @@ export class WasteMetrics extends WalletMetrics { -> Good : (0.6, 0.8] -> Very Good : (0.8, 1] */ + weightedWasteScore(feeRatePercentileHistory: FeeRatePercentile[]): number { let RFS = this.relativeFeesScore(feeRatePercentileHistory); let FAR = this.feesToAmountRatio(); From a223d8050c02e09bca0d7f23890804a86119be79 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sat, 24 Aug 2024 02:52:36 +0530 Subject: [PATCH 65/92] Fix failing tests Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/caravan-clients/src/client.test.ts b/packages/caravan-clients/src/client.test.ts index fe77b71f..fcf4e60d 100644 --- a/packages/caravan-clients/src/client.test.ts +++ b/packages/caravan-clients/src/client.test.ts @@ -10,7 +10,6 @@ import * as wallet from "./wallet"; import BigNumber from "bignumber.js"; import { UTXO } from "./types"; import axios from "axios"; -import { FeeRatePercentile } from "./types"; jest.mock("axios"); describe("ClientBase", () => { From 1e4acb1d976fbb261f499bd4cf64ee296d2ffcd0 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Sat, 24 Aug 2024 02:56:31 +0530 Subject: [PATCH 66/92] Update packages/caravan-health/src/waste.ts Co-authored-by: buck --- packages/caravan-health/src/waste.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index b3e6eea2..9c370213 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -119,7 +119,7 @@ export class WasteMetrics extends WalletMetrics { weight: number, // Estimated weight of the transaction feeRate: number, // Current Fee rate for the transaction inputAmountSum: number, // Sum of amount for each coin in input of the transaction - spendAmount: number, // Exact Amount wanted to be spent in the transaction + spendAmount: number, // Exact amount wanted to be spent in the transaction estimatedLongTermFeeRate: number, // Long term estimated fee-rate ): number { let costOfTx: number = Math.abs(spendAmount - inputAmountSum); From af160108903bed828188aba24ebc71269a917df2 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Sat, 24 Aug 2024 02:57:11 +0530 Subject: [PATCH 67/92] Update packages/caravan-health/src/waste.ts Co-authored-by: buck --- packages/caravan-health/src/waste.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 9c370213..23b72de6 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -117,7 +117,7 @@ export class WasteMetrics extends WalletMetrics { */ spendWasteAmount( weight: number, // Estimated weight of the transaction - feeRate: number, // Current Fee rate for the transaction + feeRate: number, // Current fee rate for the transaction inputAmountSum: number, // Sum of amount for each coin in input of the transaction spendAmount: number, // Exact amount wanted to be spent in the transaction estimatedLongTermFeeRate: number, // Long term estimated fee-rate From 303da1f74087698cde1b379c82dde61547f48cf6 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Sat, 24 Aug 2024 03:17:03 +0530 Subject: [PATCH 68/92] Fix CI errors Signed-off-by: Harshil-Jani --- packages/caravan-clients/src/client.ts | 4 ++-- packages/caravan-health/src/privacy.test.ts | 1 - packages/caravan-health/src/privacy.ts | 1 - packages/caravan-health/src/utility.ts | 5 +++-- packages/caravan-health/src/wallet.ts | 2 +- packages/caravan-health/src/waste.ts | 1 - 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/caravan-clients/src/client.ts b/packages/caravan-clients/src/client.ts index e25952cf..95ac4a5d 100644 --- a/packages/caravan-clients/src/client.ts +++ b/packages/caravan-clients/src/client.ts @@ -415,9 +415,9 @@ export class BlockchainClient extends ClientBase { const data = await this.Get(`/v1/mining/blocks/fee-rates/all`); - let feeRatePercentileBlocks: FeeRatePercentile[] = []; + const feeRatePercentileBlocks: FeeRatePercentile[] = []; for (const block of data) { - let feeRatePercentile: FeeRatePercentile = { + const feeRatePercentile: FeeRatePercentile = { avgHeight: block?.avgHeight, timestamp: block?.timestamp, avgFee_0: block?.avgFee_0, diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index c7195f89..f536fd0c 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -118,7 +118,6 @@ const utxos: AddressUtxos = { describe("Privacy metric scoring", () => { const privacyMetric = new PrivacyMetrics(transactions, utxos); - const addressUsageMap = privacyMetric.constructAddressUsageMap(); describe("Determine Spend Type", () => { it("Perfect Spend are transactions with 1 input and 1 output", () => { diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 4a2b7fc5..8563aa73 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -249,7 +249,6 @@ export class PrivacyMetrics extends WalletMetrics { walletAddressType: MultisigAddressType, network: Network, ): number { - let addressUsageMap = this.constructAddressUsageMap(); let meanTopologyScore = this.getMeanTopologyScore(); let ARF = this.addressReuseFactor(); let ATF = this.addressTypeFactor(walletAddressType, network); diff --git a/packages/caravan-health/src/utility.ts b/packages/caravan-health/src/utility.ts index 8dddef67..0271b9a4 100644 --- a/packages/caravan-health/src/utility.ts +++ b/packages/caravan-health/src/utility.ts @@ -60,9 +60,10 @@ export function getSpendTypeScore( return 2 / 3 - 1 / numberOfOutputs; case SpendType.Consolidation: return 1 / numberOfInputs; - case SpendType.MixingOrCoinJoin: - let x = Math.pow(numberOfOutputs, 2) / numberOfInputs; + case SpendType.MixingOrCoinJoin: { + const x = Math.pow(numberOfOutputs, 2) / numberOfInputs; return ((1 / 2) * x) / (1 + x); + } default: throw new Error("Invalid spend type"); } diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index a7a68273..e5d73428 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -1,5 +1,5 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { AddressUtxos, AddressUsageMap } from "./types"; +import { AddressUtxos } from "./types"; export class WalletMetrics { public addressUsageMap: Map = new Map(); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 23b72de6..e0bc3745 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,6 +1,5 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { WalletMetrics } from "./wallet"; -import { MultisigAddressType } from "@caravan/bitcoin"; export class WasteMetrics extends WalletMetrics { /* From 4e60eb916e8548cdb1592396d9a8eb728e2e8c80 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Tue, 27 Aug 2024 00:14:23 +0530 Subject: [PATCH 69/92] Update packages/caravan-health/src/waste.ts Co-authored-by: buck --- packages/caravan-health/src/waste.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index e0bc3745..63c0c2e6 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -78,7 +78,7 @@ export class WasteMetrics extends WalletMetrics { Spend Waste Amount (S.W.A) Definition : - A quantity that indicates whether it is economical to spend a particular output now + A quantity that indicates whether it is economical to spend a particular output now in a given transaction or wait to consolidate it later when fees could be low. Important Terms: From 15f84754a222d5cb55df9d8a77af1a7945ccb4de Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Tue, 27 Aug 2024 00:14:52 +0530 Subject: [PATCH 70/92] Update packages/caravan-health/README.md Co-authored-by: buck --- packages/caravan-health/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 3a8f3c3f..320dfd27 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -27,7 +27,7 @@ The `WasteMetrics` class focuses on transaction fee efficiency and UTXO manageme - **Relative Fees Score (RFS):** Compares transaction fees to others in the same block. - **Fees To Amount Ratio (FAR):** Evaluates the proportion of fees to transaction amounts. - **calculateDustLimits:** Calculates the dust limits for UTXOs based on the current fee rate. A utxo is dust if it costs more to send based on the size of the input. -- **Spend Waste Amount (SWA):** Determines the cost of keeping or spending the UTXOs at a given point of time +- **Spend Waste Amount (SWA):** Determines the cost of keeping or spending UTXOs in particular transaction at a given point of time. - **Weighted Waste Score (WWS):** Combines various metrics for an overall efficiency score. # Dependencies From 1235246da69d3882e15576dea91ceb2107822301 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Tue, 27 Aug 2024 00:15:07 +0530 Subject: [PATCH 71/92] Update packages/caravan-health/README.md Co-authored-by: buck --- packages/caravan-health/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/README.md b/packages/caravan-health/README.md index 320dfd27..603d9dc6 100644 --- a/packages/caravan-health/README.md +++ b/packages/caravan-health/README.md @@ -32,7 +32,7 @@ The `WasteMetrics` class focuses on transaction fee efficiency and UTXO manageme # Dependencies -This library depends on the `@caravan/clients` and `@caravan/bitcoin` package for type validations and preparing some of the required data for that type. Make sure to install and import it correctly in your project. +This library depends on the `@caravan/clients` and `@caravan/bitcoin` package for type validations and preparing some of the required data for that type. # Usage From 6d7093b3eb92370f69b2bd84d6f921398b80e628 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Tue, 27 Aug 2024 00:16:21 +0530 Subject: [PATCH 72/92] Update packages/caravan-health/src/privacy.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 8563aa73..cd776acc 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -211,8 +211,8 @@ export class PrivacyMetrics extends WalletMetrics { Definition : The UTXO value dispersion factor is a combination of UTXO Spread Factor and UTXO Mass Factor. - It signifies the combined effect of how much variance is there in the UTXO Set values is and - how many number of UTXOs are there. + It signifies the combined effect of how much variance there is in the UTXO Set values and + the total number of UTXOs there are. Calculation : The U.V.D.F is calculated as a combination of UTXO Spread Factor and UTXO Set Length Weight. From bba82f778d9de169ea2002aed619e3fe962d0b61 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 27 Aug 2024 15:23:18 +0530 Subject: [PATCH 73/92] Adding script type into calculateDustMetric Signed-off-by: Harshil-Jani --- packages/caravan-health/src/waste.test.ts | 40 ++++++++++++++++-- packages/caravan-health/src/waste.ts | 49 ++++++++++++----------- 2 files changed, 62 insertions(+), 27 deletions(-) diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index 0764e257..aa105ee7 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -129,12 +129,44 @@ describe("Waste metric scoring", () => { }); describe("Dust Limits", () => { - it("calculates the lower and upper limit of the dust amount based on the fee rate and transaction weight", () => { + it("calculates the lower and upper limit of the dust amount for P2SH script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = - uninitializedWasteMetric.calculateDustLimits(10, 300, 1.5); - expect(lowerLimit).toBe(4480); - expect(upperLimit).toBe(6720); + uninitializedWasteMetric.calculateDustLimits(10, "P2SH", 1.5); + expect(lowerLimit).toBe(2760); + expect(upperLimit).toBe(4140); + }); + + it("calculates the lower and upper limit of the dust amount for P2WSH script type and 1.5 risk multiplier", () => { + const uninitializedWasteMetric = new WasteMetrics(); + const { lowerLimit, upperLimit } = + uninitializedWasteMetric.calculateDustLimits(10, "P2WSH", 1.5); + expect(lowerLimit).toBe(1057.5); + expect(upperLimit).toBe(1586.25); + }); + + it("calculates the lower and upper limit of the dust amount for P2PKH script type and 1.5 risk multiplier", () => { + const uninitializedWasteMetric = new WasteMetrics(); + const { lowerLimit, upperLimit } = + uninitializedWasteMetric.calculateDustLimits(10, "P2PKH", 1.5); + expect(lowerLimit).toBe(1315); + expect(upperLimit).toBe(1972.5); + }); + + it("calculates the lower and upper limit of the dust amount for P2TR script type and 1.5 risk multiplier", () => { + const uninitializedWasteMetric = new WasteMetrics(); + const { lowerLimit, upperLimit } = + uninitializedWasteMetric.calculateDustLimits(10, "P2TR", 1.5); + expect(lowerLimit).toBe(575); + expect(upperLimit).toBe(862.5); + }); + + it("calculates the lower and upper limit of the dust amount for P2SH-P2WSH script type and 1.5 risk multiplier", () => { + const uninitializedWasteMetric = new WasteMetrics(); + const { lowerLimit, upperLimit } = + uninitializedWasteMetric.calculateDustLimits(10, "P2SH-P2WSH", 1.5); + expect(lowerLimit).toBe(1212.5); + expect(upperLimit).toBe(1818.75); }); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 63c0c2e6..b86bcbfd 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,5 +1,6 @@ import { FeeRatePercentile, Transaction } from "@caravan/clients"; import { WalletMetrics } from "./wallet"; +import { MultisigAddressType } from "@caravan/bitcoin"; export class WasteMetrics extends WalletMetrics { /* @@ -128,44 +129,46 @@ export class WasteMetrics extends WalletMetrics { /* Name : calculateDustLimits Definition : - "Dust" is defined in terms of dustRelayFee, - which has units satoshis-per-kilobyte. - If you'd pay more in fees than the value of the output - to spend something, then we consider it dust. - A typical spendable non-segwit txout is 34 bytes big, and will - need a CTxIn of at least 148 bytes to spend: - so dust is a spendable txout less than - 182*dustRelayFee/1000 (in satoshis). - 546 satoshis at the default rate of 3000 sat/kB. - A typical spendable segwit txout is 31 bytes big, and will - need a CTxIn of at least 67 bytes to spend: - so dust is a spendable txout less than - 98*dustRelayFee/1000 (in satoshis). - 294 satoshis at the default rate of 3000 sat/kB. + Dust limits are the limits that help to determine the lower and upper limit of the UTXO + that can be spent economically. + The lower limit is below which the UTXO will actually behave as a dust output and the + upper limit is above which the UTXO will be safe and economical to spend. Calculation : lowerLimit - Below which the UTXO will actually behave as a dust output. upperLimit - Above which the UTXO will be safe and economical to spend. riskMultiplier - A factor that helps to determine the upper limit. - lowerLimit = txWeight(default=182(34+148)) * feeRate (sats/vByte) + Reference : https://medium.com/coinmonks/on-bitcoin-transaction-sizes-97e31bc9d816 + + lowerLimit = input_size (vB) * feeRate (sats/vByte) upperLimit = lowerLimit * riskMultiplier */ calculateDustLimits( feeRate: number, - // A typical spendable non-segwit txout is 34 bytes big - txWeight: number = 34, + scriptType: MultisigAddressType, riskMultiplier: number = 2, - isWitness: boolean = false, - WITNESS_SCALE_FACTOR = 1, ): { lowerLimit: number; upperLimit: number } { - if (isWitness === true) { - txWeight += 32 + 4 + 1 + 107 / WITNESS_SCALE_FACTOR + 4; + if (riskMultiplier <= 1) { + throw new Error("Risk Multiplier should be greater than 1"); + } + + let vsize: number; + if (scriptType === "P2SH") { + vsize = 276; + } else if (scriptType === "P2WSH") { + vsize = 105.75; + } else if (scriptType === "P2SH-P2WSH") { + vsize = 121.25; + } else if (scriptType === "P2TR") { + vsize = 57.5; + } else if (scriptType === "P2PKH") { + vsize = 131.5; } else { - txWeight += 32 + 4 + 1 + 107 + 4; + vsize = 276; // Worst Case } - let lowerLimit: number = txWeight * feeRate; + let lowerLimit: number = vsize * feeRate; let upperLimit: number = lowerLimit * riskMultiplier; return { lowerLimit, upperLimit }; } From b4bf711791e2acd4745e7c08a7692d858cd5f27e Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 27 Aug 2024 19:31:37 +0530 Subject: [PATCH 74/92] Updating test cases Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index f536fd0c..48f6ec3c 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -18,7 +18,7 @@ const transactions: Transaction[] = [ vout: [ { scriptPubkeyHex: "scriptPubkeyHex1", - scriptPubkeyAddress: "scriptPubkeyAddress1", + scriptPubkeyAddress: "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4", value: 0.1, }, ], @@ -52,12 +52,14 @@ const transactions: Transaction[] = [ }, { scriptPubkeyHex: "scriptPubkeyHex2", - scriptPubkeyAddress: "scriptPubkeyAddress2", + scriptPubkeyAddress: + "bc1qng72v5ceptk07htel0wcv6k27fkg6tmmd8887jr2l2yz5a5lnawqqeceya", value: 0.2, }, { scriptPubkeyHex: "scriptPubkeyHex2", - scriptPubkeyAddress: "scriptPubkeyAddress2", + scriptPubkeyAddress: + "bc1qng72v5ceptk07htel0wcv6k27fkg6tmmd8887jr2l2yz5a5lnawqqeceya", value: 0.2, }, { @@ -311,6 +313,9 @@ describe("Privacy metric scoring", () => { }); describe("Address Reuse Factor", () => { + it.todo( + "Make multiple transactions and UTXO objects to test the address reuse factor for half used and half reused addresses.", + ); it("Calculates the amount being held by reused addresses with respect to the total amount", () => { const addressReuseFactor: number = privacyMetric.addressReuseFactor(); expect(addressReuseFactor).toBe(0); @@ -318,6 +323,7 @@ describe("Privacy metric scoring", () => { }); describe("Address Type Factor", () => { + it.todo("Test with different combination of address types and networks"); it("Calculates the the address type distribution of the wallet transactions", () => { const addressTypeFactor: number = privacyMetric.addressTypeFactor( "P2SH", @@ -325,6 +331,13 @@ describe("Privacy metric scoring", () => { ); expect(addressTypeFactor).toBe(1); }); + it("Calculates the the address type distribution of the wallet transactions", () => { + const addressTypeFactor: number = privacyMetric.addressTypeFactor( + "P2WSH", + Network.MAINNET, + ); + expect(addressTypeFactor).toBe(0.25); + }); }); describe("UTXO Spread Factor", () => { From 5f98fef84670ab6c3a7686935f6dabaac60a413e Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Tue, 27 Aug 2024 23:24:58 +0530 Subject: [PATCH 75/92] Code cleaning Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 4 +-- packages/caravan-health/src/privacy.ts | 31 ++++++++++----------- packages/caravan-health/src/types.ts | 3 ++ packages/caravan-health/src/wallet.test.ts | 3 +- packages/caravan-health/src/wallet.ts | 11 ++++---- packages/caravan-health/src/waste.ts | 27 ++++++++++-------- 6 files changed, 41 insertions(+), 38 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 48f6ec3c..89a03d0f 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,7 +1,5 @@ -import { Transaction } from "@caravan/clients"; import { PrivacyMetrics } from "./privacy"; -import { AddressUtxos, SpendType } from "./types"; -import { Network } from "@caravan/bitcoin"; +import { AddressUtxos, SpendType, Transaction, Network } from "./types"; import { determineSpendType, getSpendTypeScore } from "./utility"; const transactions: Transaction[] = [ diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index cd776acc..dc9f5652 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,6 +1,5 @@ -import { Transaction } from "@caravan/clients"; -import { SpendType } from "./types"; -import { MultisigAddressType, Network, getAddressType } from "@caravan/bitcoin"; +import { SpendType, MultisigAddressType, Network, Transaction } from "./types"; +import { getAddressType } from "@caravan/bitcoin"; import { WalletMetrics } from "./wallet"; import { determineSpendType, getSpendTypeScore } from "./utility"; @@ -41,9 +40,9 @@ export class PrivacyMetrics extends WalletMetrics { if (spendType === SpendType.Consolidation) { return score; } - for (let output of transaction.vout) { - let address = output.scriptPubkeyAddress; - let isResued = this.isReusedAddress(address); + for (const output of transaction.vout) { + const address = output.scriptPubkeyAddress; + const isResued = this.isReusedAddress(address); if (isResued === true) { return score; } @@ -72,8 +71,8 @@ export class PrivacyMetrics extends WalletMetrics { getMeanTopologyScore(): number { let privacyScore = 0; const transactions = this.transactions; - for (let tx of transactions) { - let topologyScore = this.getTopologyScore(tx); + for (const tx of transactions) { + const topologyScore = this.getTopologyScore(tx); privacyScore += topologyScore; } return privacyScore / transactions.length; @@ -105,7 +104,7 @@ export class PrivacyMetrics extends WalletMetrics { const addressUtxos = utxos[address]; for (const utxo of addressUtxos) { totalAmount += utxo.value; - let isReused = this.isReusedAddress(address); + const isReused = this.isReusedAddress(address); if (isReused) { reusedAmount += utxo.value; } @@ -226,8 +225,8 @@ export class PrivacyMetrics extends WalletMetrics { -> Very Good : (0.075, 0.15] */ utxoValueDispersionFactor(): number { - let UMF: number = this.utxoMassFactor(); - let USF: number = this.utxoSpreadFactor(); + const UMF: number = this.utxoMassFactor(); + const USF: number = this.utxoSpreadFactor(); return (USF + UMF) * 0.15 - 0.15; } @@ -249,12 +248,12 @@ export class PrivacyMetrics extends WalletMetrics { walletAddressType: MultisigAddressType, network: Network, ): number { - let meanTopologyScore = this.getMeanTopologyScore(); - let ARF = this.addressReuseFactor(); - let ATF = this.addressTypeFactor(walletAddressType, network); - let UVDF = this.utxoValueDispersionFactor(); + const meanTopologyScore = this.getMeanTopologyScore(); + const ARF = this.addressReuseFactor(); + const ATF = this.addressTypeFactor(walletAddressType, network); + const UVDF = this.utxoValueDispersionFactor(); - let WPS: number = + const WPS: number = (meanTopologyScore * (1 - 0.5 * ARF) + 0.1 * (1 - ARF)) * (1 - ATF) + 0.1 * UVDF; diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index 9d86596f..2f9bfef1 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -1,4 +1,7 @@ import { UTXO } from "@caravan/clients"; +export type { MultisigAddressType } from "@caravan/bitcoin"; +export type { Transaction, UTXO, FeeRatePercentile } from "@caravan/clients"; +export { Network } from "@caravan/bitcoin"; // Represents the Unspent Outputs of the address export interface AddressUtxos { diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index 56202a03..bb900dfd 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -1,6 +1,5 @@ import { WalletMetrics } from "./wallet"; -import { AddressUtxos } from "./types"; -import { FeeRatePercentile, Transaction } from "@caravan/clients"; +import { AddressUtxos, FeeRatePercentile, Transaction } from "./types"; const transactions: Transaction[] = [ // transactions[0] is a perfect spend transaction diff --git a/packages/caravan-health/src/wallet.ts b/packages/caravan-health/src/wallet.ts index e5d73428..f6df022d 100644 --- a/packages/caravan-health/src/wallet.ts +++ b/packages/caravan-health/src/wallet.ts @@ -1,5 +1,4 @@ -import { FeeRatePercentile, Transaction } from "@caravan/clients"; -import { AddressUtxos } from "./types"; +import { AddressUtxos, Transaction, FeeRatePercentile } from "./types"; export class WalletMetrics { public addressUsageMap: Map = new Map(); @@ -54,8 +53,8 @@ export class WalletMetrics { Utility function that helps to obtain the fee rate of the transaction */ getFeeRateForTransaction(transaction: Transaction): number { - let fees: number = transaction.fee; - let weight: number = transaction.weight; + const fees: number = transaction.fee; + const weight: number = transaction.weight; return fees / weight; } @@ -67,7 +66,7 @@ export class WalletMetrics { feeRate: number, feeRatePercentileHistory: FeeRatePercentile[], ): number { - let percentile: number = this.getClosestPercentile( + const percentile: number = this.getClosestPercentile( timestamp, feeRate, feeRatePercentileHistory, @@ -123,7 +122,7 @@ export class WalletMetrics { const transactions = this.transactions; for (const tx of transactions) { for (const output of tx.vout) { - let address = output.scriptPubkeyAddress; + const address = output.scriptPubkeyAddress; if (addressUsageMap.has(address)) { addressUsageMap.set(address, addressUsageMap.get(address)! + 1); } else { diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index b86bcbfd..dbb53f61 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,6 +1,5 @@ -import { FeeRatePercentile, Transaction } from "@caravan/clients"; +import { FeeRatePercentile, Transaction, MultisigAddressType } from "./types"; import { WalletMetrics } from "./wallet"; -import { MultisigAddressType } from "@caravan/bitcoin"; export class WasteMetrics extends WalletMetrics { /* @@ -29,8 +28,8 @@ export class WasteMetrics extends WalletMetrics { for (const tx of transactions) { if (tx.isSend === true) { numberOfSendTx++; - let feeRate: number = this.getFeeRateForTransaction(tx); - let RFS: number = this.getFeeRatePercentileScore( + const feeRate: number = this.getFeeRateForTransaction(tx); + const RFS: number = this.getFeeRatePercentileScore( tx.block_time, feeRate, feeRatePercentileHistory, @@ -122,7 +121,7 @@ export class WasteMetrics extends WalletMetrics { spendAmount: number, // Exact amount wanted to be spent in the transaction estimatedLongTermFeeRate: number, // Long term estimated fee-rate ): number { - let costOfTx: number = Math.abs(spendAmount - inputAmountSum); + const costOfTx: number = Math.abs(spendAmount - inputAmountSum); return weight * (feeRate - estimatedLongTermFeeRate) + costOfTx; } @@ -137,7 +136,13 @@ export class WasteMetrics extends WalletMetrics { Calculation : lowerLimit - Below which the UTXO will actually behave as a dust output. upperLimit - Above which the UTXO will be safe and economical to spend. - riskMultiplier - A factor that helps to determine the upper limit. + riskMultiplier - + The riskMultiplier is a factor that scales the lower limit of a UTXO to determine its + upper limit. Based on their risk tolerance and expected fee volatility, A higher + multiplier provides a greater buffer but may unnecessarily categorize some UTXOs as + safe that could otherwise be considered risky. The default value is set to 2 as a + balanced approach. It doubles the lower limit, providing a reasonable buffer for most + common fee scenarios without being overly conservative. Reference : https://medium.com/coinmonks/on-bitcoin-transaction-sizes-97e31bc9d816 @@ -168,8 +173,8 @@ export class WasteMetrics extends WalletMetrics { } else { vsize = 276; // Worst Case } - let lowerLimit: number = vsize * feeRate; - let upperLimit: number = lowerLimit * riskMultiplier; + const lowerLimit: number = vsize * feeRate; + const upperLimit: number = lowerLimit * riskMultiplier; return { lowerLimit, upperLimit }; } @@ -193,9 +198,9 @@ export class WasteMetrics extends WalletMetrics { */ weightedWasteScore(feeRatePercentileHistory: FeeRatePercentile[]): number { - let RFS = this.relativeFeesScore(feeRatePercentileHistory); - let FAR = this.feesToAmountRatio(); - let UMF = this.utxoMassFactor(); + const RFS = this.relativeFeesScore(feeRatePercentileHistory); + const FAR = this.feesToAmountRatio(); + const UMF = this.utxoMassFactor(); return 0.35 * RFS + 0.35 * FAR + 0.3 * UMF; } } From 9b68338b0a8031d02a4482e8c7e81d1b7b8cc799 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Wed, 28 Aug 2024 19:54:47 +0530 Subject: [PATCH 76/92] Update packages/caravan-health/src/privacy.test.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 89a03d0f..e16f2e53 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -265,7 +265,7 @@ describe("Privacy metric scoring", () => { No. of Output = 2 or more (MANY) Justification : - Privacy score is directly proportional to More Number of outputs AND less number of inputs in case of coin join. + Privacy score is directly proportional to higher number of outputs AND less number of inputs in case of coin join. The explanation for this to happen is that if you try to consolidate i.e lower number of output and high number of input, the privacy should be decreased and in case of coin join where there are so many outputs against few inputs it should have increased From a3032b1ddbf3e36df17e4bfe23611a68b8a67915 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Wed, 28 Aug 2024 19:54:59 +0530 Subject: [PATCH 77/92] Update packages/caravan-health/src/privacy.test.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index e16f2e53..549d166b 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -244,9 +244,9 @@ describe("Privacy metric scoring", () => { No. of Input = 2 or more (MANY) No. of Output = 1 - When the number of inputs are more against the single output then the privacy score should decrease because - it increases the fingerprint or certainty for on-chain analysers that the following transaction was made as - a consolidation and with more number of inputs we tend to expose more UTXOs for a transaction. + When the number of inputs is higher than the single output then the privacy score should decrease because + it increases the fingerprint or certainty for on-chain analysers to determine that the transaction was made as + a consolidation and with more inputs we tend to expose more UTXOs for a transaction. score = 1 / Number of Inputs */ From 17de9ee7c2dc29bfecdbcf693c374be1abfd22e2 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Wed, 28 Aug 2024 19:55:09 +0530 Subject: [PATCH 78/92] Update packages/caravan-health/src/privacy.test.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 549d166b..40d47e8f 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -216,7 +216,7 @@ describe("Privacy metric scoring", () => { Justification behind the number 0.67 : We want that the privacy score should increase with more numbers of outputs, because if in case it was a self spent transaction then producing more outputs means - producing more UTXOs which have privacy benefits for the wallet. + producing more UTXOs which has privacy benefits for the wallet. Now, We are using a multiplication factor of 1.5 for deniability in case of all the self spend transactions. So the quantity [ X - ( 1 / No. of outputs) ] should be less than 1 From 8f8eff0f1f1175301b1dd0dd862814e95248ed10 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Wed, 28 Aug 2024 19:55:18 +0530 Subject: [PATCH 79/92] Update packages/caravan-health/src/privacy.test.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 40d47e8f..bd8ae993 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -215,7 +215,7 @@ describe("Privacy metric scoring", () => { Justification behind the number 0.67 : We want that the privacy score should increase with more numbers of outputs, - because if in case it was a self spent transaction then producing more outputs means + because in the case it is a self spend transaction then producing more outputs means producing more UTXOs which has privacy benefits for the wallet. Now, We are using a multiplication factor of 1.5 for deniability in case of all the self spend transactions. From f327999e64aa62b607da724797f04fcebd94d24c Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Wed, 28 Aug 2024 19:55:26 +0530 Subject: [PATCH 80/92] Update packages/caravan-health/src/privacy.test.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index bd8ae993..145ae910 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -214,7 +214,7 @@ describe("Privacy metric scoring", () => { score = 0.67 - ( 1 / Number of Outputs ) Justification behind the number 0.67 : - We want that the privacy score should increase with more numbers of outputs, + We want that the privacy score should increase with higher numbers of outputs, because in the case it is a self spend transaction then producing more outputs means producing more UTXOs which has privacy benefits for the wallet. From ac249a9e3bea03f3bc1cee17b16c6077effae21b Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Wed, 28 Aug 2024 19:55:34 +0530 Subject: [PATCH 81/92] Update packages/caravan-health/src/waste.ts Co-authored-by: buck --- packages/caravan-health/src/waste.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index dbb53f61..4454c72c 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -138,7 +138,7 @@ export class WasteMetrics extends WalletMetrics { upperLimit - Above which the UTXO will be safe and economical to spend. riskMultiplier - The riskMultiplier is a factor that scales the lower limit of a UTXO to determine its - upper limit. Based on their risk tolerance and expected fee volatility, A higher + upper limit. Based on their risk tolerance and expected fee volatility, a higher multiplier provides a greater buffer but may unnecessarily categorize some UTXOs as safe that could otherwise be considered risky. The default value is set to 2 as a balanced approach. It doubles the lower limit, providing a reasonable buffer for most From 10d68d77b5ecf0e6f01749121edbeec69ed2c601 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 28 Aug 2024 20:02:20 +0530 Subject: [PATCH 82/92] SpendTypeScore just takes in Input and Output Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 28 ++++++--------------- packages/caravan-health/src/privacy.ts | 1 - packages/caravan-health/src/utility.ts | 2 +- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 145ae910..b1167c3d 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -179,7 +179,7 @@ describe("Privacy metric scoring", () => { score = 0.5 * (1 - 0) = 0.5 */ it("Perfect Spend has a raw score of 0.5 for external wallet payments", () => { - const score: number = getSpendTypeScore(SpendType.PerfectSpend, 1, 1); + const score: number = getSpendTypeScore(1, 1); expect(score).toBe(0.5); }); /* @@ -201,7 +201,7 @@ describe("Privacy metric scoring", () => { score = 0.67 * (1-0.33) = 0.4489 */ it("Simple Spend has a raw score of 0.44 for external wallet payments", () => { - const score: number = getSpendTypeScore(SpendType.SimpleSpend, 1, 2); + const score: number = getSpendTypeScore(1, 2); expect(score).toBeCloseTo(0.44); }); @@ -230,11 +230,7 @@ describe("Privacy metric scoring", () => { */ it("UTXO Fragmentation has a raw score of 0.33 for external wallet payments", () => { - const score: number = getSpendTypeScore( - SpendType.UTXOFragmentation, - 1, - 3, - ); + const score: number = getSpendTypeScore(1, 3); expect(score).toBeCloseTo(0.33); }); @@ -251,10 +247,10 @@ describe("Privacy metric scoring", () => { score = 1 / Number of Inputs */ it("Consolidation has raw score of ", () => { - const score: number = getSpendTypeScore(SpendType.Consolidation, 2, 1); + const score: number = getSpendTypeScore(2, 1); expect(score).toBeCloseTo(0.5); - const score2: number = getSpendTypeScore(SpendType.Consolidation, 3, 1); + const score2: number = getSpendTypeScore(3, 1); expect(score2).toBeCloseTo(0.33); }); @@ -274,21 +270,13 @@ describe("Privacy metric scoring", () => { score = 1/2 * (y2/x)/(1+y2/x) */ it("MixingOrCoinJoin has raw score of ", () => { - const score: number = getSpendTypeScore(SpendType.MixingOrCoinJoin, 2, 2); + const score: number = getSpendTypeScore(2, 2); expect(score).toBeCloseTo(0.33); - const score2: number = getSpendTypeScore( - SpendType.MixingOrCoinJoin, - 2, - 3, - ); + const score2: number = getSpendTypeScore(2, 3); expect(score2).toBeCloseTo(0.409); - const score3: number = getSpendTypeScore( - SpendType.MixingOrCoinJoin, - 3, - 2, - ); + const score3: number = getSpendTypeScore(3, 2); expect(score3).toBeCloseTo(0.285); }); }); diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index dc9f5652..2c649711 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -32,7 +32,6 @@ export class PrivacyMetrics extends WalletMetrics { numberOfOutputs, ); const score: number = getSpendTypeScore( - spendType, numberOfInputs, numberOfOutputs, ); diff --git a/packages/caravan-health/src/utility.ts b/packages/caravan-health/src/utility.ts index 0271b9a4..aa841526 100644 --- a/packages/caravan-health/src/utility.ts +++ b/packages/caravan-health/src/utility.ts @@ -47,10 +47,10 @@ export function determineSpendType(inputs: number, outputs: number): SpendType { -> Very Good : (0.6, 0.85] */ export function getSpendTypeScore( - spendType: SpendType, numberOfInputs: number, numberOfOutputs: number, ): number { + const spendType = determineSpendType(numberOfInputs, numberOfOutputs); switch (spendType) { case SpendType.PerfectSpend: return 1 / 2; From bb06e481c8ee41609129d7ee7c3342d4a96a48e1 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 28 Aug 2024 20:12:54 +0530 Subject: [PATCH 83/92] Adding nit for length of UTXO Signed-off-by: Harshil-Jani --- packages/caravan-health/src/wallet.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/caravan-health/src/wallet.test.ts b/packages/caravan-health/src/wallet.test.ts index bb900dfd..0b61268e 100644 --- a/packages/caravan-health/src/wallet.test.ts +++ b/packages/caravan-health/src/wallet.test.ts @@ -107,6 +107,7 @@ describe("Wallet Metrics", () => { const walletMetrics = new WalletMetrics(transactions, utxos); describe("UTXO Mass Factor", () => { it("should return 1 for UTXO set length = 4", () => { + expect(Object.values(walletMetrics.utxos)[0].length).toBe(4) expect(walletMetrics.utxoMassFactor()).toBe(1); }); }); From d47823b00856e57c7db2b47537cd9c3a056bc215 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Wed, 28 Aug 2024 22:05:20 +0530 Subject: [PATCH 84/92] Remove unused map Signed-off-by: Harshil-Jani --- packages/caravan-health/src/types.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/caravan-health/src/types.ts b/packages/caravan-health/src/types.ts index 2f9bfef1..ebce49ab 100644 --- a/packages/caravan-health/src/types.ts +++ b/packages/caravan-health/src/types.ts @@ -26,8 +26,3 @@ export enum SpendType { Consolidation = "Consolidation", MixingOrCoinJoin = "MixingOrCoinJoin", } - -// Represents the usage of the address in past transactions -export interface AddressUsageMap { - [address: string]: boolean; -} From 564a8ed96172bcec721d53158135c2913f7d1153 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 29 Aug 2024 02:53:24 +0530 Subject: [PATCH 85/92] Dust calculations update Signed-off-by: Harshil-Jani --- packages/caravan-health/src/waste.test.ts | 31 +++++++++++++++-------- packages/caravan-health/src/waste.ts | 26 +++++++++++++++---- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index aa105ee7..c1178cd4 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -129,26 +129,30 @@ describe("Waste metric scoring", () => { }); describe("Dust Limits", () => { + const config = { + m: 2, // Provide the required property m + n: 3, // Provide the required property n + }; it("calculates the lower and upper limit of the dust amount for P2SH script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = - uninitializedWasteMetric.calculateDustLimits(10, "P2SH", 1.5); - expect(lowerLimit).toBe(2760); - expect(upperLimit).toBe(4140); + uninitializedWasteMetric.calculateDustLimits(10, "P2SH", config, 1.5); + expect(lowerLimit).toBe(2480); + expect(upperLimit).toBe(3720); }); it("calculates the lower and upper limit of the dust amount for P2WSH script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = - uninitializedWasteMetric.calculateDustLimits(10, "P2WSH", 1.5); - expect(lowerLimit).toBe(1057.5); - expect(upperLimit).toBe(1586.25); + uninitializedWasteMetric.calculateDustLimits(10, "P2WSH", config, 1.5); + expect(lowerLimit).toBe(2580); + expect(upperLimit).toBe(3870); }); it("calculates the lower and upper limit of the dust amount for P2PKH script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = - uninitializedWasteMetric.calculateDustLimits(10, "P2PKH", 1.5); + uninitializedWasteMetric.calculateDustLimits(10, "P2PKH", config, 1.5); expect(lowerLimit).toBe(1315); expect(upperLimit).toBe(1972.5); }); @@ -156,7 +160,7 @@ describe("Waste metric scoring", () => { it("calculates the lower and upper limit of the dust amount for P2TR script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = - uninitializedWasteMetric.calculateDustLimits(10, "P2TR", 1.5); + uninitializedWasteMetric.calculateDustLimits(10, "P2TR", config, 1.5); expect(lowerLimit).toBe(575); expect(upperLimit).toBe(862.5); }); @@ -164,9 +168,14 @@ describe("Waste metric scoring", () => { it("calculates the lower and upper limit of the dust amount for P2SH-P2WSH script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); const { lowerLimit, upperLimit } = - uninitializedWasteMetric.calculateDustLimits(10, "P2SH-P2WSH", 1.5); - expect(lowerLimit).toBe(1212.5); - expect(upperLimit).toBe(1818.75); + uninitializedWasteMetric.calculateDustLimits( + 10, + "P2SH-P2WSH", + config, + 1.5, + ); + expect(lowerLimit).toBe(610); + expect(upperLimit).toBe(915); }); }); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 4454c72c..6179ee9c 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -1,5 +1,6 @@ import { FeeRatePercentile, Transaction, MultisigAddressType } from "./types"; import { WalletMetrics } from "./wallet"; +import { getWitnessSize } from "@caravan/bitcoin"; export class WasteMetrics extends WalletMetrics { /* @@ -144,7 +145,6 @@ export class WasteMetrics extends WalletMetrics { balanced approach. It doubles the lower limit, providing a reasonable buffer for most common fee scenarios without being overly conservative. - Reference : https://medium.com/coinmonks/on-bitcoin-transaction-sizes-97e31bc9d816 lowerLimit = input_size (vB) * feeRate (sats/vByte) upperLimit = lowerLimit * riskMultiplier @@ -153,6 +153,10 @@ export class WasteMetrics extends WalletMetrics { calculateDustLimits( feeRate: number, scriptType: MultisigAddressType, + config: { + m: number; + n: number; + }, riskMultiplier: number = 2, ): { lowerLimit: number; upperLimit: number } { if (riskMultiplier <= 1) { @@ -161,17 +165,29 @@ export class WasteMetrics extends WalletMetrics { let vsize: number; if (scriptType === "P2SH") { - vsize = 276; + const signatureLength = 72 + 1; // approx including push byte + const keylength = 33 + 1; // push byte + vsize = signatureLength * config.m + keylength * config.n; } else if (scriptType === "P2WSH") { - vsize = 105.75; + let total = 0; + total += 1; // segwit marker + total += 1; // segwit flag + total += getWitnessSize(config.m, config.n); // add witness for each input + vsize = total; } else if (scriptType === "P2SH-P2WSH") { - vsize = 121.25; + const signatureLength = 72; + const keylength = 33; + const witnessSize = signatureLength * config.m + keylength * config.n; + vsize = Math.ceil(0.25 * witnessSize); } else if (scriptType === "P2TR") { + // Reference : https://bitcoin.stackexchange.com/questions/111395/what-is-the-weight-of-a-p2tr-input + // Optimistic key-path-spend input size vsize = 57.5; } else if (scriptType === "P2PKH") { + // Reference : https://medium.com/coinmonks/on-bitcoin-transaction-sizes-97e31bc9d816 vsize = 131.5; } else { - vsize = 276; // Worst Case + vsize = 546; // Worst Case } const lowerLimit: number = vsize * feeRate; const upperLimit: number = lowerLimit * riskMultiplier; From 2e22bf3a4708d0421b7ac9ef02ce330ebcc2d81a Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 29 Aug 2024 03:02:12 +0530 Subject: [PATCH 86/92] Update config variable names Signed-off-by: Harshil-Jani --- packages/caravan-health/src/waste.test.ts | 4 ++-- packages/caravan-health/src/waste.ts | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/caravan-health/src/waste.test.ts b/packages/caravan-health/src/waste.test.ts index c1178cd4..a9a61b55 100644 --- a/packages/caravan-health/src/waste.test.ts +++ b/packages/caravan-health/src/waste.test.ts @@ -130,8 +130,8 @@ describe("Waste metric scoring", () => { describe("Dust Limits", () => { const config = { - m: 2, // Provide the required property m - n: 3, // Provide the required property n + requiredSignerCount: 2, // Provide the required property m + totalSignerCount: 3, // Provide the required property n }; it("calculates the lower and upper limit of the dust amount for P2SH script type and 1.5 risk multiplier", () => { const uninitializedWasteMetric = new WasteMetrics(); diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 6179ee9c..bc17dfb2 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -137,6 +137,9 @@ export class WasteMetrics extends WalletMetrics { Calculation : lowerLimit - Below which the UTXO will actually behave as a dust output. upperLimit - Above which the UTXO will be safe and economical to spend. + config - It takes two parameters, requiredSignerCount and totalSignerCount + Eg : For a 2-of-3 Multisig wallet the config will be + config : {requiredSignerCount: 2, totalSignerCount: 3} riskMultiplier - The riskMultiplier is a factor that scales the lower limit of a UTXO to determine its upper limit. Based on their risk tolerance and expected fee volatility, a higher @@ -154,8 +157,8 @@ export class WasteMetrics extends WalletMetrics { feeRate: number, scriptType: MultisigAddressType, config: { - m: number; - n: number; + requiredSignerCount: number; + totalSignerCount: number; }, riskMultiplier: number = 2, ): { lowerLimit: number; upperLimit: number } { @@ -167,17 +170,24 @@ export class WasteMetrics extends WalletMetrics { if (scriptType === "P2SH") { const signatureLength = 72 + 1; // approx including push byte const keylength = 33 + 1; // push byte - vsize = signatureLength * config.m + keylength * config.n; + vsize = + signatureLength * config.requiredSignerCount + + keylength * config.totalSignerCount; } else if (scriptType === "P2WSH") { let total = 0; total += 1; // segwit marker total += 1; // segwit flag - total += getWitnessSize(config.m, config.n); // add witness for each input + total += getWitnessSize( + config.requiredSignerCount, + config.totalSignerCount, + ); // add witness for each input vsize = total; } else if (scriptType === "P2SH-P2WSH") { const signatureLength = 72; const keylength = 33; - const witnessSize = signatureLength * config.m + keylength * config.n; + const witnessSize = + signatureLength * config.requiredSignerCount + + keylength * config.totalSignerCount; vsize = Math.ceil(0.25 * witnessSize); } else if (scriptType === "P2TR") { // Reference : https://bitcoin.stackexchange.com/questions/111395/what-is-the-weight-of-a-p2tr-input From 3a2dee16bfbbfa987b527c982cf0ba6ceba0fc8e Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Thu, 29 Aug 2024 03:05:54 +0530 Subject: [PATCH 87/92] Removed unncessary comments Signed-off-by: Harshil-Jani --- packages/caravan-health/src/waste.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index bc17dfb2..56ea5862 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -180,7 +180,7 @@ export class WasteMetrics extends WalletMetrics { total += getWitnessSize( config.requiredSignerCount, config.totalSignerCount, - ); // add witness for each input + ); vsize = total; } else if (scriptType === "P2SH-P2WSH") { const signatureLength = 72; From afa3386753cbe372aa1a0ba3eed4dc8cf4ddd60a Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 30 Aug 2024 02:42:46 +0530 Subject: [PATCH 88/92] Update logic for spend type determination Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 21 +++++++++---------- packages/caravan-health/src/utility.ts | 23 ++++++++++++++------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index b1167c3d..014b68e0 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -153,13 +153,13 @@ describe("Privacy metric scoring", () => { }); it("Mixing or CoinJoin transactions have more than 1 inputs and more than 1 outputs", () => { - const spendType: SpendType = determineSpendType(2, 2); + const spendType: SpendType = determineSpendType(3, 3); expect(spendType).toBe(SpendType.MixingOrCoinJoin); - const spendType2: SpendType = determineSpendType(2, 3); + const spendType2: SpendType = determineSpendType(4, 3); expect(spendType2).toBe(SpendType.MixingOrCoinJoin); - const spendType3: SpendType = determineSpendType(3, 2); + const spendType3: SpendType = determineSpendType(4, 4); expect(spendType3).toBe(SpendType.MixingOrCoinJoin); }); }); @@ -186,7 +186,6 @@ describe("Privacy metric scoring", () => { score = P(“An output cannot be a self-payment) * (1 - P(“involvement of any change output”)) Simple Spend Transaction - - No. of Input = 1 - No. of Output = 2 P("An output can be a self-payment") = 0.33 @@ -257,8 +256,8 @@ describe("Privacy metric scoring", () => { /* Mixing or CoinJoin Transaction MANY to MANY transaction - No. of Input = 2 or more (MANY) - No. of Output = 2 or more (MANY) + No. of Input >= No. of Outputs (MANY) + No. of Output > 2 or more (MANY) Justification : Privacy score is directly proportional to higher number of outputs AND less number of inputs in case of coin join. @@ -271,13 +270,13 @@ describe("Privacy metric scoring", () => { */ it("MixingOrCoinJoin has raw score of ", () => { const score: number = getSpendTypeScore(2, 2); - expect(score).toBeCloseTo(0.33); + expect(score).toBeCloseTo(0.44); const score2: number = getSpendTypeScore(2, 3); - expect(score2).toBeCloseTo(0.409); + expect(score2).toBeCloseTo(0.333); const score3: number = getSpendTypeScore(3, 2); - expect(score3).toBeCloseTo(0.285); + expect(score3).toBeCloseTo(0.44); }); }); @@ -287,14 +286,14 @@ describe("Privacy metric scoring", () => { expect(score).toBe(0.75); const score2: number = privacyMetric.getTopologyScore(transactions[1]); - expect(score2).toBeCloseTo(0.44); + expect(score2).toBeCloseTo(0.416); }); }); describe("Mean Topology Score", () => { it("Calculates the mean topology score for all transactions done by a wallet", () => { const meanScore: number = privacyMetric.getMeanTopologyScore(); - expect(meanScore).toBeCloseTo(0.597); + expect(meanScore).toBeCloseTo(0.583); }); }); diff --git a/packages/caravan-health/src/utility.ts b/packages/caravan-health/src/utility.ts index aa841526..825a04ff 100644 --- a/packages/caravan-health/src/utility.ts +++ b/packages/caravan-health/src/utility.ts @@ -11,19 +11,26 @@ import { SpendType } from "./types"; Calculation : We have 5 categories of transaction type each with their own impact on privacy score - Perfect Spend (1 input, 1 output) - - Simple Spend (1 input, 2 outputs) + - Simple Spend (output=2 irrespective of input it is Simple Spend) - UTXO Fragmentation (1 input, more than 2 standard outputs) - Consolidation (more than 1 input, 1 output) - - CoinJoin or Mixing (more than 1 input, more than 1 output) + - CoinJoin or Mixing (inputs more than equal to outputs, more than 2 output) */ export function determineSpendType(inputs: number, outputs: number): SpendType { - if (inputs === 1) { - if (outputs === 1) return SpendType.PerfectSpend; - if (outputs === 2) return SpendType.SimpleSpend; - return SpendType.UTXOFragmentation; + if (outputs == 1) { + if (inputs == 1) { + return SpendType.PerfectSpend; + } else { + return SpendType.Consolidation; + } + } else if (outputs == 2) { + return SpendType.SimpleSpend; } else { - if (outputs === 1) return SpendType.Consolidation; - return SpendType.MixingOrCoinJoin; + if (inputs < outputs) { + return SpendType.UTXOFragmentation; + } else { + return SpendType.MixingOrCoinJoin; + } } } From 7fc3f81860487baf96f74c5add66668ee5f0f183 Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Fri, 30 Aug 2024 02:53:31 +0530 Subject: [PATCH 89/92] Update packages/caravan-health/src/waste.ts Co-authored-by: buck --- packages/caravan-health/src/waste.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 56ea5862..036613c5 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -47,6 +47,8 @@ export class WasteMetrics extends WalletMetrics { Definition : Ratio of the fees paid by the wallet transactions to the amount spent in the transaction. + + In the future, we can make this more accurate by comparing fees to the fee market at the time the transaction was sent. This will indicate if transactions typically pay within or out of the range of the rest of the market. Calculation : We can compare this ratio against the fiat charges for cross-border transactions. From 5b7989d7964d8f3a9b3bf5ae3e69437d9993290f Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Fri, 30 Aug 2024 02:53:39 +0530 Subject: [PATCH 90/92] Update packages/caravan-health/src/waste.ts Co-authored-by: buck --- packages/caravan-health/src/waste.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/waste.ts b/packages/caravan-health/src/waste.ts index 036613c5..667dc7d6 100644 --- a/packages/caravan-health/src/waste.ts +++ b/packages/caravan-health/src/waste.ts @@ -43,7 +43,7 @@ export class WasteMetrics extends WalletMetrics { /* Name : - Fees To Amount Ration (F.A.R) + Fees To Amount Ratio (F.A.R) Definition : Ratio of the fees paid by the wallet transactions to the amount spent in the transaction. From 5eec580260ba1875c28df3abe2553c1463b29cdb Mon Sep 17 00:00:00 2001 From: Harshil Jani Date: Fri, 30 Aug 2024 02:53:58 +0530 Subject: [PATCH 91/92] Update packages/caravan-health/src/privacy.ts Co-authored-by: buck --- packages/caravan-health/src/privacy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 2c649711..2c6637d2 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -81,7 +81,7 @@ export class PrivacyMetrics extends WalletMetrics { Name : Address Reuse Factor (ARF) Definition : - The address reuse factor is evaluates the amount being held by reused addresses with respect + The address reuse factor evaluates the amount being held by reused addresses with respect to the total amount. It signifies the privacy health of the wallet based on address reuse. Calculation : From eb43fa4198d0aeb32e29a246f37d5851096b3339 Mon Sep 17 00:00:00 2001 From: Harshil-Jani Date: Fri, 30 Aug 2024 02:58:33 +0530 Subject: [PATCH 92/92] utility.js -> spendType.js Signed-off-by: Harshil-Jani --- packages/caravan-health/src/privacy.test.ts | 2 +- packages/caravan-health/src/privacy.ts | 2 +- packages/caravan-health/src/{utility.ts => spendType.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/caravan-health/src/{utility.ts => spendType.ts} (100%) diff --git a/packages/caravan-health/src/privacy.test.ts b/packages/caravan-health/src/privacy.test.ts index 014b68e0..06229af4 100644 --- a/packages/caravan-health/src/privacy.test.ts +++ b/packages/caravan-health/src/privacy.test.ts @@ -1,6 +1,6 @@ import { PrivacyMetrics } from "./privacy"; import { AddressUtxos, SpendType, Transaction, Network } from "./types"; -import { determineSpendType, getSpendTypeScore } from "./utility"; +import { determineSpendType, getSpendTypeScore } from "./spendType"; const transactions: Transaction[] = [ // transactions[0] is a perfect spend transaction diff --git a/packages/caravan-health/src/privacy.ts b/packages/caravan-health/src/privacy.ts index 2c6637d2..fee0959c 100644 --- a/packages/caravan-health/src/privacy.ts +++ b/packages/caravan-health/src/privacy.ts @@ -1,7 +1,7 @@ import { SpendType, MultisigAddressType, Network, Transaction } from "./types"; import { getAddressType } from "@caravan/bitcoin"; import { WalletMetrics } from "./wallet"; -import { determineSpendType, getSpendTypeScore } from "./utility"; +import { determineSpendType, getSpendTypeScore } from "./spendType"; // Deniability Factor is a normalizing quantity that increases the score by a certain factor in cases of self-payment. // More about deniability : https://www.truthcoin.info/blog/deniability/ diff --git a/packages/caravan-health/src/utility.ts b/packages/caravan-health/src/spendType.ts similarity index 100% rename from packages/caravan-health/src/utility.ts rename to packages/caravan-health/src/spendType.ts