diff --git a/.gitignore b/.gitignore index 7ad9e674..7d54fd58 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ node_modules package-lock.json yarn.lock .vscode +.env +.envrc +.tool-versions diff --git a/package.json b/package.json index e9b39b85..c95b525f 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^41.1.14", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index cc52af59..381983c0 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -12,7 +12,7 @@ export default { host: '127.0.0.1', port: ipfsdPort }, { - ipfsBin: (await import('go-ipfs')).default.path(), + ipfsBin: (await import('kubo')).default.path(), kuboRpcModule: kuboRpcClient, ipfsOptions: { config: { diff --git a/packages/interop/package.json b/packages/interop/package.json index 8b13403c..a224d226 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -60,16 +60,15 @@ "@libp2p/peer-id-factory": "^3.0.3", "@libp2p/tcp": "^8.0.4", "@libp2p/websockets": "^7.0.4", - "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfsd-ctl": "^13.0.0", "ipns": "^7.0.1", "it-all": "^3.0.2", "it-last": "^3.0.1", "it-map": "^3.0.3", + "kubo": "^0.24.0", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.46.6", "merge-options": "^3.0.4", @@ -79,7 +78,7 @@ }, "browser": { "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", - "go-ipfs": false + "kubo": false }, "private": true } diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index 6c07ea3c..bef69b9b 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -131,9 +131,11 @@ keyTypes.forEach(type => { message: 'Kubo could not find Helia on the DHT' }) - name = ipns(helia, [ - dht(helia) - ]) + name = ipns(helia, { + routers: [ + dht(helia) + ] + }) } afterEach(async () => { diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 518c8a68..0c7c5a5f 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,7 +1,5 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */ -// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped -import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kubo from 'kubo' import * as kuboRpcClient from 'kubo-rpc-client' import mergeOptions from 'merge-options' import { isElectronMain, isNode } from 'wherearewe' @@ -9,7 +7,7 @@ import { isElectronMain, isNode } from 'wherearewe' export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { const opts = mergeOptions({ kuboRpcModule: kuboRpcClient, - ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + ipfsBin: isNode || isElectronMain ? kubo.path() : undefined, test: true, endpoint: process.env.IPFSD_SERVER, ipfsOptions: { diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index 22fff4a3..ff32652d 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -52,9 +52,11 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { // connect the two nodes await connect(helia, kubo, '/meshsub/1.1.0') - name = ipns(helia, [ - pubsub(helia) - ]) + name = ipns(helia, { + routers: [ + pubsub(helia) + ] + }) }) afterEach(async () => { diff --git a/packages/ipns/package.json b/packages/ipns/package.json index cb19b4a5..38b87b6c 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -46,6 +46,10 @@ "./routing": { "types": "./dist/src/routing/index.d.ts", "import": "./dist/src/routing/index.js" + }, + "./dns-resolvers": { + "types": "./dist/src/dns-resolvers/index.d.ts", + "import": "./dist/src/dns-resolvers/index.js" } }, "eslintConfig": { @@ -155,10 +159,11 @@ "release": "aegir release" }, "dependencies": { - "@libp2p/interface": "^0.1.2", "@libp2p/kad-dht": "^10.0.11", "@libp2p/logger": "^3.0.2", "@libp2p/peer-id": "^3.0.2", + "dns-over-http-resolver": "^2.1.3", + "dns-packet": "^5.6.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", "ipns": "^7.0.1", @@ -169,13 +174,17 @@ "uint8arrays": "^4.0.3" }, "devDependencies": { + "@libp2p/interface": "^0.1.4", "@libp2p/peer-id-factory": "^3.0.3", - "aegir": "^41.0.0", + "@types/dns-packet": "^5.6.4", "datastore-core": "^9.0.3", "sinon": "^17.0.0", "sinon-ts": "^1.0.0" }, "browser": { - "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" + "./dist/src/dns-resolvers/resolver.js": "./dist/src/dns-resolvers/resolver.browser.js" + }, + "typedoc": { + "entryPoint": "./src/index.ts" } } diff --git a/packages/ipns/src/dns-resolvers/default.ts b/packages/ipns/src/dns-resolvers/default.ts new file mode 100644 index 00000000..0586b72c --- /dev/null +++ b/packages/ipns/src/dns-resolvers/default.ts @@ -0,0 +1,9 @@ +import { MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js' +import resolve from './resolver.js' +import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js' + +export function defaultResolver (): DNSResolver { + return async (domain: string, options: ResolveDnsLinkOptions = {}): Promise => { + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options) + } +} diff --git a/packages/ipns/src/dns-resolvers/dns-json-over-https.ts b/packages/ipns/src/dns-resolvers/dns-json-over-https.ts new file mode 100644 index 00000000..ece490df --- /dev/null +++ b/packages/ipns/src/dns-resolvers/dns-json-over-https.ts @@ -0,0 +1,90 @@ +/* eslint-env browser */ + +import PQueue from 'p-queue' +import { CustomProgressEvent } from 'progress-events' +import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js' +import { TLRU } from '../utils/tlru.js' +import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js' + +// Avoid sending multiple queries for the same hostname by caching results +const cache = new TLRU(1000) +// This TTL will be used if the remote service does not return one +const ttl = 60 * 1000 + +/** + * Uses the RFC 8427 'application/dns-json' content-type to resolve DNS queries. + * + * Supports and server that uses the same schema as Google's DNS over HTTPS + * resolver. + * + * This resolver needs fewer dependencies than the regular DNS-over-HTTPS + * resolver so can result in a smaller bundle size and consequently is preferred + * for browser use. + * + * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/ + * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers + * @see https://dnsprivacy.org/public_resolvers/ + * @see https://datatracker.ietf.org/doc/html/rfc8427 + */ +export function dnsJsonOverHttps (url: string): DNSResolver { + // browsers limit concurrent connections per host, + // we don't want preload calls to exhaust the limit (~6) + const httpQueue = new PQueue({ concurrency: 4 }) + + const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise => { + const searchParams = new URLSearchParams() + searchParams.set('name', fqdn) + searchParams.set('type', 'TXT') + + const query = searchParams.toString() + + // try cache first + if (options.nocache !== true && cache.has(query)) { + const response = cache.get(query) + + if (response != null) { + options.onProgress?.(new CustomProgressEvent('dnslink:cache', { detail: response })) + return response + } + } + + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: fqdn })) + + // query DNS-JSON over HTTPS server + const response = await httpQueue.add(async () => { + const res = await fetch(`${url}?${searchParams}`, { + headers: { + accept: 'application/dns-json' + }, + signal: options.signal + }) + + if (res.status !== 200) { + throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`) + } + + const query = new URL(res.url).search.slice(1) + const json: DNSResponse = await res.json() + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: json })) + + const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json) + + cache.set(query, ipfsPath, answer.TTL ?? ttl) + + return ipfsPath + }, { + signal: options.signal + }) + + if (response == null) { + throw new Error('No DNS response received') + } + + return response + } + + return async (domain: string, options: ResolveDnsLinkOptions = {}) => { + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options) + } +} diff --git a/packages/ipns/src/dns-resolvers/dns-over-https.ts b/packages/ipns/src/dns-resolvers/dns-over-https.ts new file mode 100644 index 00000000..b373f06f --- /dev/null +++ b/packages/ipns/src/dns-resolvers/dns-over-https.ts @@ -0,0 +1,146 @@ +/* eslint-env browser */ + +import { Buffer } from 'buffer' +import dnsPacket, { type DecodedPacket } from 'dns-packet' +import { base64url } from 'multiformats/bases/base64' +import PQueue from 'p-queue' +import { CustomProgressEvent } from 'progress-events' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js' +import { TLRU } from '../utils/tlru.js' +import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js' + +// Avoid sending multiple queries for the same hostname by caching results +const cache = new TLRU(1000) +// This TTL will be used if the remote service does not return one +const ttl = 60 * 1000 + +/** + * Uses the RFC 1035 'application/dns-message' content-type to resolve DNS + * queries. + * + * This resolver needs more dependencies than the non-standard + * DNS-JSON-over-HTTPS resolver so can result in a larger bundle size and + * consequently is not preferred for browser use. + * + * @see https://datatracker.ietf.org/doc/html/rfc1035 + * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-wireformat/ + * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers + * @see https://dnsprivacy.org/public_resolvers/ + */ +export function dnsOverHttps (url: string): DNSResolver { + // browsers limit concurrent connections per host, + // we don't want preload calls to exhaust the limit (~6) + const httpQueue = new PQueue({ concurrency: 4 }) + + const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise => { + const dnsQuery = dnsPacket.encode({ + type: 'query', + id: 0, + flags: dnsPacket.RECURSION_DESIRED, + questions: [{ + type: 'TXT', + name: fqdn + }] + }) + + const searchParams = new URLSearchParams() + searchParams.set('dns', base64url.encode(dnsQuery).substring(1)) + + const query = searchParams.toString() + + // try cache first + if (options.nocache !== true && cache.has(query)) { + const response = cache.get(query) + + if (response != null) { + options.onProgress?.(new CustomProgressEvent('dnslink:cache', { detail: response })) + return response + } + } + + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: fqdn })) + + // query DNS over HTTPS server + const response = await httpQueue.add(async () => { + const res = await fetch(`${url}?${searchParams}`, { + headers: { + accept: 'application/dns-message' + }, + signal: options.signal + }) + + if (res.status !== 200) { + throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`) + } + + const query = new URL(res.url).search.slice(1) + const buf = await res.arrayBuffer() + // map to expected response format + const json = toDNSResponse(dnsPacket.decode(Buffer.from(buf))) + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: json })) + + const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json) + + cache.set(query, ipfsPath, answer.TTL ?? ttl) + + return ipfsPath + }, { + signal: options.signal + }) + + if (response == null) { + throw new Error('No DNS response received') + } + + return response + } + + return async (domain: string, options: ResolveDnsLinkOptions = {}) => { + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options) + } +} + +function toDNSResponse (response: DecodedPacket): DNSResponse { + const txtType = 16 + + return { + Status: 0, + TC: response.flag_tc ?? false, + RD: response.flag_rd ?? false, + RA: response.flag_ra ?? false, + AD: response.flag_ad ?? false, + CD: response.flag_cd ?? false, + Question: response.questions?.map(q => ({ + name: q.name, + type: txtType + })) ?? [], + Answer: response.answers?.map(a => { + if (a.type !== 'TXT' || a.data.length < 1) { + return { + name: a.name, + type: txtType, + TTL: 0, + data: 'invalid' + } + } + + if (!Buffer.isBuffer(a.data[0])) { + return { + name: a.name, + type: txtType, + TTL: a.ttl ?? ttl, + data: String(a.data[0]) + } + } + + return { + name: a.name, + type: txtType, + TTL: a.ttl ?? ttl, + data: uint8ArrayToString(a.data[0]) + } + }) ?? [] + } +} diff --git a/packages/ipns/src/dns-resolvers/index.ts b/packages/ipns/src/dns-resolvers/index.ts new file mode 100644 index 00000000..ee8c0db8 --- /dev/null +++ b/packages/ipns/src/dns-resolvers/index.ts @@ -0,0 +1,2 @@ +export { dnsOverHttps } from './dns-over-https.js' +export { dnsJsonOverHttps } from './dns-json-over-https.js' diff --git a/packages/ipns/src/dns-resolvers/resolver.browser.ts b/packages/ipns/src/dns-resolvers/resolver.browser.ts new file mode 100644 index 00000000..561672f7 --- /dev/null +++ b/packages/ipns/src/dns-resolvers/resolver.browser.ts @@ -0,0 +1,50 @@ +import Resolver from 'dns-over-http-resolver' +import PQueue from 'p-queue' +import { CustomProgressEvent } from 'progress-events' +import { resolveFn, type DNSResponse } from '../utils/dns.js' +import { TLRU } from '../utils/tlru.js' +import type { DNSResolver } from '../index.js' + +const cache = new TLRU(1000) +// We know browsers themselves cache DNS records for at least 1 minute, +// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426 +const ttl = 60 * 1000 + +// browsers limit concurrent connections per host, +// we don't want to exhaust the limit (~6) +const httpQueue = new PQueue({ concurrency: 4 }) + +const resolve: DNSResolver = async function resolve (domain, options = {}) { + const resolver = new Resolver({ maxCache: 0 }) + // try cache first + if (options.nocache !== true && cache.has(domain)) { + const response = cache.get(domain) + + if (response != null) { + options?.onProgress?.(new CustomProgressEvent('dnslink:cache', { detail: response })) + return response + } + } + + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: domain })) + + // Add the query to the queue + const response = await httpQueue.add(async () => { + const dnslinkRecord = await resolveFn(resolver, domain) + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: dnslinkRecord })) + cache.set(domain, dnslinkRecord, ttl) + + return dnslinkRecord + }, { + signal: options?.signal + }) + + if (response == null) { + throw new Error('No DNS response received') + } + + return response +} + +export default resolve diff --git a/packages/ipns/src/dns-resolvers/resolver.ts b/packages/ipns/src/dns-resolvers/resolver.ts new file mode 100644 index 00000000..9b63f864 --- /dev/null +++ b/packages/ipns/src/dns-resolvers/resolver.ts @@ -0,0 +1,25 @@ +import { Resolver } from 'dns/promises' +import { CustomProgressEvent } from 'progress-events' +import { resolveFn, type DNSResponse } from '../utils/dns.js' +import type { DNSResolver } from '../index.js' + +const resolve: DNSResolver = async function resolve (domain, options = {}) { + const resolver = new Resolver() + const listener = (): void => { + resolver.cancel() + } + + options.signal?.addEventListener('abort', listener) + + try { + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: domain })) + const dnslinkRecord = await resolveFn(resolver, domain) + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: dnslinkRecord })) + return dnslinkRecord + } finally { + options.signal?.removeEventListener('abort', listener) + } +} + +export default resolve diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index dedc2277..cb8ca507 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -5,16 +5,20 @@ * * @example * + * With {@link IPNSRouting} routers: + * * ```typescript * import { createHelia } from 'helia' * import { dht, pubsub } from '@helia/ipns/routing' * import { unixfs } from '@helia/unixfs' * * const helia = await createHelia() - * const name = ipns(helia, [ - * dht(helia), - * pubsub(helia) - * ]) + * const name = ipns(helia, { + * routers: [ + * dht(helia), + * pubsub(helia) + * ] + * }) * * // create a public key to publish as an IPNS name * const keyInfo = await helia.libp2p.keychain.createKey('my-key') @@ -33,13 +37,76 @@ * * @example * + * With default {@link DNSResolver} resolvers: + * + * ```typescript + * import { createHelia } from 'helia' + * import { dht, pubsub } from '@helia/ipns/routing' + * import { unixfs } from '@helia/unixfs' + * + * const helia = await createHelia() + * const name = ipns(helia, { + * resolvers: [ + * dnsOverHttps('https://private-dns-server.me/dns-query'), + * ] + * }) + * + * const cid = name.resolveDns('some-domain-with-dnslink-entry.com') + * ``` + * + * @example + * + * Calling `resolveDns` with the `@helia/ipns` instance: + * * ```typescript - * // resolve a CID from a TXT record in a DNS zone file, eg: - * // > dig ipfs.io TXT + * // resolve a CID from a TXT record in a DNS zone file, using the default + * // resolver for the current platform eg: + * // > dig _dnslink.ipfs.io TXT * // ;; ANSWER SECTION: - * // ipfs.io. 435 IN TXT "dnslink=/ipfs/Qmfoo" + * // _dnslink.ipfs.io. 60 IN TXT "dnslink=/ipns/website.ipfs.io" + * // > dig _dnslink.website.ipfs.io TXT + * // ;; ANSWER SECTION: + * // _dnslink.website.ipfs.io. 60 IN TXT "dnslink=/ipfs/QmWebsite" * * const cid = name.resolveDns('ipfs.io') + * + * console.info(cid) + * // QmWebsite + * ``` + * + * @example + * + * This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This + * uses binary DNS records so requires extra dependencies to process the + * response which can increase browser bundle sizes. + * + * If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. + * + * ```typescript + * // use DNS-Over-HTTPS + * import { dnsOverHttps } from '@helia/ipns/dns-resolvers' + * + * const cid = name.resolveDns('ipfs.io', { + * resolvers: [ + * dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + * ] + * }) + * ``` + * + * @example + * + * DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can + * result in a smaller browser bundle due to the response being plain JSON. + * + * ```typescript + * // use DNS-JSON-Over-HTTPS + * import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' + * + * const cid = name.resolveDns('ipfs.io', { + * resolvers: [ + * dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + * ] + * }) * ``` */ @@ -51,9 +118,10 @@ import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' import { CID } from 'multiformats/cid' import { CustomProgressEvent } from 'progress-events' +import { defaultResolver } from './dns-resolvers/default.js' import { localStore, type LocalStore } from './routing/local-store.js' -import { resolveDnslink } from './utils/resolve-dns-link.js' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' +import type { DNSResponse } from './utils/dns.js' import type { AbortOptions } from '@libp2p/interface' import type { PeerId } from '@libp2p/interface/peer-id' import type { Datastore } from 'interface-datastore' @@ -83,6 +151,11 @@ export type RepublishProgressEvents = ProgressEvent<'ipns:republish:success', IPNSRecord> | ProgressEvent<'ipns:republish:error', { record: IPNSRecord, err: Error }> +export type ResolveDnsLinkProgressEvents = + ProgressEvent<'dnslink:cache', string> | + ProgressEvent<'dnslink:query', string> | + ProgressEvent<'dnslink:answer', DNSResponse> + export interface PublishOptions extends AbortOptions, ProgressOptions { /** * Time duration of the record in ms (default: 24hrs) @@ -108,11 +181,35 @@ export interface ResolveOptions extends AbortOptions, ProgressOptions { + /** + * Do not use cached DNS entries (default: false) + */ + nocache?: boolean +} + +export interface DNSResolver { + (domain: string, options?: ResolveDnsLinkOptions): Promise +} + +export interface ResolveDNSOptions extends AbortOptions, ProgressOptions { + /** + * Do not query the network for the IPNS record (default: false) + */ + offline?: boolean + /** * Do not use cached DNS entries (default: false) */ nocache?: boolean + + /** + * These resolvers will be used to resolve the dnslink entries, if unspecified node will + * fall back to the `dns` module and browsers fall back to querying google/cloudflare DoH + * + * @see https://github.com/ipfs/helia-ipns/pull/55#discussion_r1270096881 + */ + resolvers?: DNSResolver[] } export interface RepublishOptions extends AbortOptions, ProgressOptions { @@ -157,10 +254,12 @@ class DefaultIPNS implements IPNS { private readonly routers: IPNSRouting[] private readonly localStore: LocalStore private timeout?: ReturnType + private readonly defaultResolvers: DNSResolver[] - constructor (components: IPNSComponents, routers: IPNSRouting[] = []) { + constructor (components: IPNSComponents, routers: IPNSRouting[] = [], resolvers: DNSResolver[] = []) { this.routers = routers this.localStore = localStore(components.datastore) + this.defaultResolvers = resolvers.length > 0 ? resolvers : [defaultResolver()] } async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { @@ -201,7 +300,11 @@ class DefaultIPNS implements IPNS { } async resolveDns (domain: string, options: ResolveDNSOptions = {}): Promise { - const dnslink = await resolveDnslink(domain, options) + const resolvers = options.resolvers ?? this.defaultResolvers + + const dnslink = await Promise.any( + resolvers.map(async resolver => resolver(domain, options)) + ) return this.#resolve(dnslink, options) } @@ -298,8 +401,13 @@ class DefaultIPNS implements IPNS { } } -export function ipns (components: IPNSComponents, routers: IPNSRouting[] = []): IPNS { - return new DefaultIPNS(components, routers) +export interface IPNSOptions { + routers?: IPNSRouting[] + resolvers?: DNSResolver[] +} + +export function ipns (components: IPNSComponents, { routers = [], resolvers = [] }: IPNSOptions): IPNS { + return new DefaultIPNS(components, routers, resolvers) } export { ipnsValidator } diff --git a/packages/ipns/src/utils/dns.ts b/packages/ipns/src/utils/dns.ts new file mode 100644 index 00000000..0590efcf --- /dev/null +++ b/packages/ipns/src/utils/dns.ts @@ -0,0 +1,126 @@ +import { CodeError } from '@libp2p/interface/errors' +import * as isIPFS from 'is-ipfs' +import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js' + +export interface Question { + name: string + type: number +} + +export interface Answer { + name: string + type: number + TTL: number + data: string +} + +export interface DNSResponse { + Status: number + TC: boolean + RD: boolean + RA: boolean + AD: boolean + CD: boolean + Question: Question[] + Answer?: Answer[] +} + +export const ipfsPathForAnswer = (answer: Answer): string => { + let data = answer.data + + if (data.startsWith('"')) { + data = data.substring(1) + } + + if (data.endsWith('"')) { + data = data.substring(0, data.length - 1) + } + + return data.replace('dnslink=', '') +} + +export const ipfsPathAndAnswer = (domain: string, response: DNSResponse): { ipfsPath: string, answer: Answer } => { + const answer = findDNSLinkAnswer(domain, response) + + return { + ipfsPath: ipfsPathForAnswer(answer), + answer + } +} + +export const findDNSLinkAnswer = (domain: string, response: DNSResponse): Answer => { + const answer = response.Answer?.filter(a => a.data.includes('dnslink=/ipfs') || a.data.includes('dnslink=/ipns')).pop() + + if (answer == null) { + throw new CodeError(`No dnslink records found for domain: ${domain}`, 'ERR_DNSLINK_NOT_FOUND') + } + + return answer +} + +export const MAX_RECURSIVE_DEPTH = 32 + +export const recursiveResolveDnslink = async (domain: string, depth: number, resolve: DNSResolver, options: ResolveDnsLinkOptions = {}): Promise => { + if (depth === 0) { + throw new Error('recursion limit exceeded') + } + + let dnslinkRecord: string + + try { + dnslinkRecord = await resolve(domain, options) + } catch (err: any) { + // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error + if (err.code !== 'ENOTFOUND' && err.code !== 'ERR_DNSLINK_NOT_FOUND' && err.code !== 'ENODATA') { + throw err + } + + if (domain.startsWith('_dnslink.')) { + // The supplied domain contains a _dnslink component + // Check the non-_dnslink domain + domain = domain.replace('_dnslink.', '') + } else { + // Check the _dnslink subdomain + domain = `_dnslink.${domain}` + } + + // If this throws then we propagate the error + dnslinkRecord = await resolve(domain, options) + } + + const result = dnslinkRecord.replace('dnslink=', '') + // result is now a `/ipfs/` or `/ipns/` string + const domainOrCID = result.split('/')[2] // e.g. ["", "ipfs", ""] + const isIPFSCID = isIPFS.cid(domainOrCID) + + // if the result is a CID, or depth is 1, we've reached the end of the recursion + // if depth is 1, another recursive call will be made, but it would throw. + // we could return if depth is 1 and allow users to handle, but that may be a breaking change + if (isIPFSCID) { + return result + } + + return recursiveResolveDnslink(domainOrCID, depth - 1, resolve, options) +} + +interface DnsResolver { + resolveTxt(domain: string): Promise +} + +const DNSLINK_REGEX = /^dnslink=.+$/ +export const resolveFn = async (resolver: DnsResolver, domain: string): Promise => { + const records = await resolver.resolveTxt(domain) + const dnslinkRecords = records.flat() + .filter(record => DNSLINK_REGEX.test(record)) + + // we now have dns text entries as an array of strings + // only records passing the DNSLINK_REGEX text are included + // TODO: support multiple dnslink records + const dnslinkRecord = dnslinkRecords[0] + + if (dnslinkRecord == null) { + throw new CodeError(`No dnslink records found for domain: ${domain}`, 'ERR_DNSLINK_NOT_FOUND') + } + + return dnslinkRecord +} diff --git a/packages/ipns/src/utils/resolve-dns-link.browser.ts b/packages/ipns/src/utils/resolve-dns-link.browser.ts deleted file mode 100644 index 3a6d7074..00000000 --- a/packages/ipns/src/utils/resolve-dns-link.browser.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* eslint-env browser */ - -import PQueue from 'p-queue' -import { TLRU } from './tlru.js' -import type { AbortOptions } from '@libp2p/interface' - -// Avoid sending multiple queries for the same hostname by caching results -const cache = new TLRU<{ Path: string, Message: string }>(1000) -// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884 -// However we know browsers themselves cache DNS records for at least 1 minute, -// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426 -const ttl = 60 * 1000 - -// browsers limit concurrent connections per host, -// we don't want preload calls to exhaust the limit (~6) -const httpQueue = new PQueue({ concurrency: 4 }) - -const ipfsPath = (response: { Path: string, Message: string }): string => { - if (response.Path != null) { - return response.Path - } - throw new Error(response.Message) -} - -export interface ResolveDnsLinkOptions extends AbortOptions { - nocache?: boolean -} - -export async function resolveDnslink (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise { // eslint-disable-line require-await - const resolve = async (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise => { - // @ts-expect-error - URLSearchParams does not take boolean options, only strings - const searchParams = new URLSearchParams(opts) - searchParams.set('arg', fqdn) - - // try cache first - const query = searchParams.toString() - if (opts.nocache !== true && cache.has(query)) { - const response = cache.get(query) - - if (response != null) { - return ipfsPath(response) - } - } - - // fallback to delegated DNS resolver - const response = await httpQueue.add(async () => { - // Delegated HTTP resolver sending DNSLink queries to ipfs.io - // TODO: replace hardcoded host with configurable DNS over HTTPS: https://github.com/ipfs/js-ipfs/issues/2212 - const res = await fetch(`https://ipfs.io/api/v0/dns?${searchParams}`) - const query = new URL(res.url).search.slice(1) - const json = await res.json() - cache.set(query, json, ttl) - - return json - }) - - return ipfsPath(response) - } - - return resolve(fqdn, opts) -} diff --git a/packages/ipns/src/utils/resolve-dns-link.ts b/packages/ipns/src/utils/resolve-dns-link.ts deleted file mode 100644 index 1975cf9f..00000000 --- a/packages/ipns/src/utils/resolve-dns-link.ts +++ /dev/null @@ -1,65 +0,0 @@ -import dns from 'dns' -import { promisify } from 'util' -import * as isIPFS from 'is-ipfs' -import type { AbortOptions } from '@libp2p/interface' - -const MAX_RECURSIVE_DEPTH = 32 - -export async function resolveDnslink (domain: string, options: AbortOptions = {}): Promise { - return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, options) -} - -async function recursiveResolveDnslink (domain: string, depth: number, options: AbortOptions = {}): Promise { - if (depth === 0) { - throw new Error('recursion limit exceeded') - } - - let dnslinkRecord - - try { - dnslinkRecord = await resolve(domain) - } catch (err: any) { - // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error - if (err.code !== 'ENOTFOUND' && err.code !== 'ERR_DNSLINK_NOT_FOUND' && err.code !== 'ENODATA') { - throw err - } - - if (domain.startsWith('_dnslink.')) { - // The supplied domain contains a _dnslink component - // Check the non-_dnslink domain - dnslinkRecord = await resolve(domain.replace('_dnslink.', '')) - } else { - // Check the _dnslink subdomain - const _dnslinkDomain = `_dnslink.${domain}` - // If this throws then we propagate the error - dnslinkRecord = await resolve(_dnslinkDomain) - } - } - - const result = dnslinkRecord.replace('dnslink=', '') - const domainOrCID = result.split('/')[2] - const isIPFSCID = isIPFS.cid(domainOrCID) - - if (isIPFSCID || depth === 0) { - return result - } - - return recursiveResolveDnslink(domainOrCID, depth - 1, options) -} - -async function resolve (domain: string, options: AbortOptions = {}): Promise { - const DNSLINK_REGEX = /^dnslink=.+$/ - const records = await promisify(dns.resolveTxt)(domain) - const dnslinkRecords = records.reduce((rs, r) => rs.concat(r), []) - .filter(record => DNSLINK_REGEX.test(record)) - - const dnslinkRecord = dnslinkRecords[0] - - // we now have dns text entries as an array of strings - // only records passing the DNSLINK_REGEX text are included - if (dnslinkRecord == null) { - throw new Error(`No dnslink records found for domain: ${domain}`) - } - - return dnslinkRecord -} diff --git a/packages/ipns/test/dns-resolvers.spec.ts b/packages/ipns/test/dns-resolvers.spec.ts new file mode 100644 index 00000000..6913933b --- /dev/null +++ b/packages/ipns/test/dns-resolvers.spec.ts @@ -0,0 +1,86 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { defaultResolver } from '../src/dns-resolvers/default.js' +import { dnsJsonOverHttps } from '../src/dns-resolvers/dns-json-over-https.js' +import { dnsOverHttps } from '../src/dns-resolvers/dns-over-https.js' +import type { DNSResolver } from '../src/index.js' + +const resolvers: Record = { + 'dns-json-over-https': dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), + 'dns-over-https': dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), + default: defaultResolver() +} + +describe('dns resolvers', () => { + Object.entries(resolvers).forEach(([name, resolver]) => { + it(`${name} should resolve`, async () => { + const result = await resolver('ipfs.io') + + expect(result).to.startWith('/ipfs') + }) + + it(`${name} should cache results`, async function () { + if (name === 'default' && globalThis.Buffer != null) { + // node dns uses OS-level caching + this.skip() + } + + let usedCache = false + + // resolve once + await resolver('ipfs.io') + + const makeCall = async (): Promise => { + await resolver('ipfs.io', { + onProgress: (evt) => { + if (evt.type.includes('dnslink:cache')) { + usedCache = true + } + } + }) + } + + for (let i = 0; i < 15; i++) { + // resolve again, should use the cache + // TTL can be pretty low which means this can be flaky if executed more slowly than the TTL + await makeCall() + if (usedCache) { + break + } + } + + expect(usedCache).to.be.true() + }) + + it(`${name} should skip cache results`, async function () { + if (name === 'default' && globalThis.Buffer != null) { + // node dns uses OS-level caching + this.skip() + } + + let usedCache = false + + // resolve once + await resolver('ipfs.io') + + // resolve again, should skip the cache + await resolver('ipfs.io', { + nocache: true, + onProgress: (evt) => { + if (evt.type.includes('dnslink:cache')) { + usedCache = true + } + } + }) + + expect(usedCache).to.be.false() + }) + + it(`${name} should abort if signal is aborted`, async () => { + const signal = AbortSignal.timeout(1) + + await expect(resolver('ipfs.io', { nocache: true, signal })).to.eventually.be.rejected() + }) + }) +}) diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index 184e835b..82c966d6 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -20,7 +20,7 @@ describe('publish', () => { routing = stubInterface() routing.get.throws(new Error('Not found')) - name = ipns({ datastore }, [routing]) + name = ipns({ datastore }, { routers: [routing] }) }) it('should publish an IPNS record with the default params', async function () { diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index e605da32..cd02d830 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -25,7 +25,7 @@ describe('resolve', () => { routing = stubInterface() routing.get.throws(new Error('Not found')) - name = ipns({ datastore }, [routing]) + name = ipns({ datastore }, { routers: [routing] }) }) it('should resolve a record', async () => { @@ -75,23 +75,6 @@ describe('resolve', () => { expect(resolvedValue.toString()).to.equal(cid.toV1().toString()) }) - it('should resolve /ipns/tableflip.io', async function () { - const domain = 'tableflip.io' - - try { - const resolvedValue = await name.resolveDns(domain) - - expect(resolvedValue).to.be.an.instanceOf(CID) - } catch (err: any) { - // happens when running tests offline - if (err.message.includes(`ECONNREFUSED ${domain}`) === true) { - return this.skip() - } - - throw err - } - }) - it('should emit progress events', async function () { const onProgress = Sinon.stub() const key = await createEd25519PeerId() diff --git a/packages/ipns/test/resolveDns.spec.ts b/packages/ipns/test/resolveDns.spec.ts new file mode 100644 index 00000000..a2288d90 --- /dev/null +++ b/packages/ipns/test/resolveDns.spec.ts @@ -0,0 +1,41 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import { type Datastore } from 'interface-datastore' +import { stub } from 'sinon' +import { type StubbedInstance, stubInterface } from 'sinon-ts' +import { type IPNSRouting, ipns } from '../src/index.js' + +describe('resolveDns', () => { + let routing: StubbedInstance + let datastore: Datastore + + beforeEach(async () => { + datastore = new MemoryDatastore() + routing = stubInterface() + routing.get.throws(new Error('Not found')) + }) + + it('should use resolvers passed in constructor', async () => { + const stubbedResolver1 = stub().returns('dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + + const name = ipns({ datastore }, { routers: [routing], resolvers: [stubbedResolver1] }) + const result = await name.resolveDns('foobar.baz', { nocache: true, offline: true }) + expect(stubbedResolver1.called).to.be.true() + expect(stubbedResolver1.calledWith('foobar.baz')).to.be.true() + expect(result.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + }) + + it('should allow overriding of resolvers passed in constructor', async () => { + const stubbedResolver1 = stub().returns('dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const stubbedResolver2 = stub().returns('dnslink=/ipfs/bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq') + + const name = ipns({ datastore }, { routers: [routing], resolvers: [stubbedResolver1] }) + const result = await name.resolveDns('foobar.baz', { nocache: true, offline: true, resolvers: [stubbedResolver2] }) + expect(stubbedResolver1.called).to.be.false() + expect(stubbedResolver2.called).to.be.true() + expect(stubbedResolver2.calledWith('foobar.baz')).to.be.true() + expect(result.toString()).to.equal('bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq') + }) +})