Skip to content

Commit

Permalink
fix: convert date to mtime in glob source (#106)
Browse files Browse the repository at this point in the history
The UnixFS importer requires mtimes to follow the `Mtime` interface
so convert files/folders read from glob source.
  • Loading branch information
achingbrain authored Dec 3, 2023
1 parent 2421ee2 commit cd9e903
Show file tree
Hide file tree
Showing 9 changed files with 147 additions and 20 deletions.
2 changes: 1 addition & 1 deletion packages/interop/.aegir.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default {
host: '127.0.0.1',
port: ipfsdPort
}, {
ipfsBin: (await import('go-ipfs')).default.path(),
ipfsBin: (await import('kubo')).default.path(),
kuboRpcModule: kuboRpcClient,
ipfsOptions: {
config: {
Expand Down
5 changes: 2 additions & 3 deletions packages/interop/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
"test:chrome": "aegir test -t browser --cov",
"test:chrome-webworker": "aegir test -t webworker",
"test:firefox": "aegir test -t browser -- --browser firefox",
"test:firefox-webworker": "aegir test -t webworker -- --browser firefox",
"test:node": "aegir test -t node --cov",
"test:electron-main": "aegir test -t electron-main"
},
Expand All @@ -58,7 +57,7 @@
"aegir": "^41.0.0",
"blockstore-core": "^4.0.1",
"datastore-core": "^9.0.3",
"go-ipfs": "^0.22.0",
"kubo": "^0.24.0",
"helia": "^2.0.1",
"ipfs-core-types": "^0.14.0",
"ipfs-unixfs-importer": "^15.1.0",
Expand All @@ -72,7 +71,7 @@
},
"browser": {
"./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js",
"go-ipfs": false
"kubo": false
},
"private": true
}
6 changes: 5 additions & 1 deletion packages/interop/test/fixtures/create-helia.browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { all } from '@libp2p/websockets/filters'
import { MemoryBlockstore } from 'blockstore-core'
import { MemoryDatastore } from 'datastore-core'
import { createHelia } from 'helia'
import { bitswap } from 'helia/block-brokers'
import { createLibp2p, type Libp2pOptions } from 'libp2p'
import { identifyService } from 'libp2p/identify'
import type { Helia } from '@helia/interface'
Expand Down Expand Up @@ -39,7 +40,10 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise<Heli
const helia = await createHelia({
libp2p,
blockstore,
datastore
datastore,
blockBrokers: [
bitswap()
]
})

return helia
Expand Down
6 changes: 5 additions & 1 deletion packages/interop/test/fixtures/create-helia.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tcp } from '@libp2p/tcp'
import { MemoryBlockstore } from 'blockstore-core'
import { MemoryDatastore } from 'datastore-core'
import { createHelia } from 'helia'
import { bitswap } from 'helia/block-brokers'
import { createLibp2p, type Libp2pOptions } from 'libp2p'
import { identifyService } from 'libp2p/identify'
import type { Helia } from '@helia/interface'
Expand Down Expand Up @@ -32,7 +33,10 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise<Heli
const helia = await createHelia({
libp2p,
blockstore,
datastore
datastore,
blockBrokers: [
bitswap()
]
})

return helia
Expand Down
3 changes: 1 addition & 2 deletions packages/interop/test/fixtures/create-kubo.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped
import * as goIpfs from 'go-ipfs'
import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl'
import * as goIpfs from 'kubo'
import * as kuboRpcClient from 'kubo-rpc-client'
import mergeOptions from 'merge-options'
import { isElectronMain, isNode } from 'wherearewe'
Expand Down
3 changes: 2 additions & 1 deletion packages/unixfs/src/utils/glob-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import fsp from 'fs/promises'
import Path from 'path'
import glob from 'it-glob'
import { InvalidParametersError } from '../errors.js'
import { toMtime } from './to-mtime.js'
import type { MtimeLike } from 'ipfs-unixfs'
import type { ImportCandidateStream } from 'ipfs-unixfs-importer'

Expand Down Expand Up @@ -84,7 +85,7 @@ export async function * globSource (cwd: string, pattern: string, options: GlobS
path: toPosix(p.replace(cwd, '')),
content: stat.isFile() ? fs.createReadStream(p) : undefined,
mode,
mtime
mtime: toMtime(mtime)
}
}
}
Expand Down
57 changes: 57 additions & 0 deletions packages/unixfs/src/utils/to-mtime.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import type { Mtime, MtimeLike } from 'ipfs-unixfs'

export function toMtime (mtimeLike: MtimeLike): Mtime
export function toMtime (mtimeLike?: MtimeLike): Mtime | undefined
export function toMtime (mtimeLike?: MtimeLike): Mtime | undefined {
if (mtimeLike == null) {
return undefined
}

if (isMtime(mtimeLike)) {
return mtimeLike
}

if (mtimeLike instanceof Date) {
return dateToTimespec(mtimeLike)
}

if (Array.isArray(mtimeLike)) {
const output: Mtime = {
secs: BigInt(mtimeLike[0])
}

if (mtimeLike.length > 1) {
output.nsecs = mtimeLike[1]
}

return output
}

if (typeof mtimeLike.Seconds === 'number') {
const output: Mtime = {
secs: BigInt(mtimeLike.Seconds)
}

if (mtimeLike.FractionalNanoseconds != null) {
output.nsecs = mtimeLike.FractionalNanoseconds
}

return output
}

throw new Error('Cannot convert object to mtime')
}

function dateToTimespec (date: Date): Mtime {
const ms = date.getTime()
const secs = Math.floor(ms / 1000)

return {
secs: BigInt(secs),
nsecs: (ms - (secs * 1000)) * 1000
}
}

function isMtime (obj: any): obj is Mtime {
return typeof obj.secs === 'bigint'
}
24 changes: 13 additions & 11 deletions packages/unixfs/test/utils/glob-source.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { expect } from 'aegir/chai'
import all from 'it-all'
import { isNode } from 'wherearewe'
import { globSource } from '../../src/utils/glob-source.js'
import { toMtime } from '../../src/utils/to-mtime.js'
import type { Mtime } from 'ipfs-unixfs'

function fixtureDir (): string {
const filename = fileURLToPath(import.meta.url)
Expand All @@ -23,8 +25,8 @@ function findMode (file: string): number {
return fs.statSync(fixture(file)).mode
}

function findMtime (file: string): Date {
return fs.statSync(fixture(file)).mtime
function findMtime (file: string): Mtime {
return toMtime(fs.statSync(fixture(file)).mtime)
}

describe('glob-source', () => {
Expand Down Expand Up @@ -228,28 +230,28 @@ describe('glob-source', () => {
}

const result = await all(globSource(fixtureDir(), '{dir,dir/**/*}', {
mtime: new Date(5)
mtime: toMtime(new Date(5))
}))

expect(result).to.have.lengthOf(6)
expect(result).to.containSubset([{
path: '/dir',
mtime: new Date(5)
mtime: toMtime(new Date(5))
}, {
path: '/dir/file-1.txt',
mtime: new Date(5)
mtime: toMtime(new Date(5))
}, {
path: '/dir/file-2.js',
mtime: new Date(5)
mtime: toMtime(new Date(5))
}, {
path: '/dir/file-3.css',
mtime: new Date(5)
mtime: toMtime(new Date(5))
}, {
path: '/dir/nested-dir',
mtime: new Date(5)
mtime: toMtime(new Date(5))
}, {
path: '/dir/nested-dir/other.txt',
mtime: new Date(5)
mtime: toMtime(new Date(5))
}])
})

Expand All @@ -274,7 +276,7 @@ describe('glob-source', () => {
mtime: [5, 0]
}))

expect(result).to.have.deep.nested.property('[0].mtime', [5, 0])
expect(result).to.have.deep.nested.property('[0].mtime', toMtime([5, 0]))
})

it('overrides mtime for file with UnixFS timespec', async function () {
Expand All @@ -286,6 +288,6 @@ describe('glob-source', () => {
mtime: { Seconds: 5, FractionalNanoseconds: 0 }
}))

expect(result).to.have.deep.nested.property('[0].mtime', { Seconds: 5, FractionalNanoseconds: 0 })
expect(result).to.have.deep.nested.property('[0].mtime', toMtime({ Seconds: 5, FractionalNanoseconds: 0 }))
})
})
61 changes: 61 additions & 0 deletions packages/unixfs/test/utils/to-mtime.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/* eslint-env mocha */

import { expect } from 'aegir/chai'
import { toMtime } from '../../src/utils/to-mtime.js'

describe('to-mtime', () => {
it('should survive undefined', async function () {
const result = toMtime()

expect(result).to.equal(undefined)
})

it('should convert a date', async function () {
const input = new Date()
const result = toMtime(input)

expect(result?.secs).to.equal(BigInt(Math.floor(input.getTime() / 1000)))
})

it('should convert a timespec', async function () {
const input = {
Seconds: 100
}
const result = toMtime(input)

expect(result?.secs).to.equal(BigInt(input.Seconds))
expect(result?.nsecs).to.be.undefined()
})

it('should convert a timespec with fractional nanoseconds', async function () {
const input = {
Seconds: 100,
FractionalNanoseconds: 5
}
const result = toMtime(input)

expect(result?.secs).to.equal(BigInt(input.Seconds))
expect(result?.nsecs).to.equal(input.FractionalNanoseconds)
})

it('should convert a mtime', async function () {
const input = {
secs: 100n
}
const result = toMtime(input)

expect(result?.secs).to.equal(input.secs)
expect(result?.nsecs).to.be.undefined()
})

it('should convert a mtime with fractional nanoseconds', async function () {
const input = {
secs: 100n,
nsecs: 5
}
const result = toMtime(input)

expect(result?.secs).to.equal(input.secs)
expect(result?.nsecs).to.equal(input.nsecs)
})
})

0 comments on commit cd9e903

Please sign in to comment.