feat!: update to latest datastore interface (#144)

- Updates datastore interface
- Open/close are no longer part of the interface
- Converts this module to typescript
- Updates all dependencies

BREAKING CHANGE: update to latest datastore interface

Author: achingbrain

package.json

@@ -49,43 +49,43 @@
   "exports": {
     ".": {
       "types": "./dist/src/index.d.ts",
-      "import": "./src/index.js"
+      "import": "./dist/src/index.js"
     },
     "./base": {
-      "types": "./src/base.d.ts",
-      "import": "./src/base.js"
+      "types": "./dist/src/base.d.ts",
+      "import": "./dist/src/base.js"
     },
     "./errors": {
-      "types": "./src/errors.d.ts",
-      "import": "./src/errors.js"
+      "types": "./dist/src/errors.d.ts",
+      "import": "./dist/src/errors.js"
     },
     "./keytransform": {
-      "types": "./src/keytransform.d.ts",
-      "import": "./src/keytransform.js"
+      "types": "./dist/src/keytransform.d.ts",
+      "import": "./dist/src/keytransform.js"
     },
     "./memory": {
-      "types": "./src/memory.d.ts",
-      "import": "./src/memory.js"
+      "types": "./dist/src/memory.d.ts",
+      "import": "./dist/src/memory.js"
     },
     "./mount": {
-      "types": "./src/mount.d.ts",
-      "import": "./src/mount.js"
+      "types": "./dist/src/mount.d.ts",
+      "import": "./dist/src/mount.js"
     },
     "./namespace": {
-      "types": "./src/namespace.d.ts",
-      "import": "./src/namespace.js"
+      "types": "./dist/src/namespace.d.ts",
+      "import": "./dist/src/namespace.js"
     },
     "./shard": {
-      "types": "./src/shard.d.ts",
-      "import": "./src/shard.js"
+      "types": "./dist/src/shard.d.ts",
+      "import": "./dist/src/shard.js"
     },
     "./sharding": {
-      "types": "./src/sharding.d.ts",
-      "import": "./src/sharding.js"
+      "types": "./dist/src/sharding.d.ts",
+      "import": "./dist/src/sharding.js"
     },
     "./tiered": {
-      "types": "./src/tiered.d.ts",
-      "import": "./src/tiered.js"
+      "types": "./dist/src/tiered.d.ts",
+      "import": "./dist/src/tiered.js"
     }
   },
   "eslintConfig": {
@@ -196,19 +196,21 @@
   "dependencies": {
     "@libp2p/logger": "^2.0.0",
     "err-code": "^3.0.1",
-    "interface-datastore": "^7.0.0",
+    "interface-store": "^4.0.0",
     "it-all": "^2.0.0",
     "it-drain": "^2.0.0",
     "it-filter": "^2.0.0",
     "it-map": "^2.0.0",
     "it-merge": "^2.0.0",
     "it-pipe": "^2.0.3",
     "it-pushable": "^3.0.0",
+    "it-sort": "^2.0.1",
     "it-take": "^2.0.0",
     "uint8arrays": "^4.0.2"
   },
   "devDependencies": {
-    "aegir": "^37.4.5",
-    "interface-datastore-tests": "^3.0.0"
+    "aegir": "^38.1.7",
+    "interface-datastore": "^8.0.0",
+    "interface-datastore-tests": "^4.0.0"
   }
 }

src/base.js

@@ -0,0 +1,149 @@
+import sort from 'it-sort'
+import drain from 'it-drain'
+import filter from 'it-filter'
+import take from 'it-take'
+import type { Batch, Datastore, Key, KeyQuery, Options, Pair, Query } from 'interface-datastore'
+import type { AwaitIterable } from 'interface-store'
+
+export class BaseDatastore implements Datastore {
+  async open (): Promise<void> {
+
+  }
+
+  async close (): Promise<void> {
+
+  }
+
+  async put (key: Key, val: Uint8Array, options?: Options): Promise<void> {
+    await Promise.reject(new Error('.put is not implemented'))
+  }
+
+  async get (key: Key, options?: Options): Promise<Uint8Array> {
+    return await Promise.reject(new Error('.get is not implemented'))
+  }
+
+  async has (key: Key, options?: Options): Promise<boolean> {
+    return await Promise.reject(new Error('.has is not implemented'))
+  }
+
+  async delete (key: Key, options?: Options): Promise<void> {
+    await Promise.reject(new Error('.delete is not implemented'))
+  }
+
+  async * putMany (source: AwaitIterable<Pair>, options: Options = {}): AsyncIterable<Pair> {
+    for await (const { key, value } of source) {
+      await this.put(key, value, options)
+      yield { key, value }
+    }
+  }
+
+  async * getMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Uint8Array> {
+    for await (const key of source) {
+      yield this.get(key, options)
+    }
+  }
+
+  async * deleteMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Key> {
+    for await (const key of source) {
+      await this.delete(key, options)
+      yield key
+    }
+  }
+
+  batch (): Batch {
+    let puts: Pair[] = []
+    let dels: Key[] = []
+
+    return {
+      put (key, value) {
+        puts.push({ key, value })
+      },
+
+      delete (key) {
+        dels.push(key)
+      },
+      commit: async (options) => {
+        await drain(this.putMany(puts, options))
+        puts = []
+        await drain(this.deleteMany(dels, options))
+        dels = []
+      }
+    }
+  }
+
+  /**
+   * Extending classes should override `query` or implement this method
+   */
+  // eslint-disable-next-line require-yield
+  async * _all (q: Query, options?: Options): AsyncIterable<Pair> {
+    throw new Error('._all is not implemented')
+  }
+
+  /**
+   * Extending classes should override `queryKeys` or implement this method
+   */
+  // eslint-disable-next-line require-yield
+  async * _allKeys (q: KeyQuery, options?: Options): AsyncIterable<Key> {
+    throw new Error('._allKeys is not implemented')
+  }
+
+  query (q: Query, options?: Options): AsyncIterable<Pair> {
+    let it = this._all(q, options)
+
+    if (q.prefix != null) {
+      const prefix = q.prefix
+      it = filter(it, (e) => e.key.toString().startsWith(prefix))
+    }
+
+    if (Array.isArray(q.filters)) {
+      it = q.filters.reduce((it, f) => filter(it, f), it)
+    }
+
+    if (Array.isArray(q.orders)) {
+      it = q.orders.reduce((it, f) => sort(it, f), it)
+    }
+
+    if (q.offset != null) {
+      let i = 0
+      const offset = q.offset
+      it = filter(it, () => i++ >= offset)
+    }
+
+    if (q.limit != null) {
+      it = take(it, q.limit)
+    }
+
+    return it
+  }
+
+  queryKeys (q: KeyQuery, options?: Options): AsyncIterable<Key> {
+    let it = this._allKeys(q, options)
+
+    if (q.prefix != null) {
+      const prefix = q.prefix
+      it = filter(it, (key) =>
+        key.toString().startsWith(prefix)
+      )
+    }
+
+    if (Array.isArray(q.filters)) {
+      it = q.filters.reduce((it, f) => filter(it, f), it)
+    }
+
+    if (Array.isArray(q.orders)) {
+      it = q.orders.reduce((it, f) => sort(it, f), it)
+    }
+
+    if (q.offset != null) {
+      const offset = q.offset
+      let i = 0
+      it = filter(it, () => i++ >= offset)
+    }
+
+    if (q.limit != null) {
+      it = take(it, q.limit)
+    }
+
+    return it
+  }
+}

src/base.ts

@@ -0,0 +1,26 @@
+import errCode from 'err-code'
+
+export function dbOpenFailedError (err?: Error): Error {
+  err = err ?? new Error('Cannot open database')
+  return errCode(err, 'ERR_DB_OPEN_FAILED')
+}
+
+export function dbDeleteFailedError (err?: Error): Error {
+  err = err ?? new Error('Delete failed')
+  return errCode(err, 'ERR_DB_DELETE_FAILED')
+}
+
+export function dbWriteFailedError (err?: Error): Error {
+  err = err ?? new Error('Write failed')
+  return errCode(err, 'ERR_DB_WRITE_FAILED')
+}
+
+export function notFoundError (err?: Error): Error {
+  err = err ?? new Error('Not Found')
+  return errCode(err, 'ERR_NOT_FOUND')
+}
+
+export function abortedError (err?: Error): Error {
+  err = err ?? new Error('Aborted')
+  return errCode(err, 'ERR_ABORTED')
+}

src/errors.js

@@ -1,3 +1,5 @@
+import type { Key } from 'interface-datastore'
+
 import * as Errors from './errors.js'
 import * as shard from './shard.js'
 
@@ -12,7 +14,15 @@ export { NamespaceDatastore } from './namespace.js'
 export { Errors }
 export { shard }
 
-/**
- * @typedef {import("./types").Shard } Shard
- * @typedef {import("./types").KeyTransform } KeyTransform
- */
+export interface Shard {
+  name: string
+  param: number
+  readonly _padding: string
+  fun: (s: string) => string
+  toString: () => string
+}
+
+export interface KeyTransform {
+  convert: (key: Key) => Key
+  invert: (key: Key) => Key
+}

src/errors.ts

@@ -1,85 +1,43 @@
 import { BaseDatastore } from './base.js'
 import map from 'it-map'
 import { pipe } from 'it-pipe'
-
-/**
- * @typedef {import('interface-datastore').Datastore} Datastore
- * @typedef {import('interface-datastore').Options} Options
- * @typedef {import('interface-datastore').Batch} Batch
- * @typedef {import('interface-datastore').Query} Query
- * @typedef {import('interface-datastore').KeyQuery} KeyQuery
- * @typedef {import('interface-datastore').Key} Key
- * @typedef {import('interface-datastore').Pair} Pair
- * @typedef {import('./types').KeyTransform} KeyTransform
- */
-
-/**
- * @template TEntry
- * @typedef {import('interface-store').AwaitIterable<TEntry>} AwaitIterable
- */
+import type { KeyTransform } from './index.js'
+import type { Batch, Datastore, Key, KeyQuery, Options, Pair, Query } from 'interface-datastore'
+import type { AwaitIterable } from 'interface-store'
 
 /**
  * A datastore shim, that wraps around a given datastore, changing
  * the way keys look to the user, for example namespacing
  * keys, reversing them, etc.
- *
- * @implements {Datastore}
  */
 export class KeyTransformDatastore extends BaseDatastore {
-  /**
-   * @param {Datastore} child
-   * @param {KeyTransform} transform
-   */
-  constructor (child, transform) {
+  private readonly child: Datastore
+  public transform: KeyTransform
+
+  constructor (child: Datastore, transform: KeyTransform) {
     super()
 
     this.child = child
     this.transform = transform
   }
 
-  open () {
-    return this.child.open()
-  }
-
-  /**
-   * @param {Key} key
-   * @param {Uint8Array} val
-   * @param {Options} [options]
-   */
-  put (key, val, options) {
-    return this.child.put(this.transform.convert(key), val, options)
+  async put (key: Key, val: Uint8Array, options?: Options): Promise<void> {
+    await this.child.put(this.transform.convert(key), val, options)
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  get (key, options) {
-    return this.child.get(this.transform.convert(key), options)
+  async get (key: Key, options?: Options): Promise<Uint8Array> {
+    return await this.child.get(this.transform.convert(key), options)
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  has (key, options) {
-    return this.child.has(this.transform.convert(key), options)
+  async has (key: Key, options?: Options): Promise<boolean> {
+    return await this.child.has(this.transform.convert(key), options)
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  delete (key, options) {
-    return this.child.delete(this.transform.convert(key), options)
+  async delete (key: Key, options?: Options): Promise<void> {
+    await this.child.delete(this.transform.convert(key), options)
   }
 
-  /**
-   * @param {AwaitIterable<Pair>} source
-   * @param {Options} [options]
-   * @returns {AsyncIterable<Pair>}
-   */
-  async * putMany (source, options = {}) {
+  async * putMany (source: AwaitIterable<Pair>, options: Options = {}): AsyncIterable<Pair> {
     const transform = this.transform
     const child = this.child
 
@@ -103,12 +61,7 @@ export class KeyTransformDatastore extends BaseDatastore {
     )
   }
 
-  /**
-   * @param {AwaitIterable<Key>} source
-   * @param {Options} [options]
-   * @returns {AsyncIterable<Uint8Array>}
-   */
-  async * getMany (source, options = {}) {
+  async * getMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Uint8Array> {
     const transform = this.transform
     const child = this.child
 
@@ -123,12 +76,7 @@ export class KeyTransformDatastore extends BaseDatastore {
     )
   }
 
-  /**
-   * @param {AwaitIterable<Key>} source
-   * @param {Options} [options]
-   * @returns {AsyncIterable<Key>}
-   */
-  async * deleteMany (source, options = {}) {
+  async * deleteMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Key> {
     const transform = this.transform
     const child = this.child
 
@@ -146,10 +94,7 @@ export class KeyTransformDatastore extends BaseDatastore {
     )
   }
 
-  /**
-   * @returns {Batch}
-   */
-  batch () {
+  batch (): Batch {
     const b = this.child.batch()
     return {
       put: (key, value) => {
@@ -158,23 +103,18 @@ export class KeyTransformDatastore extends BaseDatastore {
       delete: (key) => {
         b.delete(this.transform.convert(key))
       },
-      commit: (options) => {
-        return b.commit(options)
+      commit: async (options) => {
+        await b.commit(options)
       }
     }
   }
 
-  /**
-   * @param {Query} q
-   * @param {Options} [options]
-   */
-  query (q, options) {
-    /** @type {Query} */
-    const query = {
+  query (q: Query, options?: Options): AsyncIterable<Pair> {
+    const query: Query = {
       ...q
     }
 
-    query.filters = (query.filters || []).map(filter => {
+    query.filters = (query.filters ?? []).map(filter => {
       return ({ key, value }) => filter({ key: this.transform.convert(key), value })
     })
 
@@ -186,7 +126,7 @@ export class KeyTransformDatastore extends BaseDatastore {
       })
     }
 
-    if (query.orders) {
+    if (query.orders != null) {
       query.orders = query.orders.map(order => {
         return (a, b) => order(
           { key: this.transform.invert(a.key), value: a.value },
@@ -203,16 +143,12 @@ export class KeyTransformDatastore extends BaseDatastore {
     })
   }
 
-  /**
-   * @param {KeyQuery} q
-   * @param {Options} [options]
-   */
-  queryKeys (q, options) {
+  queryKeys (q: KeyQuery, options?: Options): AsyncIterable<Key> {
     const query = {
       ...q
     }
 
-    query.filters = (query.filters || []).map(filter => {
+    query.filters = (query.filters ?? []).map(filter => {
       return (key) => filter(this.transform.convert(key))
     })
 
@@ -224,7 +160,7 @@ export class KeyTransformDatastore extends BaseDatastore {
       })
     }
 
-    if (query.orders) {
+    if (query.orders != null) {
       query.orders = query.orders.map(order => {
         return (a, b) => order(
           this.transform.invert(a),
@@ -237,8 +173,4 @@ export class KeyTransformDatastore extends BaseDatastore {
       return this.transform.invert(key)
     })
   }
-
-  close () {
-    return this.child.close()
-  }
 }

src/index.js renamed to src/index.ts

@@ -0,0 +1,48 @@
+import { BaseDatastore } from './base.js'
+import { Key } from 'interface-datastore/key'
+import * as Errors from './errors.js'
+import type { Pair } from 'interface-datastore'
+
+export class MemoryDatastore extends BaseDatastore {
+  private readonly data: Map<string, Uint8Array>
+
+  constructor () {
+    super()
+
+    this.data = new Map()
+  }
+
+  async put (key: Key, val: Uint8Array): Promise<void> { // eslint-disable-line require-await
+    this.data.set(key.toString(), val)
+  }
+
+  async get (key: Key): Promise<Uint8Array> {
+    const result = this.data.get(key.toString())
+
+    if (result == null) {
+      throw Errors.notFoundError()
+    }
+
+    return result
+  }
+
+  async has (key: Key): Promise<boolean> { // eslint-disable-line require-await
+    return this.data.has(key.toString())
+  }
+
+  async delete (key: Key): Promise<void> { // eslint-disable-line require-await
+    this.data.delete(key.toString())
+  }
+
+  async * _all (): AsyncIterable<Pair> {
+    for (const [key, value] of this.data.entries()) {
+      yield { key: new Key(key), value }
+    }
+  }
+
+  async * _allKeys (): AsyncIterable<Key> {
+    for (const key of this.data.keys()) {
+      yield new Key(key)
+    }
+  }
+}

src/keytransform.js renamed to src/keytransform.ts

@@ -3,49 +3,27 @@ import take from 'it-take'
 import merge from 'it-merge'
 import { BaseDatastore } from './base.js'
 import * as Errors from './errors.js'
-import {
-  sortAll
-} from './utils.js'
-
-/**
- * @typedef {import('interface-datastore').Datastore} Datastore
- * @typedef {import('interface-datastore').Key} Key
- * @typedef {import('interface-datastore').Pair} Pair
- * @typedef {import('interface-datastore').Options} Options
- * @typedef {import('interface-datastore').Batch} Batch
- * @typedef {import('interface-datastore').Query} Query
- * @typedef {import('interface-datastore').KeyQuery} KeyQuery
- * @typedef {import('./types').KeyTransform} KeyTransform
- */
+import sort from 'it-sort'
+import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore'
+import type { Options } from 'interface-store'
 
 /**
  * A datastore that can combine multiple stores inside various
  * key prefixes
- *
- * @implements {Datastore}
  */
 export class MountDatastore extends BaseDatastore {
-  /**
-   * @param {Array<{prefix: Key, datastore: Datastore}>} mounts
-   */
-  constructor (mounts) {
+  private readonly mounts: Array<{ prefix: Key, datastore: Datastore }>
+
+  constructor (mounts: Array<{ prefix: Key, datastore: Datastore }>) {
     super()
 
     this.mounts = mounts.slice()
   }
 
-  async open () {
-    await Promise.all(this.mounts.map((m) => m.datastore.open()))
-  }
-
   /**
    * Lookup the matching datastore for the given key
-   *
-   * @private
-   * @param {Key} key
-   * @returns {{datastore: Datastore, mountpoint: Key} | undefined}
    */
-  _lookup (key) {
+  private _lookup (key: Key): { datastore: Datastore, mountpoint: Key } | undefined {
     for (const mount of this.mounts) {
       if (mount.prefix.toString() === key.toString() || mount.prefix.isAncestorOf(key)) {
         return {
@@ -56,73 +34,48 @@ export class MountDatastore extends BaseDatastore {
     }
   }
 
-  /**
-   * @param {Key} key
-   * @param {Uint8Array} value
-   * @param {Options} [options]
-   */
-  put (key, value, options) {
+  async put (key: Key, value: Uint8Array, options?: Options): Promise<void> {
     const match = this._lookup(key)
     if (match == null) {
       throw Errors.dbWriteFailedError(new Error('No datastore mounted for this key'))
     }
 
-    return match.datastore.put(key, value, options)
+    await match.datastore.put(key, value, options)
   }
 
   /**
    * @param {Key} key
    * @param {Options} [options]
    */
-  get (key, options) {
+  async get (key: Key, options: Options = {}): Promise<Uint8Array> {
     const match = this._lookup(key)
     if (match == null) {
       throw Errors.notFoundError(new Error('No datastore mounted for this key'))
     }
-    return match.datastore.get(key, options)
+    return await match.datastore.get(key, options)
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  has (key, options) {
+  async has (key: Key, options?: Options): Promise<boolean> {
     const match = this._lookup(key)
     if (match == null) {
-      return Promise.resolve(false)
+      return await Promise.resolve(false)
     }
-    return match.datastore.has(key, options)
+    return await match.datastore.has(key, options)
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  delete (key, options) {
+  async delete (key: Key, options?: Options): Promise<void> {
     const match = this._lookup(key)
     if (match == null) {
       throw Errors.dbDeleteFailedError(new Error('No datastore mounted for this key'))
     }
 
-    return match.datastore.delete(key, options)
+    await match.datastore.delete(key, options)
   }
 
-  async close () {
-    await Promise.all(this.mounts.map((m) => {
-      return m.datastore.close()
-    }))
-  }
+  batch (): Batch {
+    const batchMounts: Record<string, Batch> = {}
 
-  /**
-   * @returns {Batch}
-   */
-  batch () {
-    /** @type {Record<string, Batch>} */
-    const batchMounts = {}
-    /**
-     * @param {Key} key
-     */
-    const lookup = (key) => {
+    const lookup = (key: Key): { batch: Batch } => {
       const match = this._lookup(key)
       if (match == null) {
         throw new Error('No datastore mounted for this key')
@@ -148,41 +101,33 @@ export class MountDatastore extends BaseDatastore {
         match.batch.delete(key)
       },
       commit: async (options) => {
-        await Promise.all(Object.keys(batchMounts).map(p => batchMounts[p].commit(options)))
+        await Promise.all(Object.keys(batchMounts).map(async p => { await batchMounts[p].commit(options) }))
       }
     }
   }
 
-  /**
-   * @param {Query} q
-   * @param {Options} [options]
-   */
-  query (q, options) {
+  query (q: Query, options?: Options): AsyncIterable<Pair> {
     const qs = this.mounts.map(m => {
       return m.datastore.query({
         prefix: q.prefix,
         filters: q.filters
       }, options)
     })
 
-    /** @type AsyncIterable<Pair> */
     let it = merge(...qs)
-    if (q.filters) q.filters.forEach(f => { it = filter(it, f) })
-    if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) })
+    if (q.filters != null) q.filters.forEach(f => { it = filter(it, f) })
+    if (q.orders != null) q.orders.forEach(o => { it = sort(it, o) })
     if (q.offset != null) {
       let i = 0
-      it = filter(it, () => i++ >= /** @type {number} */ (q.offset))
+      const offset = q.offset
+      it = filter(it, () => i++ >= offset)
     }
     if (q.limit != null) it = take(it, q.limit)
 
     return it
   }
 
-  /**
-   * @param {KeyQuery} q
-   * @param {Options} [options]
-   */
-  queryKeys (q, options) {
+  queryKeys (q: KeyQuery, options?: Options): AsyncIterable<Key> {
     const qs = this.mounts.map(m => {
       return m.datastore.queryKeys({
         prefix: q.prefix,
@@ -192,11 +137,12 @@ export class MountDatastore extends BaseDatastore {
 
     /** @type AsyncIterable<Key> */
     let it = merge(...qs)
-    if (q.filters) q.filters.forEach(f => { it = filter(it, f) })
-    if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) })
+    if (q.filters != null) q.filters.forEach(f => { it = filter(it, f) })
+    if (q.orders != null) q.orders.forEach(o => { it = sort(it, o) })
     if (q.offset != null) {
       let i = 0
-      it = filter(it, () => i++ >= /** @type {number} */ (q.offset))
+      const offset = q.offset
+      it = filter(it, () => i++ >= offset)
     }
     if (q.limit != null) it = take(it, q.limit)
 

src/memory.js

@@ -1,13 +1,6 @@
 import { Key } from 'interface-datastore'
+import type { Datastore } from 'interface-datastore'
 import { KeyTransformDatastore } from './keytransform.js'
-/**
- * @typedef {import('interface-datastore').Datastore} Datastore
- * @typedef {import('interface-datastore').Query} Query
- * @typedef {import('interface-datastore').KeyQuery} KeyQuery
- * @typedef {import('interface-datastore').Options} Options
- * @typedef {import('interface-datastore').Batch} Batch
- * @typedef {import('./types').KeyTransform} KeyTransform
- */
 
 /**
  * Wraps a given datastore into a keytransform which
@@ -18,11 +11,7 @@ import { KeyTransformDatastore } from './keytransform.js'
  * `/hello/world`.
  */
 export class NamespaceDatastore extends KeyTransformDatastore {
-  /**
-   * @param {Datastore} child
-   * @param {Key} prefix
-   */
-  constructor (child, prefix) {
+  constructor (child: Datastore, prefix: Key) {
     super(child, {
       convert (key) {
         return prefix.child(key)

src/memory.ts

@@ -1,108 +1,74 @@
+import type { Datastore } from 'interface-datastore'
 import { Key } from 'interface-datastore/key'
-// @ts-expect-error readme is unused
-// eslint-disable-next-line no-unused-vars
-import readme from './shard-readme.js'
-
-/**
- * @typedef {import('interface-datastore').Datastore} Datastore
- * @typedef {import('./types').Shard} Shard
- */
+import type { Shard } from './index.js'
 
 export const PREFIX = '/repo/flatfs/shard/'
 export const SHARDING_FN = 'SHARDING'
-export const README_FN = '_README'
 
-/**
- * @implements {Shard}
- */
-export class ShardBase {
-  /**
-   * @param {any} param
-   */
-  constructor (param) {
+export class ShardBase implements Shard {
+  public param: number
+  public name: string
+  public _padding: string
+
+  constructor (param: number) {
     this.param = param
     this.name = 'base'
     this._padding = ''
   }
 
-  /**
-   * @param {string} s
-   */
-  fun (s) {
+  fun (s: string): string {
     return 'implement me'
   }
 
-  toString () {
+  toString (): string {
     return `${PREFIX}v1/${this.name}/${this.param}`
   }
 }
-/**
- * @implements {Shard}
- */
+
 export class Prefix extends ShardBase {
-  /**
-   * @param {number} prefixLen
-   */
-  constructor (prefixLen) {
+  constructor (prefixLen: number) {
     super(prefixLen)
     this._padding = ''.padStart(prefixLen, '_')
     this.name = 'prefix'
   }
 
-  /**
-   * @param {string} noslash
-   */
-  fun (noslash) {
+  fun (noslash: string): string {
     return (noslash + this._padding).slice(0, this.param)
   }
 }
 
 export class Suffix extends ShardBase {
-  /**
-   * @param {number} suffixLen
-   */
-  constructor (suffixLen) {
+  constructor (suffixLen: number) {
     super(suffixLen)
+
     this._padding = ''.padStart(suffixLen, '_')
     this.name = 'suffix'
   }
 
-  /**
-   * @param {string} noslash
-   */
-  fun (noslash) {
+  fun (noslash: string): string {
     const s = this._padding + noslash
     return s.slice(s.length - this.param)
   }
 }
 
 export class NextToLast extends ShardBase {
-  /**
-   * @param {number} suffixLen
-   */
-  constructor (suffixLen) {
+  constructor (suffixLen: number) {
     super(suffixLen)
     this._padding = ''.padStart(suffixLen + 1, '_')
     this.name = 'next-to-last'
   }
 
-  /**
-   * @param {string} noslash
-   */
-  fun (noslash) {
+  fun (noslash: string): string {
     const s = this._padding + noslash
     const offset = s.length - this.param - 1
     return s.slice(offset, offset + this.param)
   }
 }
 
 /**
- * Convert a given string to the matching sharding function.
- *
- * @param {string} str
- * @returns {Shard}
+ * Convert a given string to the matching sharding function
  */
-export function parseShardFun (str) {
+export function parseShardFun (str: string): Shard {
   str = str.trim()
 
   if (str.length === 0) {
@@ -122,7 +88,7 @@ export function parseShardFun (str) {
 
   const name = parts[1]
 
-  if (!parts[2]) {
+  if (parts[2] == null || parts[2] === '') {
     throw new Error('missing param')
   }
 
@@ -140,16 +106,10 @@ export function parseShardFun (str) {
   }
 }
 
-/**
- * @param {string | Uint8Array} path
- * @param {Datastore} store
- */
-export const readShardFun = async (path, store) => {
+export const readShardFun = async (path: string | Uint8Array, store: Datastore): Promise<Shard> => {
   const key = new Key(path).child(new Key(SHARDING_FN))
-  // @ts-ignore
+  // @ts-expect-error
   const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store)
   const res = await get(key)
-  return parseShardFun(new TextDecoder().decode(res || '').trim())
+  return parseShardFun(new TextDecoder().decode(res ?? '').trim())
 }
-
-export { default as readme } from './shard-readme.js'

src/mount.js renamed to src/mount.ts

@@ -0,0 +1,139 @@
+import { Batch, Key, KeyQuery, KeyQueryFilter, Options, Pair, Query, QueryFilter } from 'interface-datastore'
+import {
+  readShardFun,
+  SHARDING_FN
+} from './shard.js'
+import { BaseDatastore } from './base.js'
+import { KeyTransformDatastore } from './keytransform.js'
+import * as Errors from './errors.js'
+import type { Shard } from './index.js'
+import type { Datastore } from 'interface-datastore'
+import type { AwaitIterable } from 'interface-store'
+
+const shardKey = new Key(SHARDING_FN)
+
+/**
+ * Backend independent abstraction of go-ds-flatfs.
+ *
+ * Wraps another datastore such that all values are stored
+ * sharded according to the given sharding function.
+ */
+export class ShardingDatastore extends BaseDatastore {
+  private readonly child: KeyTransformDatastore
+  private shard: Shard
+
+  constructor (store: Datastore, shard: Shard) {
+    super()
+
+    this.child = new KeyTransformDatastore(store, {
+      convert: this._convertKey.bind(this),
+      invert: this._invertKey.bind(this)
+    })
+    this.shard = shard
+  }
+
+  async open (): Promise<void> {
+    this.shard = await ShardingDatastore.create(this.child, this.shard)
+  }
+
+  _convertKey (key: Key): Key {
+    const s = key.toString()
+    if (s === shardKey.toString()) {
+      return key
+    }
+
+    const parent = new Key(this.shard.fun(s))
+    return parent.child(key)
+  }
+
+  _invertKey (key: Key): Key {
+    const s = key.toString()
+    if (s === shardKey.toString()) {
+      return key
+    }
+    return Key.withNamespaces(key.list().slice(1))
+  }
+
+  static async create (store: Datastore, shard: Shard): Promise<Shard> {
+    const hasShard = await store.has(shardKey)
+    if (!hasShard && shard == null) {
+      throw Errors.dbOpenFailedError(Error('Shard is required when datastore doesn\'t have a shard key already.'))
+    }
+    if (!hasShard) {
+      // @ts-expect-error i have no idea what putRaw is or saw any implementation
+      const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store)
+      await Promise.all([
+        put(shardKey, new TextEncoder().encode(shard.toString() + '\n'))
+      ])
+
+      return shard
+    }
+
+    // test shards
+    const diskShard = await readShardFun('/', store)
+    const a = diskShard.toString()
+    const b = shard.toString()
+    if (a !== b) {
+      throw new Error(`specified fun ${b} does not match repo shard fun ${a}`)
+    }
+    return diskShard
+  }
+
+  async put (key: Key, val: Uint8Array, options?: Options): Promise<void> {
+    await this.child.put(key, val, options)
+  }
+
+  async get (key: Key, options?: Options): Promise<Uint8Array> {
+    return await this.child.get(key, options)
+  }
+
+  async has (key: Key, options?: Options): Promise<boolean> {
+    return await this.child.has(key, options)
+  }
+
+  async delete (key: Key, options?: Options): Promise<void> {
+    await this.child.delete(key, options)
+  }
+
+  async * putMany (source: AwaitIterable<Pair>, options: Options = {}): AsyncIterable<Pair> {
+    yield * this.child.putMany(source, options)
+  }
+
+  async * getMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Uint8Array> {
+    yield * this.child.getMany(source, options)
+  }
+
+  async * deleteMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Key> {
+    yield * this.child.deleteMany(source, options)
+  }
+
+  batch (): Batch {
+    return this.child.batch()
+  }
+
+  query (q: Query, options?: Options): AsyncIterable<Pair> {
+    const omitShard: QueryFilter = ({ key }) => key.toString() !== shardKey.toString()
+
+    const tq: Query = {
+      ...q,
+      filters: [
+        omitShard
+      ].concat(q.filters ?? [])
+    }
+
+    return this.child.query(tq, options)
+  }
+
+  queryKeys (q: KeyQuery, options?: Options): AsyncIterable<Key> {
+    const omitShard: KeyQueryFilter = (key) => key.toString() !== shardKey.toString()
+
+    const tq: KeyQuery = {
+      ...q,
+      filters: [
+        omitShard
+      ].concat(q.filters ?? [])
+    }
+
+    return this.child.queryKeys(tq, options)
+  }
+}

src/namespace.js renamed to src/namespace.ts

@@ -3,24 +3,11 @@ import * as Errors from './errors.js'
 import { logger } from '@libp2p/logger'
 import { pushable } from 'it-pushable'
 import drain from 'it-drain'
+import type { Batch, Datastore, Key, KeyQuery, Options, Pair, Query } from 'interface-datastore'
+import type { AwaitIterable } from 'interface-store'
 
 const log = logger('datastore:core:tiered')
 
-/**
- * @typedef {import('interface-datastore').Datastore} Datastore
- * @typedef {import('interface-datastore').Options} Options
- * @typedef {import('interface-datastore').Batch} Batch
- * @typedef {import('interface-datastore').Query} Query
- * @typedef {import('interface-datastore').KeyQuery} KeyQuery
- * @typedef {import('interface-datastore').Key} Key
- * @typedef {import('interface-datastore').Pair} Pair
- */
-
-/**
- * @template TEntry
- * @typedef {import('interface-store').AwaitIterable<TEntry>} AwaitIterable
- */
-
 /**
  * A datastore that can combine multiple stores. Puts and deletes
  * will write through to all datastores. Has and get will
@@ -29,57 +16,35 @@ const log = logger('datastore:core:tiered')
  *
  */
 export class TieredDatastore extends BaseDatastore {
-  /**
-   * @param {Datastore[]} stores
-   */
-  constructor (stores) {
+  private readonly stores: Datastore[]
+
+  constructor (stores: Datastore[]) {
     super()
 
     this.stores = stores.slice()
   }
 
-  async open () {
+  async put (key: Key, value: Uint8Array, options?: Options): Promise<void> {
     try {
-      await Promise.all(this.stores.map((store) => store.open()))
-    } catch (/** @type {any} */ err) {
-      throw Errors.dbOpenFailedError(err)
-    }
-  }
-
-  /**
-   * @param {Key} key
-   * @param {Uint8Array} value
-   * @param {Options} [options]
-   */
-  async put (key, value, options) {
-    try {
-      await Promise.all(this.stores.map(store => store.put(key, value, options)))
-    } catch (/** @type {any} */ err) {
+      await Promise.all(this.stores.map(async store => { await store.put(key, value, options) }))
+    } catch (err: any) {
       throw Errors.dbWriteFailedError(err)
     }
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  async get (key, options) {
+  async get (key: Key, options?: Options): Promise<Uint8Array> {
     for (const store of this.stores) {
       try {
         const res = await store.get(key, options)
-        if (res) return res
+        if (res != null) return res
       } catch (err) {
         log.error(err)
       }
     }
     throw Errors.notFoundError()
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  async has (key, options) {
+  async has (key: Key, options?: Options): Promise<boolean> {
     for (const s of this.stores) {
       if (await s.has(key, options)) {
         return true
@@ -89,27 +54,18 @@ export class TieredDatastore extends BaseDatastore {
     return false
   }
 
-  /**
-   * @param {Key} key
-   * @param {Options} [options]
-   */
-  async delete (key, options) {
+  async delete (key: Key, options?: Options): Promise<void> {
     try {
-      await Promise.all(this.stores.map(store => store.delete(key, options)))
-    } catch (/** @type {any} */ err) {
+      await Promise.all(this.stores.map(async store => { await store.delete(key, options) }))
+    } catch (err: any) {
       throw Errors.dbDeleteFailedError(err)
     }
   }
 
-  /**
-   * @param {AwaitIterable<Pair>} source
-   * @param {Options} [options]
-   * @returns {AsyncIterable<Pair>}
-   */
-  async * putMany (source, options = {}) {
-    let error
+  async * putMany (source: AwaitIterable<Pair>, options: Options = {}): AsyncIterable<Pair> {
+    let error: Error | undefined
     const pushables = this.stores.map(store => {
-      const source = pushable({
+      const source = pushable<Pair>({
         objectMode: true
       })
 
@@ -124,7 +80,7 @@ export class TieredDatastore extends BaseDatastore {
 
     try {
       for await (const pair of source) {
-        if (error) {
+        if (error != null) {
           throw error
         }
 
@@ -137,15 +93,10 @@ export class TieredDatastore extends BaseDatastore {
     }
   }
 
-  /**
-   * @param {AwaitIterable<Key>} source
-   * @param {Options} [options]
-   * @returns {AsyncIterable<Key>}
-   */
-  async * deleteMany (source, options = {}) {
-    let error
+  async * deleteMany (source: AwaitIterable<Key>, options: Options = {}): AsyncIterable<Key> {
+    let error: Error | undefined
     const pushables = this.stores.map(store => {
-      const source = pushable({
+      const source = pushable<Key>({
         objectMode: true
       })
 
@@ -160,7 +111,7 @@ export class TieredDatastore extends BaseDatastore {
 
     try {
       for await (const key of source) {
-        if (error) {
+        if (error != null) {
           throw error
         }
 
@@ -173,22 +124,15 @@ export class TieredDatastore extends BaseDatastore {
     }
   }
 
-  async close () {
-    await Promise.all(this.stores.map(store => store.close()))
-  }
-
-  /**
-   * @returns {Batch}
-   */
-  batch () {
+  batch (): Batch {
     const batches = this.stores.map(store => store.batch())
 
     return {
       put: (key, value) => {
-        batches.forEach(b => b.put(key, value))
+        batches.forEach(b => { b.put(key, value) })
       },
       delete: (key) => {
-        batches.forEach(b => b.delete(key))
+        batches.forEach(b => { b.delete(key) })
       },
       commit: async (options) => {
         for (const batch of batches) {
@@ -198,19 +142,11 @@ export class TieredDatastore extends BaseDatastore {
     }
   }
 
-  /**
-   * @param {Query} q
-   * @param {Options} [options]
-   */
-  query (q, options) {
+  query (q: Query, options?: Options): AsyncIterable<Pair> {
     return this.stores[this.stores.length - 1].query(q, options)
   }
 
-  /**
-   * @param {KeyQuery} q
-   * @param {Options} [options]
-   */
-  queryKeys (q, options) {
+  queryKeys (q: KeyQuery, options?: Options): AsyncIterable<Key> {
     return this.stores[this.stores.length - 1].queryKeys(q, options)
   }
 }

src/shard-readme.js

@@ -0,0 +1,5 @@
+
+export const replaceStartWith = (s: string, r: string): string => {
+  const matcher = new RegExp('^' + r)
+  return s.replace(matcher, '')
+}

src/shard.js renamed to src/shard.ts

@@ -10,16 +10,10 @@ describe('KeyTransformDatastore', () => {
   it('basic', async () => {
     const mStore = new MemoryDatastore()
     const transform = {
-      /**
-       * @param {Key} key
-       */
-      convert (key) {
+      convert (key: Key): Key {
         return new Key('/abc').child(key)
       },
-      /**
-       * @param {Key} key
-       */
-      invert (key) {
+      invert (key: Key): Key {
         const l = key.list()
         if (l[0] !== 'abc') {
           throw new Error('missing prefix, convert failed?')
@@ -38,9 +32,9 @@ describe('KeyTransformDatastore', () => {
       'foo/bar/bazb',
       'foo/bar/baz/barb'
     ].map((s) => new Key(s))
-    await Promise.all(keys.map((key) => kStore.put(key, key.uint8Array())))
-    const kResults = Promise.all(keys.map((key) => kStore.get(key)))
-    const mResults = Promise.all(keys.map((key) => mStore.get(new Key('abc').child(key))))
+    await Promise.all(keys.map(async (key) => { await kStore.put(key, key.uint8Array()) }))
+    const kResults = Promise.all(keys.map(async (key) => await kStore.get(key)))
+    const mResults = Promise.all(keys.map(async (key) => await mStore.get(new Key('abc').child(key))))
     const results = await Promise.all([kResults, mResults])
     expect(results[0]).to.eql(results[1])
 
@@ -54,6 +48,5 @@ describe('KeyTransformDatastore', () => {
       expect(transform.invert(kA)).to.eql(kB)
       expect(kA).to.eql(transform.convert(kB))
     })
-    await kStore.close()
   })
 })

src/sharding.js

@@ -9,12 +9,9 @@ import { MountDatastore } from '../src/mount.js'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { interfaceDatastoreTests } from 'interface-datastore-tests'
 import { KeyTransformDatastore } from '../src/keytransform.js'
+import type { Datastore } from 'interface-datastore'
 
-/**
- * @param {import('interface-datastore').Datastore} datastore
- * @param {Key} prefix
- */
-const stripPrefixDatastore = (datastore, prefix) => {
+const stripPrefixDatastore = (datastore: Datastore, prefix: Key): Datastore => {
   return new KeyTransformDatastore(
     datastore, {
       convert: (key) => {

src/sharding.ts

@@ -26,9 +26,9 @@ describe('NamespaceDatastore', () => {
       'foo/bar/baz/barb'
     ].map((s) => new Key(s))
 
-    await Promise.all(keys.map(key => store.put(key, uint8ArrayFromString(key.toString()))))
-    const nResults = Promise.all(keys.map((key) => store.get(key)))
-    const mResults = Promise.all(keys.map((key) => mStore.get(new Key(prefix).child(key))))
+    await Promise.all(keys.map(async key => { await store.put(key, uint8ArrayFromString(key.toString())) }))
+    const nResults = Promise.all(keys.map(async (key) => await store.get(key)))
+    const mResults = Promise.all(keys.map(async (key) => await mStore.get(new Key(prefix).child(key))))
     const results = await Promise.all([nResults, mResults])
     const mRes = await all(mStore.query({}))
     const nRes = await all(store.query({}))
@@ -41,7 +41,6 @@ describe('NamespaceDatastore', () => {
       expect(store.transform.invert(kA)).to.eql(kB)
       expect(kA).to.eql(store.transform.convert(kB))
     })
-    await store.close()
 
     expect(results[0]).to.eql(results[1])
   }))

src/tiered.js renamed to src/tiered.ts

@@ -7,9 +7,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import {
   NextToLast,
-  SHARDING_FN,
-  README_FN,
-  readme
+  SHARDING_FN
 } from '../src/shard.js'
 import {
   ShardingDatastore
@@ -23,11 +21,9 @@ describe('ShardingDatastore', () => {
     const store = new ShardingDatastore(ms, shard)
     await store.open()
     const res = await Promise.all([
-      ms.get(new Key(SHARDING_FN)),
-      ms.get(new Key(README_FN))
+      store.get(new Key(SHARDING_FN))
     ])
     expect(uint8ArrayToString(res[0])).to.eql(shard.toString() + '\n')
-    expect(uint8ArrayToString(res[1])).to.eql(readme)
   })
 
   it('open - empty', () => {

src/types.ts

@@ -6,17 +6,16 @@ import { MemoryDatastore } from '../src/memory.js'
 import { TieredDatastore } from '../src/tiered.js'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { interfaceDatastoreTests } from 'interface-datastore-tests'
+import type { Datastore } from 'interface-datastore'
 
 /**
  * @typedef {import('interface-datastore').Datastore} Datastore
  */
 
 describe('Tiered', () => {
   describe('all stores', () => {
-    /** @type {Datastore[]} */
-    const ms = []
-    /** @type {TieredDatastore} */
-    let store
+    const ms: Datastore[] = []
+    let store: TieredDatastore
     beforeEach(() => {
       ms.push(new MemoryDatastore())
       ms.push(new MemoryDatastore())

src/utils.js

@@ -0,0 +1,20 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import * as utils from '../src/utils.js'
+
+describe('utils', () => {
+  it('replaceStartWith', () => {
+    expect(
+      utils.replaceStartWith('helloworld', 'hello')
+    ).to.eql(
+      'world'
+    )
+
+    expect(
+      utils.replaceStartWith('helloworld', 'world')
+    ).to.eql(
+      'helloworld'
+    )
+  })
+})

src/utils.ts

@@ -1,11 +1,10 @@
 {
-    "extends": "aegir/src/config/tsconfig.aegir.json",
-    "compilerOptions": {
-      "outDir": "dist",
-      "emitDeclarationOnly": true
-    },
-    "include": [
-      "test",
-      "src"
-    ]
+  "extends": "aegir/src/config/tsconfig.aegir.json",
+  "compilerOptions": {
+    "outDir": "dist"
+  },
+  "include": [
+    "src",
+    "test"
+  ]
 }