Add HTTP over libp2p and http blockstore #6
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| import type { Blockstore, Pair } from 'interface-blockstore' | ||
| import type { Connection } from '@libp2p/interface-connection' | ||
| import type { | ||
| AbortOptions, | ||
| Await, | ||
| AwaitIterable | ||
| } from 'interface-store' | ||
| import type { Libp2p } from 'libp2p' | ||
| import type { CID } from 'multiformats/cid' | ||
| import { fetchViaDuplex } from '@marcopolo_/libp2p-fetch' | ||
|
|
||
| // TODOS: | ||
| // - Validate the blocks we fetch | ||
|
Contributor
There was a problem hiding this comment. We need to do this and it should be fairly straightforward to do (just grab the multihash, lookup an implementation of the hash function, run it over the bytes and see if they match) |
||
| // - Use format=car to get the whole dag at once rather than per block (probably faster?) | ||
|
Contributor
There was a problem hiding this comment. This is almost certainly a separate PR once we've got the rest of this done. |
||
| // - Keep a list of HTTPS endpoints, not just libp2p peers. | ||
|
Contributor
There was a problem hiding this comment. I'd say we should (if we have time) just make up some record data and chuck it into IPNI for our records where the gateways are non-recursive (i.e. only based on data we have locally similar to https://github.com/ipfs/kubo/blob/c58aadb887a5ca88256ac8f3a8fbf60b0263c978/docs/config.md#gatewaynofetch). Happy to help here, it shouldn't be too bad as I've done some hacking here before. Basically we'd use this format ipni/specs#6 to avoid dealing with the global code table (and associated bikeshedding) as much as possible. It's a small amount of parsing in the code here, and then we can hook up some hacky IPNI record publishing. For recursive gateways (e.g. dweb.link, ipfs.io, cf-ipfs.com) we could either hard code them, get them from https://ipfs.github.io/public-gateway-checker/ or have an IPNI rendezvous record. If there's no time, then hard coding it is 😄 |
||
| // - Keep a list of previous peers we were connected to, so even if we lose a webtransport connection, we can spin it up later. | ||
|
|
||
| export class HttpBlockstore implements Blockstore { | ||
|
Contributor
There was a problem hiding this comment. IIUC this approach will cause problems when also enabling fetching over Bitswap (I think it might check the blockstore before doing a Bitswap request which would basically cause these lookups to be sequential). An alternative way to do this would be (although @achingbrain might have more helia-friendly suggestions):
Later we can basically do combined task prioritization with Bitswap whereby we hit peers (Bitswap or HTTP or HTTP over libp2p) based on how quickly they respond to us with good data. |
||
| private peersWithHttpOverLibp2p: Connection[] = [] | ||
| constructor (private readonly innerBlockstore: Blockstore, libp2p: Libp2p) { | ||
| libp2p.addEventListener('peer:connect', (connEvent) => { | ||
| // TODO check if they support http over libp2p and the ipfs gateway api | ||
| const conn = connEvent.detail | ||
| this.peersWithHttpOverLibp2p.push(conn) | ||
| }) | ||
| libp2p.addEventListener('peer:disconnect', (connEvent) => { | ||
| const conn = connEvent.detail | ||
| this.peersWithHttpOverLibp2p = this.peersWithHttpOverLibp2p.filter((c) => c !== conn) | ||
| }) | ||
| } | ||
|
|
||
| getAll (): AwaitIterable<Pair> { | ||
| return this.innerBlockstore.getAll() | ||
| } | ||
|
|
||
| async has (key: CID, options?: AbortOptions): Promise<boolean> { | ||
| const innerHas = await this.innerBlockstore.has(key, options) | ||
| if (innerHas) { | ||
| return true | ||
| } | ||
|
|
||
| for (const connection of this.peersWithHttpOverLibp2p) { | ||
| // TODO unsure how to wait for identify to finish, just trying to see if it works | ||
| // const protos = await this.libp2p.peerStore.protoBook.get(connection.remotePeer) | ||
| // if (!protos.includes('/libp2p-http')) { | ||
| // // TODO remove from peersWithHttpOverLibp2p | ||
| // continue | ||
| // } | ||
|
|
||
| try { | ||
| const s = await connection.newStream('/libp2p-http') | ||
| const fetch = fetchViaDuplex(s) | ||
| const resp = await fetch(new Request(`https://example.com/ipfs/${key.toString()}/`, { method: 'HEAD' })) | ||
| if (resp.ok) { | ||
| console.log('HTTP Blockstore has it!', key.toString(), 'via libp2p') | ||
| return true | ||
| } | ||
| } catch (err) { | ||
| console.warn('http over libp2p err', err) | ||
| continue | ||
| } | ||
| } | ||
|
|
||
| return false | ||
| } | ||
|
|
||
| put (key: CID, val: Uint8Array, options?: AbortOptions): Await<CID> { | ||
| return this.innerBlockstore.put(key, val, options) | ||
| } | ||
|
|
||
| putMany (source: AwaitIterable<Pair>, options?: AbortOptions): AwaitIterable<CID> { | ||
| return this.innerBlockstore.putMany(source, options) | ||
| } | ||
|
|
||
| delete (key: CID, options?: AbortOptions): Await<void> { | ||
| return this.innerBlockstore.delete(key, options) | ||
| } | ||
|
|
||
| deleteMany (source: AwaitIterable<CID>, options?: AbortOptions): AwaitIterable<CID> { | ||
| return this.innerBlockstore.deleteMany(source, options) | ||
| } | ||
|
|
||
| async get (key: CID, options?: AbortOptions): Promise<Uint8Array> { | ||
| if (await this.innerBlockstore.has(key, options)) { | ||
| return await this.innerBlockstore.get(key, options) | ||
| } | ||
|
|
||
| for (const connection of this.peersWithHttpOverLibp2p) { | ||
| try { | ||
| const s = await connection.newStream('/libp2p-http') | ||
| const fetch = fetchViaDuplex(s) | ||
| const resp = await fetch(new Request(`https://example.com/ipfs/${key.toString()}/?format=raw`)) | ||
| if (resp.ok) { | ||
| const ab = (await resp.arrayBuffer()) | ||
| await this.innerBlockstore.put(key, new Uint8Array(ab)) | ||
| return new Uint8Array(ab) | ||
| } | ||
| } catch (err) { | ||
| console.warn('http over libp2p err', err) | ||
| continue | ||
| } | ||
| } | ||
| return await this.innerBlockstore.get(key, options) | ||
| } | ||
|
|
||
| getMany (source: AwaitIterable<CID>, options?: AbortOptions): AwaitIterable<Pair> { | ||
| return this.innerBlockstore.getMany(source, options) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
where's the code that runs this server? Some out of curiosity and some out of wanting to be able to debug if there are issues 😅
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
MarcoPolo/go-ipfs@83f40a6