From 35e8a8fd7c1ca68b21320b95211304bf01b30086 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 3 Feb 2023 00:29:59 +0100 Subject: [PATCH 001/253] chore: initial commit --- README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..ffff3cb3 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# @helia/unixfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> A Helia-compatible wrapper for UnixFS + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) From bcb060d880175ab885479388049a1ca2e5873629 Mon Sep 17 00:00:00 2001 From: "ipfs-mgmt-read-write[bot]" <104492829+ipfs-mgmt-read-write[bot]@users.noreply.github.com> Date: Fri, 3 Feb 2023 19:06:58 +0000 Subject: [PATCH 002/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true From d120453b168da21624dc4a9b8d61be56073bc634 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 16:42:47 +0100 Subject: [PATCH 003/253] chore: initial import --- README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..4345fd89 --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +# @helia/ipns + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> An implementation of IPNS for Helia + +## Table of contents + +- - [Structure](#structure) +- [helia ](#helia----omit-in-toc---) + - [Project status](#project-status) + - [Name](#name) + - [Background](#background) + - [Roadmap](#roadmap) + - [API Docs](#api-docs) + - [License](#license) + - [Contribute](#contribute) + +## Structure + +- [`/packages/interop`](./packages/interop) Interop tests for @helia/ipns +- [`/packages/ipns`](./packages/ipns) An implementation of IPNS for Helia + +helia logo + +# helia + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +## Project status + +This project is pre-alpha and is currently in development. An initial v1 release is planned for [late Q1 2023](/ROADMAP.md#late-q1-march). Helia is being built in the open; community contributors are welcome! + +The core of IPFS is the [Files API](https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md), which will likewise be implemented in Helia. These initial building blocks are in development now; have a look at this repo's PR(s). For more info about Helia, please see the [Roadmap](https://github.com/ipfs/helia/issues/5) and the [Manifesto](MANIFESTO.md). + +We are also sharing about the progress so far, and discussing how you can get involved, at [Helia Demo Day](https://lu.ma/helia) every couple weeks. We'd love to see you there! + +## Name + +Helia (*HEE-lee-ah*) is the Latin spelling of Ἡλιη -- in Greek mythology, one of the [Heliades](https://www.wikidata.org/wiki/Q12656412): the daughters of the sun god Helios. When their brother Phaethon died trying to drive the sun chariot across the sky, their tears of mourning fell to earth as amber, which is yellow (sort of), and so is JavaScript. They were then turned into [poplar](https://en.wiktionary.org/wiki/poplar) trees and, well, JavaScript is quite popular. + +In Oct–Dec 2022, IP Stewards [sought](https://github.com/ipfs/pomegranate/issues/3) community input for the name of this project. After considering 20 suggestions and holding a couple of polls, the name **Helia** was chosen. Here's [why it's not named IPFS](https://github.com/ipfs/ipfs/issues/470). + +## Background + +This project aims to build a lean, modular, and modern implementation of IPFS, the Interplanetary File System. + +For more information, see the [State of IPFS in JS (blog post)](https://blog.ipfs.tech/state-of-ipfs-in-js/). + +## Roadmap + +Please find and comment on [the Roadmap here](https://github.com/ipfs/helia/issues/5). + +This IPFS implementation in JavaScript is a work in progress. [Here are some ways you can help](https://blog.ipfs.tech/state-of-ipfs-in-js/#%F0%9F%A4%9D-ways-you-can-help)! + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) From b1761795f023e9150201e41f0b9e2c3021425f26 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 14 Feb 2023 15:57:57 +0100 Subject: [PATCH 004/253] feat: initial implementation (#1) Adds initial implementation of IPNS for helia --- .github/dependabot.yml | 11 + .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 143 +++++++++ .gitignore | 8 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ README.md | 6 + package.json | 50 +++ packages/interop/.aegir.js | 45 +++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 59 ++++ packages/interop/package.json | 198 ++++++++++++ packages/interop/src/index.ts | 1 + packages/interop/test/dht.spec.ts | 181 +++++++++++ packages/interop/test/fixtures/connect.ts | 18 ++ .../test/fixtures/create-helia.browser.ts | 57 ++++ .../interop/test/fixtures/create-helia.ts | 46 +++ packages/interop/test/fixtures/create-kubo.ts | 28 ++ .../interop/test/fixtures/create-peer-ids.ts | 46 +++ packages/interop/test/fixtures/wait-for.ts | 27 ++ packages/interop/test/pubsub.spec.ts | 176 +++++++++++ packages/interop/tsconfig.json | 15 + packages/ipns/LICENSE | 4 + packages/ipns/LICENSE-APACHE | 5 + packages/ipns/LICENSE-MIT | 19 ++ packages/ipns/README.md | 59 ++++ packages/ipns/package.json | 191 +++++++++++ packages/ipns/src/index.ts | 296 ++++++++++++++++++ packages/ipns/src/routing/dht.ts | 85 +++++ packages/ipns/src/routing/index.ts | 26 ++ packages/ipns/src/routing/local-store.ts | 63 ++++ packages/ipns/src/routing/pubsub.ts | 195 ++++++++++++ .../src/utils/resolve-dns-link.browser.ts | 61 ++++ packages/ipns/src/utils/resolve-dns-link.ts | 65 ++++ packages/ipns/src/utils/tlru.ts | 52 +++ packages/ipns/test/publish.spec.ts | 50 +++ packages/ipns/test/resolve.spec.ts | 78 +++++ packages/ipns/tsconfig.json | 10 + 41 files changed, 2438 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 package.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/dht.spec.ts create mode 100644 packages/interop/test/fixtures/connect.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/test/fixtures/create-peer-ids.ts create mode 100644 packages/interop/test/fixtures/wait-for.ts create mode 100644 packages/interop/test/pubsub.spec.ts create mode 100644 packages/interop/tsconfig.json create mode 100644 packages/ipns/LICENSE create mode 100644 packages/ipns/LICENSE-APACHE create mode 100644 packages/ipns/LICENSE-MIT create mode 100644 packages/ipns/README.md create mode 100644 packages/ipns/package.json create mode 100644 packages/ipns/src/index.ts create mode 100644 packages/ipns/src/routing/dht.ts create mode 100644 packages/ipns/src/routing/index.ts create mode 100644 packages/ipns/src/routing/local-store.ts create mode 100644 packages/ipns/src/routing/pubsub.ts create mode 100644 packages/ipns/src/utils/resolve-dns-link.browser.ts create mode 100644 packages/ipns/src/utils/resolve-dns-link.ts create mode 100644 packages/ipns/src/utils/tlru.ts create mode 100644 packages/ipns/test/publish.spec.ts create mode 100644 packages/ipns/test/resolve.spec.ts create mode 100644 packages/ipns/tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..7da2cd26 --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,143 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: firefox-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..910f6339 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md index 4345fd89..08516c56 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +

+ + Helia logo + +

+ # @helia/ipns [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) diff --git a/package.json b/package.json new file mode 100644 index 00000000..0ad2b58c --- /dev/null +++ b/package.json @@ -0,0 +1,50 @@ +{ + "name": "@helia/ipns", + "version": "0.0.0", + "description": "An implementation of IPNS for Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-ipns#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-ipns.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-ipns/issues" + }, + "keywords": [ + "ipfs" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && npm run release:npm && npm run docs", + "release:npm": "aegir exec npm -- publish", + "release:rc": "aegir release-rc" + }, + "dependencies": { + "aegir": "^38.1.0" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..e905e050 --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/ipns-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/ipns + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..aebe1cf1 --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,198 @@ +{ + "name": "@helia/ipns-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/ipns", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-ipns.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-ipns/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./routing": { + "types": "./dist/src/routing/index.d.ts", + "import": "./dist/src/routing/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "devDependencies": { + "@chainsafe/libp2p-gossipsub": "^6.1.0", + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^3.0.5", + "@helia/interface": "next", + "@helia/ipns": "~0.0.0", + "@libp2p/interface-peer-id": "^2.0.1", + "@libp2p/kad-dht": "^7.0.0", + "@libp2p/peer-id": "^2.0.1", + "@libp2p/peer-id-factory": "^2.0.1", + "@libp2p/tcp": "^6.1.2", + "@libp2p/websockets": "^5.0.3", + "aegir": "^38.1.0", + "blockstore-core": "^3.0.0", + "datastore-core": "^8.0.4", + "go-ipfs": "^0.18.1", + "helia": "next", + "ipfsd-ctl": "^13.0.0", + "ipns": "^5.0.1", + "it-all": "^2.0.0", + "it-last": "^2.0.0", + "it-map": "^2.0.0", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.42.2", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1", + "uint8arrays": "^4.0.3", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts new file mode 100644 index 00000000..053424dc --- /dev/null +++ b/packages/interop/test/dht.spec.ts @@ -0,0 +1,181 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import type { IPNS } from '@helia/ipns' +import { ipns } from '@helia/ipns' +import { dht } from '@helia/ipns/routing' +import last from 'it-last' +import { kadDHT } from '@libp2p/kad-dht' +import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { sortClosestPeers } from './fixtures/create-peer-ids.js' +import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' +import { concat as uint8ArrayConcat } from 'uint8arrays/concat' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { waitFor } from './fixtures/wait-for.js' +import { connect } from './fixtures/connect.js' +import { isElectronMain } from 'wherearewe' + +describe('dht routing', () => { + let helia: Helia + let kubo: Controller + let name: IPNS + + // the CID we are going to publish + let value: CID + + // the public key we will use to publish the value + let key: PeerId + + /** + * Ensure that for the CID we are going to publish, the resolver has a peer ID that + * is KAD-closer to the routing key so we can predict the the resolver will receive + * the DHT record containing the IPNS record + */ + async function createNodes (resolver: 'kubo' | 'helia'): Promise { + const input = Uint8Array.from([0, 1, 2, 3, 4]) + const digest = await sha256.digest(input) + value = CID.createV1(raw.code, digest) + + helia = await createHeliaNode({ + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }) + }) + kubo = await createKuboNode() + + // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key + while (true) { + key = await createEd25519PeerId() + const routingKey = uint8ArrayConcat([ + uint8ArrayFromString('/ipns/'), + key.toBytes() + ]) + + const [closest] = await sortClosestPeers(routingKey, [ + helia.libp2p.peerId, + kubo.peer.id + ]) + + if (resolver === 'kubo' && closest.equals(kubo.peer.id)) { + break + } + + if (resolver === 'helia' && closest.equals(helia.libp2p.peerId)) { + break + } + } + + // connect the two nodes over the KAD-DHT protocol, this should ensure + // both nodes have each other in their KAD buckets + await connect(helia, kubo, '/ipfs/lan/kad/1.0.0') + + await waitFor(async () => { + let found = false + + for await (const event of helia.libp2p.dht.findPeer(kubo.peer.id)) { + if (event.name === 'FINAL_PEER') { + found = true + } + } + + return found + }, { + timeout: 30000, + delay: 1000, + message: 'Helia could not find Kubo on the DHT' + }) + + await waitFor(async () => { + let found = false + + for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) { + if (event.name === 'FINAL_PEER') { + found = true + } + } + + return found + }, { + timeout: 30000, + delay: 1000, + message: 'Kubo could not find Helia on the DHT' + }) + + name = ipns(helia, [ + dht(helia) + ]) + } + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should publish on helia and resolve on kubo', async () => { + await createNodes('kubo') + + const keyName = 'my-ipns-key' + await helia.libp2p.keychain.importPeer(keyName, key) + + await name.publish(key, value) + + const resolved = await last(kubo.api.name.resolve(key)) + + if (resolved == null) { + throw new Error('kubo failed to resolve name') + } + + expect(resolved).to.equal(`/ipfs/${value.toString()}`) + }) + + it('should publish on kubo and resolve on helia', async function () { + if (isElectronMain) { + // electron main does not have fetch, FormData or Blob APIs + // can revisit when kubo-rpc-client supports the key.import API + return this.skip() + } + + await createNodes('helia') + + const keyName = 'my-ipns-key' + const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) + + // ensure the key is in the kubo keychain so we can use it to publish the IPNS record + const body = new FormData() + body.append('key', new Blob([key.privateKey ?? new Uint8Array(0)])) + + // can't use the kubo-rpc-api for this call yet + const response = await fetch(`http://${kubo.api.apiHost}:${kubo.api.apiPort}/api/v0/key/import?arg=${keyName}`, { + method: 'POST', + body + }) + + expect(response).to.have.property('status', 200) + + await kubo.api.name.publish(cid, { + key: keyName + }) + + const resolvedCid = await name.resolve(key) + expect(resolvedCid.toString()).to.equal(cid.toString()) + }) +}) diff --git a/packages/interop/test/fixtures/connect.ts b/packages/interop/test/fixtures/connect.ts new file mode 100644 index 00000000..e7685a3b --- /dev/null +++ b/packages/interop/test/fixtures/connect.ts @@ -0,0 +1,18 @@ +import { expect } from 'aegir/chai' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' + +/** + * Connect the two nodes by dialing a protocol stream + */ +export async function connect (helia: Helia, kubo: Controller, protocol: string): Promise { + let connected = false + for (const addr of kubo.peer.addresses) { + try { + await helia.libp2p.dialProtocol(addr, protocol) + connected = true + break + } catch { } + } + expect(connected).to.be.true('could not connect hHlia to Kubo') +} diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..a3684dfc --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,57 @@ +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' +import { kadDHT } from '@libp2p/kad-dht' +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }), + pubsub: gossipsub(), + datastore, + nat: { + enabled: false + }, + relay: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..881b545d --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,46 @@ +import { createHelia } from 'helia' +import { createLibp2p, Libp2pOptions } from 'libp2p' +import { tcp } from '@libp2p/tcp' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + addresses: { + listen: [ + '/ip4/0.0.0.0/tcp/0' + ] + }, + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + relay: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..7fd48729 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { Controller, ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import { isElectronMain, isNode } from 'wherearewe' +import mergeOptions from 'merge-options' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return await createController(opts) +} diff --git a/packages/interop/test/fixtures/create-peer-ids.ts b/packages/interop/test/fixtures/create-peer-ids.ts new file mode 100644 index 00000000..c38834ef --- /dev/null +++ b/packages/interop/test/fixtures/create-peer-ids.ts @@ -0,0 +1,46 @@ +import { xor as uint8ArrayXor } from 'uint8arrays/xor' +import { compare as uint8ArrayCompare } from 'uint8arrays/compare' +import all from 'it-all' +import map from 'it-map' +import type { PeerId } from '@libp2p/interface-peer-id' +import { sha256 } from 'multiformats/hashes/sha2' + +/** + * Sort peers by distance to the KadID of the passed buffer + */ +export async function sortClosestPeers (buf: Uint8Array, peers: PeerId[]): Promise { + const kadId = await convertBuffer(buf) + + const distances = await all( + map(peers, async (peer) => { + const id = await convertPeerId(peer) + + return { + peer, + distance: uint8ArrayXor(id, kadId) + } + }) + ) + + return distances + .sort((a, b) => { + return uint8ArrayCompare(a.distance, b.distance) + }) + .map((d) => d.peer) +} + +/** + * Creates a DHT ID by hashing a Peer ID + */ +export async function convertPeerId (peerId: PeerId): Promise { + return await convertBuffer(peerId.toBytes()) +} + +/** + * Creates a DHT ID by hashing a given Uint8Array + */ +export async function convertBuffer (buf: Uint8Array): Promise { + const multihash = await sha256.digest(buf) + + return multihash.digest +} diff --git a/packages/interop/test/fixtures/wait-for.ts b/packages/interop/test/fixtures/wait-for.ts new file mode 100644 index 00000000..3c1451f1 --- /dev/null +++ b/packages/interop/test/fixtures/wait-for.ts @@ -0,0 +1,27 @@ + +export interface WaitForOptions { + timeout: number + delay?: number + message?: string +} + +export async function waitFor (fn: () => Promise, options: WaitForOptions): Promise { + const delay = options.delay ?? 100 + const timeoutAt = Date.now() + options.timeout + + while (true) { + const result = await fn() + + if (result) { + return + } + + await new Promise((resolve) => { + setTimeout(() => { resolve() }, delay) + }) + + if (Date.now() > timeoutAt) { + throw new Error(options.message ?? 'WaitFor timed out') + } + } +} diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts new file mode 100644 index 00000000..02453832 --- /dev/null +++ b/packages/interop/test/pubsub.spec.ts @@ -0,0 +1,176 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { identity } from 'multiformats/hashes/identity' +import { base36 } from 'multiformats/bases/base36' +import type { IPNS } from '@helia/ipns' +import { ipns } from '@helia/ipns' +import { pubsub } from '@helia/ipns/routing' +import last from 'it-last' +import { peerIdFromKeys } from '@libp2p/peer-id' +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { waitFor } from './fixtures/wait-for.js' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { concat as uint8ArrayConcat } from 'uint8arrays/concat' +import { connect } from './fixtures/connect.js' + +const LIBP2P_KEY_CODEC = 0x72 + +describe('pubsub routing', () => { + let helia: Helia + let kubo: Controller + let name: IPNS + + beforeEach(async () => { + helia = await createHeliaNode({ + pubsub: gossipsub() + }) + kubo = await createKuboNode({ + ipfsOptions: { + config: { + Routing: { + Type: 'none' + } + } + }, + args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] + }) + + // connect the two nodes + await connect(helia, kubo, '/meshsub/1.1.0') + + name = ipns(helia, [ + pubsub(helia) + ]) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should publish on helia and resolve on kubo', async () => { + const input = Uint8Array.from([0, 1, 2, 3, 4]) + const digest = await sha256.digest(input) + const cid = CID.createV1(raw.code, digest) + + const keyName = 'my-ipns-key' + await helia.libp2p.keychain.createKey(keyName, 'Ed25519') + const peerId = await helia.libp2p.keychain.exportPeerId(keyName) + + if (peerId.publicKey == null) { + throw new Error('No public key present') + } + + // first publish should fail because kubo isn't subscribed to key update channel + await expect(name.publish(peerId, cid)).to.eventually.be.rejected() + .with.property('message', 'PublishError.InsufficientPeers') + + // should fail to resolve the first time as kubo was not subscribed to the pubsub channel + await expect(last(kubo.api.name.resolve(peerId, { + timeout: 100 + }))).to.eventually.be.undefined() + + // magic pubsub subscription name + const subscriptionName = `/ipns/${CID.createV1(LIBP2P_KEY_CODEC, identity.digest(peerId.publicKey)).toString(base36)}` + + // wait for kubo to be subscribed to updates + await waitFor(async () => { + const subs = await kubo.api.name.pubsub.subs() + + return subs.includes(subscriptionName) + }, { + timeout: 30000 + }) + + // publish should now succeed + await name.publish(peerId, cid) + + // kubo should now be able to resolve IPNS name + const resolved = await last(kubo.api.name.resolve(peerId, { + timeout: 100 + })) + + expect(resolved).to.equal(`/ipfs/${cid.toString()}`) + }) + + it('should publish on kubo and resolve on helia', async () => { + const keyName = 'my-ipns-key' + const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) + const result = await kubo.api.key.gen(keyName, { + // @ts-expect-error kubo needs this in lower case + type: 'ed25519' + }) + + // the generated id is libp2p-key CID with the public key as an identity multihash + const peerCid = CID.parse(result.id, base36) + const peerId = await peerIdFromKeys(peerCid.multihash.digest) + + // first call to pubsub resolver should fail but we should now be subscribed for updates + await expect(name.resolve(peerId)).to.eventually.be.rejected() + + // actual pubsub subscription name + const subscriptionName = `/record/${uint8ArrayToString(uint8ArrayConcat([ + uint8ArrayFromString('/ipns/'), + peerId.toBytes() + ]), 'base64url')}` + + // wait for helia to be subscribed to the topic for record updates + await waitFor(async () => { + return helia.libp2p.pubsub.getTopics().includes(subscriptionName) + }, { + timeout: 30000, + message: 'Helia did not register for record updates' + }) + + // wait for kubo to see that helia is subscribed to the topic for record updates + await waitFor(async () => { + const peers = await kubo.api.pubsub.peers(subscriptionName) + + return peers.map(p => p.toString()).includes(helia.libp2p.peerId.toString()) + }, { + timeout: 30000, + message: 'Kubo did not see that Helia was registered for record updates' + }) + + // now publish, this should cause a pubsub message on the topic for record updates + await kubo.api.name.publish(cid, { + key: keyName + }) + + let resolvedCid: CID | undefined + + // we should get an update eventually + await waitFor(async () => { + try { + resolvedCid = await name.resolve(peerId) + + return true + } catch { + return false + } + }, { + timeout: 10000, + message: 'Helia could not resolve the IPNS record' + }) + + if (resolvedCid == null) { + throw new Error('Failed to resolve CID') + } + + expect(resolvedCid.toString()).to.equal(cid.toString()) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..61f84a69 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../ipns" + } + ] +} diff --git a/packages/ipns/LICENSE b/packages/ipns/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/ipns/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/ipns/LICENSE-APACHE b/packages/ipns/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/ipns/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/ipns/LICENSE-MIT b/packages/ipns/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/ipns/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/ipns/README.md b/packages/ipns/README.md new file mode 100644 index 00000000..cdca30ff --- /dev/null +++ b/packages/ipns/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/ipns + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> An implementation of IPNS for Helia + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/ipns/package.json b/packages/ipns/package.json new file mode 100644 index 00000000..5b087703 --- /dev/null +++ b/packages/ipns/package.json @@ -0,0 +1,191 @@ +{ + "name": "@helia/ipns", + "version": "0.0.0", + "description": "An implementation of IPNS for Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-ipns.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-ipns/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./routing": { + "types": "./dist/src/routing/index.d.ts", + "import": "./dist/src/routing/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "next", + "@libp2p/interface-dht": "^2.0.1", + "@libp2p/interface-peer-id": "^2.0.1", + "@libp2p/interface-pubsub": "^3.0.6", + "@libp2p/interfaces": "^3.3.1", + "@libp2p/logger": "^2.0.5", + "@libp2p/peer-id": "^2.0.1", + "@libp2p/record": "^3.0.0", + "hashlru": "^2.3.0", + "interface-datastore": "^7.0.4", + "ipns": "^5.0.1", + "is-ipfs": "^8.0.1", + "multiformats": "^11.0.1", + "p-queue": "^7.3.0", + "progress-events": "^1.0.0", + "uint8arrays": "^4.0.3" + }, + "devDependencies": { + "@libp2p/peer-id-factory": "^2.0.1", + "aegir": "^38.1.0", + "datastore-core": "^8.0.4", + "sinon": "^15.0.1" + }, + "browser": { + "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts new file mode 100644 index 00000000..2e13c946 --- /dev/null +++ b/packages/ipns/src/index.ts @@ -0,0 +1,296 @@ +/** + * @packageDocumentation + * + * IPNS operations using a Helia node + * + * @example + * + * ```typescript + * import { gossipsub } from '@chainsafe/libp2p' + * import { kadDHT } from '@libp2p/kad-dht' + * import { createLibp2p } from 'libp2p' + * import { createHelia } from 'helia' + * import { ipns, ipnsValidator, ipnsSelector } from '@helia/ipns' + * import { dht, pubsub } from '@helia/ipns/routing' + * import { unixfs } from '@helia/unixfs + * + * const libp2p = await createLibp2p({ + * dht: kadDHT({ + * validators: { + * ipns: ipnsValidator + * }, + * selectors: { + * ipns: ipnsSelector + * } + * }), + * pubsub: gossipsub() + * }) + * + * const helia = await createHelia({ + * libp2p, + * //.. other options + * }) + * const name = ipns(helia, [ + * dht(helia) + * pubsub(helia) + * ]) + * + * // create a public key to publish as an IPNS name + * const keyInfo = await helia.libp2p.keychain.createKey('my-key') + * const peerId = await helia.libp2p.keychain.exportPeerId(keyInfo.name) + * + * // store some data to publish + * const fs = unixfs(helia) + * const cid = await fs.add(Uint8Array.from([0, 1, 2, 3, 4])) + * + * // publish the name + * await name.publish(peerId, cid) + * + * // resolve the name + * const cid = name.resolve(peerId) + * ``` + * + * @example + * + * ```typescript + * // resolve a CID from a TXT record in a DNS zone file, eg: + * // > dig ipfs.io TXT + * // ;; ANSWER SECTION: + * // ipfs.io. 435 IN TXT "dnslink=/ipfs/Qmfoo" + * + * const cid = name.resolveDns('ipfs.io') + * ``` + */ + +import type { AbortOptions } from '@libp2p/interfaces' +import { isPeerId, PeerId } from '@libp2p/interface-peer-id' +import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' +import type { IPNSEntry } from 'ipns' +import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' +import { ipnsValidator } from 'ipns/validator' +import { CID } from 'multiformats/cid' +import { resolveDnslink } from './utils/resolve-dns-link.js' +import { logger } from '@libp2p/logger' +import { peerIdFromString } from '@libp2p/peer-id' +import type { ProgressEvent, ProgressOptions } from 'progress-events' +import { CustomProgressEvent } from 'progress-events' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import type { Datastore } from 'interface-datastore' +import { localStore, LocalStore } from './routing/local-store.js' + +const log = logger('helia:ipns') + +const MINUTE = 60 * 1000 +const HOUR = 60 * MINUTE + +const DEFAULT_LIFETIME_MS = 24 * HOUR +const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR + +export type PublishProgressEvents = + ProgressEvent<'ipns:publish:start'> | + ProgressEvent<'ipns:publish:success', IPNSEntry> | + ProgressEvent<'ipns:publish:error', Error> + +export type ResolveProgressEvents = + ProgressEvent<'ipns:resolve:start', unknown> | + ProgressEvent<'ipns:resolve:success', IPNSEntry> | + ProgressEvent<'ipns:resolve:error', Error> + +export type RepublishProgressEvents = + ProgressEvent<'ipns:republish:start', unknown> | + ProgressEvent<'ipns:republish:success', IPNSEntry> | + ProgressEvent<'ipns:republish:error', { record: IPNSEntry, err: Error }> + +export interface PublishOptions extends AbortOptions, ProgressOptions { + /** + * Time duration of the record in ms + */ + lifetime?: number +} + +export interface ResolveOptions extends AbortOptions, ProgressOptions { + /** + * do not use cached entries + */ + nocache?: boolean +} + +export interface RepublishOptions extends AbortOptions, ProgressOptions { + /** + * The republish interval in ms (default: 24hrs) + */ + interval?: number +} + +export interface IPNS { + /** + * Creates an IPNS record signed by the passed PeerId that will resolve to the passed value + * + * If the valid is a PeerId, a recursive IPNS record will be created. + */ + publish: (key: PeerId, value: CID | PeerId, options?: PublishOptions) => Promise + + /** + * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS record + * corresponding to that public key until a value is found + */ + resolve: (key: PeerId, options?: ResolveOptions) => Promise + + /** + * Resolve a CID from a dns-link style IPNS record + */ + resolveDns: (domain: string, options?: ResolveOptions) => Promise + + /** + * Periodically republish all IPNS records found in the datastore + */ + republish: (options?: RepublishOptions) => void +} + +export type { IPNSRouting } from './routing/index.js' + +export interface IPNSComponents { + datastore: Datastore +} + +class DefaultIPNS implements IPNS { + private readonly routers: IPNSRouting[] + private readonly localStore: LocalStore + private timeout?: ReturnType + + constructor (components: IPNSComponents, routers: IPNSRouting[] = []) { + this.routers = routers + this.localStore = localStore(components.datastore) + } + + async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { + try { + let sequenceNumber = 1n + const routingKey = peerIdToRoutingKey(key) + + if (await this.localStore.has(routingKey, options)) { + // if we have published under this key before, increment the sequence number + const buf = await this.localStore.get(routingKey, options) + const existingRecord = unmarshal(buf) + sequenceNumber = existingRecord.sequence + 1n + } + + let str + + if (isPeerId(value)) { + str = `/ipns/${value.toString()}` + } else { + str = `/ipfs/${value.toString()}` + } + + const bytes = uint8ArrayFromString(str) + + // create record + const record = await create(key, bytes, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS) + const marshaledRecord = marshal(record) + + await this.localStore.put(routingKey, marshaledRecord, options) + + // publish record to routing + await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) + + return record + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:publish:error', err)) + throw err + } + } + + async resolve (key: PeerId, options: ResolveOptions = {}): Promise { + const routingKey = peerIdToRoutingKey(key) + const record = await this.#findIpnsRecord(routingKey, options) + const str = uint8ArrayToString(record.value) + + return await this.#resolve(str) + } + + async resolveDns (domain: string, options: ResolveOptions = {}): Promise { + const dnslink = await resolveDnslink(domain, options) + + return await this.#resolve(dnslink) + } + + republish (options: RepublishOptions = {}): void { + if (this.timeout != null) { + throw new Error('Republish is already running') + } + + options.signal?.addEventListener('abort', () => { + clearTimeout(this.timeout) + }) + + async function republish (): Promise { + const startTime = Date.now() + + options.onProgress?.(new CustomProgressEvent('ipns:republish:start')) + + const finishType = Date.now() + const timeTaken = finishType - startTime + let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken + + if (nextInterval < 0) { + nextInterval = options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS + } + + setTimeout(() => { + republish().catch(err => { + log.error('error republishing', err) + }) + }, nextInterval) + } + + this.timeout = setTimeout(() => { + republish().catch(err => { + log.error('error republishing', err) + }) + }, options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS) + } + + async #resolve (ipfsPath: string): Promise { + const parts = ipfsPath.split('/') + + if (parts.length === 3) { + const scheme = parts[1] + + if (scheme === 'ipns') { + return await this.resolve(peerIdFromString(parts[2])) + } else if (scheme === 'ipfs') { + return CID.parse(parts[2]) + } + } + + log.error('invalid ipfs path %s', ipfsPath) + throw new Error('Invalid value') + } + + async #findIpnsRecord (routingKey: Uint8Array, options: AbortOptions): Promise { + const routers = [ + this.localStore, + ...this.routers + ] + + const unmarshaledRecord = await Promise.any( + routers.map(async (router) => { + const unmarshaledRecord = await router.get(routingKey, options) + await ipnsValidator(routingKey, unmarshaledRecord) + + return unmarshaledRecord + }) + ) + + return unmarshal(unmarshaledRecord) + } +} + +export function ipns (components: IPNSComponents, routers: IPNSRouting[] = []): IPNS { + return new DefaultIPNS(components, routers) +} + +export { ipnsValidator } +export { ipnsSelector } from 'ipns/selector' diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/dht.ts new file mode 100644 index 00000000..a160cecc --- /dev/null +++ b/packages/ipns/src/routing/dht.ts @@ -0,0 +1,85 @@ +import { logger } from '@libp2p/logger' +import type { IPNSRouting } from '../index.js' +import type { DHT, QueryEvent } from '@libp2p/interface-dht' +import type { GetOptions, PutOptions } from './index.js' +import { CustomProgressEvent, ProgressEvent } from 'progress-events' + +const log = logger('helia:ipns:routing:dht') + +export interface DHTRoutingComponents { + libp2p: { + dht: DHT + } +} + +export type DHTProgressEvents = + ProgressEvent<'ipns:routing:dht:query', QueryEvent> | + ProgressEvent<'ipns:routing:dht:error', Error> + +export class DHTRouting implements IPNSRouting { + private readonly dht: DHT + + constructor (components: DHTRoutingComponents) { + this.dht = components.libp2p.dht + } + + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise { + let putValue = false + + try { + for await (const event of this.dht.put(routingKey, marshaledRecord, options)) { + logEvent('DHT put event', event) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:query', event)) + + if (event.name === 'PEER_RESPONSE' && event.messageName === 'PUT_VALUE') { + putValue = true + } + } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) + } + + if (!putValue) { + throw new Error('Could not put value to DHT') + } + } + + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + for await (const event of this.dht.get(routingKey, options)) { + logEvent('DHT get event', event) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:query', event)) + + if (event.name === 'VALUE') { + return event.value + } + } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) + } + + throw new Error('Not found') + } +} + +function logEvent (prefix: string, event: QueryEvent): void { + if (event.name === 'SENDING_QUERY') { + log(prefix, event.name, event.messageName, '->', event.to.toString()) + } else if (event.name === 'PEER_RESPONSE') { + log(prefix, event.name, event.messageName, '<-', event.from.toString()) + } else if (event.name === 'FINAL_PEER') { + log(prefix, event.name, event.peer.id.toString()) + } else if (event.name === 'QUERY_ERROR') { + log(prefix, event.name, event.error.message) + } else if (event.name === 'PROVIDER') { + log(prefix, event.name, event.providers.map(p => p.id.toString()).join(', ')) + } else { + log(prefix, event.name) + } +} + +export function dht (components: DHTRoutingComponents): IPNSRouting { + return new DHTRouting(components) +} diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts new file mode 100644 index 00000000..38374874 --- /dev/null +++ b/packages/ipns/src/routing/index.ts @@ -0,0 +1,26 @@ +import type { ProgressOptions } from 'progress-events' +import type { AbortOptions } from '@libp2p/interfaces' +import type { DHTProgressEvents } from './dht.js' +import type { DatastoreProgressEvents } from './local-store.js' +import type { PubSubProgressEvents } from './pubsub.js' + +export interface PutOptions extends AbortOptions, ProgressOptions { + +} + +export interface GetOptions extends AbortOptions, ProgressOptions { + +} + +export interface IPNSRouting { + put: (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions) => Promise + get: (routingKey: Uint8Array, options?: GetOptions) => Promise +} + +export type IPNSRoutingEvents = + DatastoreProgressEvents | + DHTProgressEvents | + PubSubProgressEvents + +export { dht } from './dht.js' +export { pubsub } from './pubsub.js' diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts new file mode 100644 index 00000000..4ea0a375 --- /dev/null +++ b/packages/ipns/src/routing/local-store.ts @@ -0,0 +1,63 @@ +import { CustomProgressEvent, ProgressEvent } from 'progress-events' +import type { AbortOptions } from '@libp2p/interfaces' +import { Libp2pRecord } from '@libp2p/record' +import { Datastore, Key } from 'interface-datastore' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import type { GetOptions, IPNSRouting, PutOptions } from '../routing' + +function dhtRoutingKey (key: Uint8Array): Key { + return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) +} + +export type DatastoreProgressEvents = + ProgressEvent<'ipns:routing:datastore:put'> | + ProgressEvent<'ipns:routing:datastore:get'> | + ProgressEvent<'ipns:routing:datastore:error', Error> + +export interface LocalStore extends IPNSRouting { + has: (routingKey: Uint8Array, options?: AbortOptions) => Promise +} + +/** + * Returns an IPNSRouting implementation that reads/writes IPNS records to the + * datastore as DHT records. This lets us publish IPNS records offline then + * serve them to the network later in response to DHT queries. + */ +export function localStore (datastore: Datastore): LocalStore { + return { + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}) { + try { + const key = dhtRoutingKey(routingKey) + + // Marshal to libp2p record as the DHT does + const record = new Libp2pRecord(routingKey, marshaledRecord, new Date()) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put')) + await datastore.put(key, record.serialize(), options) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + }, + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + const key = dhtRoutingKey(routingKey) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:get')) + const buf = await datastore.get(key, options) + + // Unmarshal libp2p record as the DHT does + const record = Libp2pRecord.deserialize(buf) + + return record.value + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + }, + async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + const key = dhtRoutingKey(routingKey) + return await datastore.has(key, options) + } + } +} diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts new file mode 100644 index 00000000..bb10e300 --- /dev/null +++ b/packages/ipns/src/routing/pubsub.ts @@ -0,0 +1,195 @@ +import { peerIdToRoutingKey } from 'ipns' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { logger } from '@libp2p/logger' +import type { PeerId } from '@libp2p/interface-peer-id' +import type { Message, PublishResult, PubSub } from '@libp2p/interface-pubsub' +import type { Datastore } from 'interface-datastore' +import type { GetOptions, IPNSRouting, PutOptions } from './index.js' +import { CodeError } from '@libp2p/interfaces/errors' +import { localStore, LocalStore } from './local-store.js' +import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' +import { CustomProgressEvent, ProgressEvent } from 'progress-events' + +const log = logger('helia:ipns:routing:pubsub') + +export interface PubsubRoutingComponents { + datastore: Datastore + libp2p: { + peerId: PeerId + pubsub: PubSub + } +} + +export type PubSubProgressEvents = + ProgressEvent<'ipns:pubsub:publish', { topic: string, result: PublishResult }> | + ProgressEvent<'ipns:pubsub:subscribe', { topic: string }> | + ProgressEvent<'ipns:pubsub:error', Error> + +/** + * This IPNS routing receives IPNS record updates via dedicated + * pubsub topic. + * + * Note we must first be subscribed to the topic in order to receive + * updated records, so the first call to `.get` should be expected + * to fail! + */ +class PubSubRouting implements IPNSRouting { + private subscriptions: string[] + private readonly localStore: LocalStore + private readonly peerId: PeerId + private readonly pubsub: PubSub + + constructor (components: PubsubRoutingComponents) { + this.subscriptions = [] + this.localStore = localStore(components.datastore) + this.peerId = components.libp2p.peerId + this.pubsub = components.libp2p.pubsub + + this.pubsub.addEventListener('message', (evt) => { + const message = evt.detail + + if (!this.subscriptions.includes(message.topic)) { + return + } + + this.#processPubSubMessage(message).catch(err => { + log.error('Error processing message', err) + }) + }) + } + + async #processPubSubMessage (message: Message): Promise { + log('message received for topic', message.topic) + + if (message.type !== 'signed') { + log.error('unsigned message received, this module can only work with signed messages') + return + } + + if (message.from.equals(this.peerId)) { + log('not storing record from self') + return + } + + const routingKey = topicToKey(message.topic) + + await ipnsValidator(routingKey, message.data) + + if (await this.localStore.has(routingKey)) { + const currentRecord = await this.localStore.get(routingKey) + + if (uint8ArrayEquals(currentRecord, message.data)) { + log('not storing record as we already have it') + return + } + + const records = [currentRecord, message.data] + const index = ipnsSelector(routingKey, records) + + if (index === 0) { + log('not storing record as the one we have is better') + return + } + } + + await this.localStore.put(routingKey, message.data) + } + + /** + * Put a value to the pubsub datastore indexed by the received key properly encoded + */ + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise { + try { + const topic = keyToTopic(routingKey) + + log('publish value for topic %s', topic) + const result = await this.pubsub.publish(topic, marshaledRecord) + + log('published record on topic %s to %d recipients', topic, result.recipients) + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:publish', { topic, result })) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:error', err)) + throw err + } + } + + /** + * Get a value from the pubsub datastore indexed by the received key properly encoded. + * Also, the identifier topic is subscribed to and the pubsub datastore records will be + * updated once new publishes occur + */ + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + const topic = keyToTopic(routingKey) + + // ensure we are subscribed to topic + if (!this.pubsub.getTopics().includes(topic)) { + log('add subscription for topic', topic) + this.pubsub.subscribe(topic) + this.subscriptions.push(topic) + + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:subscribe', { topic })) + } + + // chain through to local store + return await this.localStore.get(routingKey, options) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:error', err)) + throw err + } + } + + /** + * Get pubsub subscriptions related to ipns + */ + getSubscriptions (): string[] { + return this.subscriptions + } + + /** + * Cancel pubsub subscriptions related to ipns + */ + cancel (key: PeerId): void { + const routingKey = peerIdToRoutingKey(key) + const topic = keyToTopic(routingKey) + + // Not found topic + if (!this.subscriptions.includes(topic)) { + return + } + + this.pubsub.unsubscribe(topic) + this.subscriptions = this.subscriptions.filter(t => t !== topic) + } +} + +const PUBSUB_NAMESPACE = '/record/' + +/** + * converts a binary record key to a pubsub topic key + */ +function keyToTopic (key: Uint8Array): string { + const b64url = uint8ArrayToString(key, 'base64url') + + return `${PUBSUB_NAMESPACE}${b64url}` +} + +/** + * converts a pubsub topic key to a binary record key + */ +function topicToKey (topic: string): Uint8Array { + if (topic.substring(0, PUBSUB_NAMESPACE.length) !== PUBSUB_NAMESPACE) { + throw new CodeError('topic received is not from a record', 'ERR_TOPIC_IS_NOT_FROM_RECORD_NAMESPACE') + } + + const key = topic.substring(PUBSUB_NAMESPACE.length) + + return uint8ArrayFromString(key, 'base64url') +} + +export function pubsub (components: PubsubRoutingComponents): IPNSRouting { + return new PubSubRouting(components) +} diff --git a/packages/ipns/src/utils/resolve-dns-link.browser.ts b/packages/ipns/src/utils/resolve-dns-link.browser.ts new file mode 100644 index 00000000..459f387c --- /dev/null +++ b/packages/ipns/src/utils/resolve-dns-link.browser.ts @@ -0,0 +1,61 @@ +/* eslint-env browser */ + +import { TLRU } from './tlru.js' +import PQueue from 'p-queue' +import type { AbortOptions } from '@libp2p/interfaces' + +// Avoid sending multiple queries for the same hostname by caching results +const cache = new TLRU<{ Path: string, Message: string }>(1000) +// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884 +// However we know browsers themselves cache DNS records for at least 1 minute, +// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426 +const ttl = 60 * 1000 + +// browsers limit concurrent connections per host, +// we don't want preload calls to exhaust the limit (~6) +const httpQueue = new PQueue({ concurrency: 4 }) + +const ipfsPath = (response: { Path: string, Message: string }): string => { + if (response.Path != null) { + return response.Path + } + throw new Error(response.Message) +} + +export interface ResolveDnsLinkOptions extends AbortOptions { + nocache?: boolean +} + +export async function resolveDnslink (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise { // eslint-disable-line require-await + const resolve = async (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise => { + // @ts-expect-error - URLSearchParams does not take boolean options, only strings + const searchParams = new URLSearchParams(opts) + searchParams.set('arg', fqdn) + + // try cache first + const query = searchParams.toString() + if (opts.nocache !== true && cache.has(query)) { + const response = cache.get(query) + + if (response != null) { + return ipfsPath(response) + } + } + + // fallback to delegated DNS resolver + const response = await httpQueue.add(async () => { + // Delegated HTTP resolver sending DNSLink queries to ipfs.io + // TODO: replace hardcoded host with configurable DNS over HTTPS: https://github.com/ipfs/js-ipfs/issues/2212 + const res = await fetch(`https://ipfs.io/api/v0/dns?${searchParams}`) + const query = new URL(res.url).search.slice(1) + const json = await res.json() + cache.set(query, json, ttl) + + return json + }) + + return ipfsPath(response) + } + + return await resolve(fqdn, opts) +} diff --git a/packages/ipns/src/utils/resolve-dns-link.ts b/packages/ipns/src/utils/resolve-dns-link.ts new file mode 100644 index 00000000..34ec0341 --- /dev/null +++ b/packages/ipns/src/utils/resolve-dns-link.ts @@ -0,0 +1,65 @@ +import dns from 'dns' +import { promisify } from 'util' +import type { AbortOptions } from '@libp2p/interfaces' +import * as isIPFS from 'is-ipfs' + +const MAX_RECURSIVE_DEPTH = 32 + +export async function resolveDnslink (domain: string, options: AbortOptions = {}): Promise { + return await recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, options) +} + +async function recursiveResolveDnslink (domain: string, depth: number, options: AbortOptions = {}): Promise { + if (depth === 0) { + throw new Error('recursion limit exceeded') + } + + let dnslinkRecord + + try { + dnslinkRecord = await resolve(domain) + } catch (err: any) { + // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error + if (err.code !== 'ENOTFOUND' && err.code !== 'ERR_DNSLINK_NOT_FOUND' && err.code !== 'ENODATA') { + throw err + } + + if (domain.startsWith('_dnslink.')) { + // The supplied domain contains a _dnslink component + // Check the non-_dnslink domain + dnslinkRecord = await resolve(domain.replace('_dnslink.', '')) + } else { + // Check the _dnslink subdomain + const _dnslinkDomain = `_dnslink.${domain}` + // If this throws then we propagate the error + dnslinkRecord = await resolve(_dnslinkDomain) + } + } + + const result = dnslinkRecord.replace('dnslink=', '') + const domainOrCID = result.split('/')[2] + const isIPFSCID = isIPFS.cid(domainOrCID) + + if (isIPFSCID || depth === 0) { + return result + } + + return await recursiveResolveDnslink(domainOrCID, depth - 1, options) +} + +async function resolve (domain: string, options: AbortOptions = {}): Promise { + const DNSLINK_REGEX = /^dnslink=.+$/ + const records = await promisify(dns.resolveTxt)(domain) + const dnslinkRecords = records.reduce((rs, r) => rs.concat(r), []) + .filter(record => DNSLINK_REGEX.test(record)) + + const dnslinkRecord = dnslinkRecords[0] + + // we now have dns text entries as an array of strings + // only records passing the DNSLINK_REGEX text are included + if (dnslinkRecord == null) { + throw new Error(`No dnslink records found for domain: ${domain}`) + } + + return dnslinkRecord +} diff --git a/packages/ipns/src/utils/tlru.ts b/packages/ipns/src/utils/tlru.ts new file mode 100644 index 00000000..0556c0e6 --- /dev/null +++ b/packages/ipns/src/utils/tlru.ts @@ -0,0 +1,52 @@ +import hashlru from 'hashlru' + +/** + * Time Aware Least Recent Used Cache + * + * @see https://arxiv.org/pdf/1801.00390 + */ +export class TLRU { + private readonly lru: ReturnType + + constructor (maxSize: number) { + this.lru = hashlru(maxSize) + } + + get (key: string): T | undefined { + const value = this.lru.get(key) + + if (value != null) { + if (value.expire != null && value.expire < Date.now()) { + this.lru.remove(key) + + return undefined + } + + return value.value + } + + return undefined + } + + set (key: string, value: T, ttl: number): void { + this.lru.set(key, { value, expire: Date.now() + ttl }) + } + + has (key: string): boolean { + const value = this.get(key) + + if (value != null) { + return true + } + + return false + } + + remove (key: string): void { + this.lru.remove(key) + } + + clear (): void { + this.lru.clear() + } +} diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts new file mode 100644 index 00000000..2ff3ea49 --- /dev/null +++ b/packages/ipns/test/publish.spec.ts @@ -0,0 +1,50 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import type { IPNS } from '../src/index.js' +import { ipns } from '../src/index.js' +import { CID } from 'multiformats/cid' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import Sinon from 'sinon' + +const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + +describe('publish', () => { + let name: IPNS + + before(async () => { + const datastore = new MemoryDatastore() + + name = ipns({ datastore }) + }) + + it('should publish an IPNS record with the default params', async function () { + const key = await createEd25519PeerId() + const ipnsEntry = await name.publish(key, cid) + + expect(ipnsEntry).to.have.property('sequence', 1n) + expect(ipnsEntry).to.have.property('ttl', 8640000000000n) // 24 hours + }) + + it('should publish an IPNS record with a custom ttl params', async function () { + const key = await createEd25519PeerId() + const lifetime = 123000 + const ipnsEntry = await name.publish(key, cid, { + lifetime + }) + + expect(ipnsEntry).to.have.property('sequence', 1n) + expect(ipnsEntry).to.have.property('ttl', BigInt(lifetime) * 100000n) + }) + + it('should emit progress events', async function () { + const key = await createEd25519PeerId() + const onProgress = Sinon.stub() + await name.publish(key, cid, { + onProgress + }) + + expect(onProgress).to.have.property('called', true) + }) +}) diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts new file mode 100644 index 00000000..e4891278 --- /dev/null +++ b/packages/ipns/test/resolve.spec.ts @@ -0,0 +1,78 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import type { IPNS } from '../src/index.js' +import { ipns } from '../src/index.js' +import { CID } from 'multiformats/cid' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import Sinon from 'sinon' + +const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + +describe('resolve', () => { + let name: IPNS + + before(async () => { + const datastore = new MemoryDatastore() + + name = ipns({ datastore }) + }) + + it('should resolve a record', async () => { + const key = await createEd25519PeerId() + await name.publish(key, cid) + + const resolvedValue = await name.resolve(key) + + if (resolvedValue == null) { + throw new Error('Did not resolve entry') + } + + expect(resolvedValue.toString()).to.equal(cid.toString()) + }) + + it('should resolve a recursive record', async () => { + const key1 = await createEd25519PeerId() + const key2 = await createEd25519PeerId() + await name.publish(key2, cid) + await name.publish(key1, key2) + + const resolvedValue = await name.resolve(key1) + + if (resolvedValue == null) { + throw new Error('Did not resolve entry') + } + + expect(resolvedValue.toString()).to.equal(cid.toString()) + }) + + it('should resolve /ipns/tableflip.io', async function () { + const domain = 'tableflip.io' + + try { + const resolvedValue = await name.resolveDns(domain) + + expect(resolvedValue).to.be.an.instanceOf(CID) + } catch (err: any) { + // happens when running tests offline + if (err.message.includes(`ECONNREFUSED ${domain}`) === true) { + return this.skip() + } + + throw err + } + }) + + it('should emit progress events', async function () { + const onProgress = Sinon.stub() + const key = await createEd25519PeerId() + await name.publish(key, cid) + + await name.resolve(key, { + onProgress + }) + + expect(onProgress).to.have.property('called', true) + }) +}) diff --git a/packages/ipns/tsconfig.json b/packages/ipns/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/ipns/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} From 8396e687fd2cbc6ca4ff80144654fa032080cf2e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 Feb 2023 15:59:23 +0100 Subject: [PATCH 005/253] docs: update readme --- README.md | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/README.md b/README.md index 08516c56..51428438 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,6 @@ ## Table of contents - - [Structure](#structure) -- [helia ](#helia----omit-in-toc---) - - [Project status](#project-status) - - [Name](#name) - - [Background](#background) - - [Roadmap](#roadmap) - [API Docs](#api-docs) - [License](#license) - [Contribute](#contribute) @@ -30,41 +25,6 @@ - [`/packages/interop`](./packages/interop) Interop tests for @helia/ipns - [`/packages/ipns`](./packages/ipns) An implementation of IPNS for Helia -helia logo - -# helia - -[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) -[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/js-test-and-release.yml?query=branch%3Amain) - -## Project status - -This project is pre-alpha and is currently in development. An initial v1 release is planned for [late Q1 2023](/ROADMAP.md#late-q1-march). Helia is being built in the open; community contributors are welcome! - -The core of IPFS is the [Files API](https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md), which will likewise be implemented in Helia. These initial building blocks are in development now; have a look at this repo's PR(s). For more info about Helia, please see the [Roadmap](https://github.com/ipfs/helia/issues/5) and the [Manifesto](MANIFESTO.md). - -We are also sharing about the progress so far, and discussing how you can get involved, at [Helia Demo Day](https://lu.ma/helia) every couple weeks. We'd love to see you there! - -## Name - -Helia (*HEE-lee-ah*) is the Latin spelling of Ἡλιη -- in Greek mythology, one of the [Heliades](https://www.wikidata.org/wiki/Q12656412): the daughters of the sun god Helios. When their brother Phaethon died trying to drive the sun chariot across the sky, their tears of mourning fell to earth as amber, which is yellow (sort of), and so is JavaScript. They were then turned into [poplar](https://en.wiktionary.org/wiki/poplar) trees and, well, JavaScript is quite popular. - -In Oct–Dec 2022, IP Stewards [sought](https://github.com/ipfs/pomegranate/issues/3) community input for the name of this project. After considering 20 suggestions and holding a couple of polls, the name **Helia** was chosen. Here's [why it's not named IPFS](https://github.com/ipfs/ipfs/issues/470). - -## Background - -This project aims to build a lean, modular, and modern implementation of IPFS, the Interplanetary File System. - -For more information, see the [State of IPFS in JS (blog post)](https://blog.ipfs.tech/state-of-ipfs-in-js/). - -## Roadmap - -Please find and comment on [the Roadmap here](https://github.com/ipfs/helia/issues/5). - -This IPFS implementation in JavaScript is a work in progress. [Here are some ways you can help](https://blog.ipfs.tech/state-of-ipfs-in-js/#%F0%9F%A4%9D-ways-you-can-help)! - ## API Docs - From bc17d8915bd920be1ad90f4683531bbef4e7d59f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 Feb 2023 16:00:05 +0100 Subject: [PATCH 006/253] docs: update readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 51428438..4523f8f3 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ ## Table of contents -- - [Structure](#structure) - - [API Docs](#api-docs) - - [License](#license) - - [Contribute](#contribute) +- [Structure](#structure) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) ## Structure From 3c0e47d6d0710c438adf86a941ee2436680a4483 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Feb 2023 07:52:10 +0100 Subject: [PATCH 007/253] chore: update ci --- .github/workflows/js-test-and-release.yml | 16 ++++++++-------- .gitignore | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 7da2cd26..081b272a 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -33,7 +33,7 @@ jobs: node-version: ${{ matrix.node }} - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:node - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: node @@ -47,7 +47,7 @@ jobs: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: chrome @@ -61,7 +61,7 @@ jobs: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: chrome-webworker @@ -75,7 +75,7 @@ jobs: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: firefox @@ -89,7 +89,7 @@ jobs: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: firefox-webworker @@ -103,7 +103,7 @@ jobs: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: electron-main @@ -117,7 +117,7 @@ jobs: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 with: flags: electron-renderer @@ -139,5 +139,5 @@ jobs: docker-username: ${{ secrets.DOCKER_USERNAME }} - run: npm run --if-present release env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore index 910f6339..7ad9e674 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist node_modules package-lock.json yarn.lock +.vscode From 8852aee6beee187b8ecb2d271a10f896a83bc216 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Feb 2023 08:23:22 +0100 Subject: [PATCH 008/253] chore: update release scripts --- package.json | 6 ++---- packages/interop/package.json | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 0ad2b58c..46be1855 100644 --- a/package.json +++ b/package.json @@ -36,11 +36,9 @@ "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", "docs:no-publish": "npm run docs -- --publish false", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && npm run release:npm && npm run docs", - "release:npm": "aegir exec npm -- publish", - "release:rc": "aegir release-rc" + "release": "npm run docs:no-publish && aegir run release && npm run docs" }, - "dependencies": { + "devDependencies": { "aegir": "^38.1.0" }, "type": "module", diff --git a/packages/interop/package.json b/packages/interop/package.json index aebe1cf1..ba54238e 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -155,8 +155,7 @@ "test:firefox": "aegir test -t browser -- --browser firefox", "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", "test:node": "aegir test -t node --cov", - "test:electron-main": "aegir test -t electron-main", - "release": "aegir release" + "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { "@chainsafe/libp2p-gossipsub": "^6.1.0", From 61e677f6dd3298cb4f74603e894ce4af6923cec3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 15 Feb 2023 07:31:20 +0000 Subject: [PATCH 009/253] chore(release): 1.0.0 [skip ci] ## @helia/ipns-v1.0.0 (2023-02-15) ### Features * initial implementation ([#1](https://github.com/ipfs/helia-ipns/issues/1)) ([b176179](https://github.com/ipfs/helia-ipns/commit/b1761795f023e9150201e41f0b9e2c3021425f26)) --- packages/ipns/CHANGELOG.md | 6 ++++++ packages/ipns/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 packages/ipns/CHANGELOG.md diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md new file mode 100644 index 00000000..e1284b39 --- /dev/null +++ b/packages/ipns/CHANGELOG.md @@ -0,0 +1,6 @@ +## @helia/ipns-v1.0.0 (2023-02-15) + + +### Features + +* initial implementation ([#1](https://github.com/ipfs/helia-ipns/issues/1)) ([b176179](https://github.com/ipfs/helia-ipns/commit/b1761795f023e9150201e41f0b9e2c3021425f26)) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 5b087703..d13438bb 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "0.0.0", + "version": "1.0.0", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From a04bbb683dfb43c69c2fe3dca834672cf6199f21 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Wed, 15 Feb 2023 07:31:31 +0000 Subject: [PATCH 010/253] deps: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index ba54238e..0696952b 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -162,7 +162,7 @@ "@chainsafe/libp2p-noise": "^11.0.0", "@chainsafe/libp2p-yamux": "^3.0.5", "@helia/interface": "next", - "@helia/ipns": "~0.0.0", + "@helia/ipns": "^1.0.0", "@libp2p/interface-peer-id": "^2.0.1", "@libp2p/kad-dht": "^7.0.0", "@libp2p/peer-id": "^2.0.1", From 9110f33a91cb18de880ca029235b05e958120a80 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 16 Feb 2023 10:51:08 +0100 Subject: [PATCH 011/253] chore: remove release config from interop suite (#2) The interop suite is a private module so doesn't need release config. --- packages/interop/package.json | 85 ----------------------------------- 1 file changed, 85 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 0696952b..46b3b9f3 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,91 +58,6 @@ "sourceType": "module" } }, - "release": { - "branches": [ - "main" - ], - "plugins": [ - [ - "@semantic-release/commit-analyzer", - { - "preset": "conventionalcommits", - "releaseRules": [ - { - "breaking": true, - "release": "major" - }, - { - "revert": true, - "release": "patch" - }, - { - "type": "feat", - "release": "minor" - }, - { - "type": "fix", - "release": "patch" - }, - { - "type": "docs", - "release": "patch" - }, - { - "type": "test", - "release": "patch" - }, - { - "type": "deps", - "release": "patch" - }, - { - "scope": "no-release", - "release": false - } - ] - } - ], - [ - "@semantic-release/release-notes-generator", - { - "preset": "conventionalcommits", - "presetConfig": { - "types": [ - { - "type": "feat", - "section": "Features" - }, - { - "type": "fix", - "section": "Bug Fixes" - }, - { - "type": "chore", - "section": "Trivial Changes" - }, - { - "type": "docs", - "section": "Documentation" - }, - { - "type": "deps", - "section": "Dependencies" - }, - { - "type": "test", - "section": "Tests" - } - ] - } - } - ], - "@semantic-release/changelog", - "@semantic-release/npm", - "@semantic-release/github", - "@semantic-release/git" - ] - }, "scripts": { "clean": "aegir clean", "lint": "aegir lint", From adb6b0e2626a3bdd08cdc4445e3367f104bc5bb8 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 16 Feb 2023 11:24:34 +0100 Subject: [PATCH 012/253] feat: initial implementation (#1) Adds unixfs commands: - cat - chmod - cp - ls - mkdir - rm - stat - touch Each file operates on a CID and returns a new CID that refers to a DAG with the changes made to it. This is largely a lift & shift of the MFS code from js-ipfs except it returns a CID instead of writing it to the datastore. The tests have been ported from the interface suite. Sharding support is mostly complete, a couple of rm tests are skipped for now. --- .github/dependabot.yml | 11 + .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 143 ++++++++ .gitignore | 9 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ README.md | 38 ++- package.json | 167 +++++++++ src/commands/cat.ts | 32 ++ src/commands/chmod.ts | 138 ++++++++ src/commands/cp.ts | 44 +++ src/commands/ls.ts | 37 ++ src/commands/mkdir.ts | 73 ++++ src/commands/rm.ts | 35 ++ src/commands/stat.ts | 142 ++++++++ src/commands/touch.ts | 141 ++++++++ src/commands/utils/add-link.ts | 320 ++++++++++++++++++ src/commands/utils/cid-to-directory.ts | 23 ++ src/commands/utils/cid-to-pblink.ts | 26 ++ src/commands/utils/constants.ts | 2 + src/commands/utils/dir-sharded.ts | 318 +++++++++++++++++ src/commands/utils/errors.ts | 65 ++++ src/commands/utils/hamt-constants.ts | 14 + src/commands/utils/hamt-utils.ts | 294 ++++++++++++++++ src/commands/utils/is-over-shard-threshold.ts | 78 +++++ src/commands/utils/persist.ts | 27 ++ src/commands/utils/remove-link.ts | 252 ++++++++++++++ src/commands/utils/resolve.ts | 135 ++++++++ src/index.ts | 172 ++++++++++ test/cat.spec.ts | 87 +++++ test/chmod.spec.ts | 95 ++++++ test/cp.spec.ts | 169 +++++++++ test/fixtures/create-sharded-directory.ts | 27 ++ test/fixtures/create-subsharded-directory.ts | 93 +++++ test/fixtures/print-tree.ts | 33 ++ test/ls.spec.ts | 123 +++++++ test/mkdir.spec.ts | 123 +++++++ test/rm.spec.ts | 219 ++++++++++++ test/stat.spec.ts | 204 +++++++++++ test/touch.spec.ts | 150 ++++++++ tsconfig.json | 10 + 42 files changed, 4096 insertions(+), 9 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 package.json create mode 100644 src/commands/cat.ts create mode 100644 src/commands/chmod.ts create mode 100644 src/commands/cp.ts create mode 100644 src/commands/ls.ts create mode 100644 src/commands/mkdir.ts create mode 100644 src/commands/rm.ts create mode 100644 src/commands/stat.ts create mode 100644 src/commands/touch.ts create mode 100644 src/commands/utils/add-link.ts create mode 100644 src/commands/utils/cid-to-directory.ts create mode 100644 src/commands/utils/cid-to-pblink.ts create mode 100644 src/commands/utils/constants.ts create mode 100644 src/commands/utils/dir-sharded.ts create mode 100644 src/commands/utils/errors.ts create mode 100644 src/commands/utils/hamt-constants.ts create mode 100644 src/commands/utils/hamt-utils.ts create mode 100644 src/commands/utils/is-over-shard-threshold.ts create mode 100644 src/commands/utils/persist.ts create mode 100644 src/commands/utils/remove-link.ts create mode 100644 src/commands/utils/resolve.ts create mode 100644 src/index.ts create mode 100644 test/cat.spec.ts create mode 100644 test/chmod.spec.ts create mode 100644 test/cp.spec.ts create mode 100644 test/fixtures/create-sharded-directory.ts create mode 100644 test/fixtures/create-subsharded-directory.ts create mode 100644 test/fixtures/print-tree.ts create mode 100644 test/ls.spec.ts create mode 100644 test/mkdir.spec.ts create mode 100644 test/rm.spec.ts create mode 100644 test/stat.spec.ts create mode 100644 test/touch.spec.ts create mode 100644 tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..081b272a --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,143 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ github.token }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7ad9e674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md index ffff3cb3..dc448fc9 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,26 @@ +

+ + Helia logo + +

+ # @helia/unixfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/js-test-and-release.yml?query=branch%3Amain) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) > A Helia-compatible wrapper for UnixFS ## Table of contents -- [Install](#install) - - [Browser ` ``` +

+ + Helia logo + +

+ +# @helia/unixfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + ## API Docs -- +- ## License @@ -42,7 +62,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/package.json b/package.json new file mode 100644 index 00000000..eb974593 --- /dev/null +++ b/package.json @@ -0,0 +1,167 @@ +{ + "name": "@helia/unixfs", + "version": "0.0.0", + "description": "A Helia-compatible wrapper for UnixFS", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-unixfs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-unixfs.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-unixfs/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "next", + "@ipld/dag-pb": "^4.0.0", + "@libp2p/interfaces": "^3.3.1", + "@libp2p/logger": "^2.0.5", + "@multiformats/murmur3": "^2.1.2", + "hamt-sharding": "^3.0.2", + "interface-blockstore": "^4.0.1", + "ipfs-unixfs": "^11.0.0", + "ipfs-unixfs-exporter": "^12.0.0", + "ipfs-unixfs-importer": "^14.0.0", + "it-last": "^2.0.0", + "it-pipe": "^2.0.5", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1" + }, + "devDependencies": { + "aegir": "^38.1.0", + "blockstore-core": "^3.0.0", + "delay": "^5.0.0", + "it-all": "^2.0.0", + "it-drain": "^2.0.0", + "it-first": "^2.0.0", + "it-to-buffer": "^3.0.0", + "uint8arrays": "^4.0.3" + } +} diff --git a/src/commands/cat.ts b/src/commands/cat.ts new file mode 100644 index 00000000..2c6d1bd0 --- /dev/null +++ b/src/commands/cat.ts @@ -0,0 +1,32 @@ +import { exporter } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' +import type { CatOptions } from '../index.js' +import { resolve } from './utils/resolve.js' +import mergeOpts from 'merge-options' +import type { Blockstore } from 'interface-blockstore' +import { NoContentError, NotAFileError } from './utils/errors.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) + +const defaultOptions: CatOptions = { + +} + +export async function * cat (cid: CID, blockstore: Blockstore, options: Partial = {}): AsyncIterable { + const opts: CatOptions = mergeOptions(defaultOptions, options) + const resolved = await resolve(cid, opts.path, blockstore, opts) + const result = await exporter(resolved.cid, blockstore, opts) + + if (result.type !== 'file' && result.type !== 'raw') { + throw new NotAFileError() + } + + if (result.content == null) { + throw new NoContentError() + } + + yield * result.content({ + offset: opts.offset, + length: opts.length + }) +} diff --git a/src/commands/chmod.ts b/src/commands/chmod.ts new file mode 100644 index 00000000..2f3f6105 --- /dev/null +++ b/src/commands/chmod.ts @@ -0,0 +1,138 @@ +import { recursive } from 'ipfs-unixfs-exporter' +import { CID } from 'multiformats/cid' +import type { ChmodOptions } from '../index.js' +import mergeOpts from 'merge-options' +import { logger } from '@libp2p/logger' +import { UnixFS } from 'ipfs-unixfs' +import { pipe } from 'it-pipe' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' +import * as dagPB from '@ipld/dag-pb' +import type { PBNode, PBLink } from '@ipld/dag-pb' +import { importer } from 'ipfs-unixfs-importer' +import { persist } from './utils/persist.js' +import type { Blockstore } from 'interface-blockstore' +import last from 'it-last' +import { sha256 } from 'multiformats/hashes/sha2' +import { resolve, updatePathCids } from './utils/resolve.js' +import * as raw from 'multiformats/codecs/raw' +import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) +const log = logger('helia:unixfs:chmod') + +const defaultOptions: ChmodOptions = { + recursive: false, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES +} + +export async function chmod (cid: CID, mode: number, blockstore: Blockstore, options: Partial = {}): Promise { + const opts: ChmodOptions = mergeOptions(defaultOptions, options) + const resolved = await resolve(cid, opts.path, blockstore, options) + + log('chmod %c %d', resolved.cid, mode) + + if (opts.recursive) { + // recursively export from root CID, change perms of each entry then reimport + // but do not reimport files, only manipulate dag-pb nodes + const root = await pipe( + async function * () { + for await (const entry of recursive(resolved.cid, blockstore)) { + let metadata: UnixFS + let links: PBLink[] = [] + + if (entry.type === 'raw') { + // convert to UnixFS + metadata = new UnixFS({ type: 'file', data: entry.node }) + } else if (entry.type === 'file' || entry.type === 'directory') { + metadata = entry.unixfs + links = entry.node.Links + } else { + throw new NotUnixFSError() + } + + metadata.mode = mode + + const node = { + Data: metadata.marshal(), + Links: links + } + + yield { + path: entry.path, + content: node + } + } + }, + // @ts-expect-error we account for the incompatible source type with our custom dag builder below + (source) => importer(source, blockstore, { + ...opts, + pin: false, + dagBuilder: async function * (source, block) { + for await (const entry of source) { + yield async function () { + // @ts-expect-error cannot derive type + const node: PBNode = entry.content + + const buf = dagPB.encode(node) + const updatedCid = await persist(buf, block, { + ...opts, + cidVersion: cid.version + }) + + if (node.Data == null) { + throw new InvalidPBNodeError(`${updatedCid} had no data`) + } + + const unixfs = UnixFS.unmarshal(node.Data) + + return { + cid: updatedCid, + size: buf.length, + path: entry.path, + unixfs + } + } + } + } + }), + async (nodes) => await last(nodes) + ) + + if (root == null) { + throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`) + } + + return await updatePathCids(root.cid, resolved, blockstore, opts) + } + + const block = await blockstore.get(resolved.cid) + let metadata: UnixFS + let links: PBLink[] = [] + + if (resolved.cid.code === raw.code) { + // convert to UnixFS + metadata = new UnixFS({ type: 'file', data: block }) + } else { + const node = dagPB.decode(block) + + if (node.Data == null) { + throw new InvalidPBNodeError(`${resolved.cid.toString()} had no data`) + } + + links = node.Links + metadata = UnixFS.unmarshal(node.Data) + } + + metadata.mode = mode + const updatedBlock = dagPB.encode({ + Data: metadata.marshal(), + Links: links + }) + + const hash = await sha256.digest(updatedBlock) + const updatedCid = CID.create(resolved.cid.version, dagPB.code, hash) + + await blockstore.put(updatedCid, updatedBlock) + + return await updatePathCids(updatedCid, resolved, blockstore, opts) +} diff --git a/src/commands/cp.ts b/src/commands/cp.ts new file mode 100644 index 00000000..4ca837fb --- /dev/null +++ b/src/commands/cp.ts @@ -0,0 +1,44 @@ +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' +import type { CpOptions } from '../index.js' +import mergeOpts from 'merge-options' +import { logger } from '@libp2p/logger' +import { addLink } from './utils/add-link.js' +import { cidToPBLink } from './utils/cid-to-pblink.js' +import { cidToDirectory } from './utils/cid-to-directory.js' +import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' +import { InvalidParametersError } from './utils/errors.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) +const log = logger('helia:unixfs:cp') + +const defaultOptions: CpOptions = { + force: false, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES +} + +export async function cp (source: CID, target: CID, name: string, blockstore: Blockstore, options: Partial = {}): Promise { + const opts: CpOptions = mergeOptions(defaultOptions, options) + + if (name.includes('/')) { + throw new InvalidParametersError('Name must not have slashes') + } + + const [ + directory, + pblink + ] = await Promise.all([ + cidToDirectory(target, blockstore, opts), + cidToPBLink(source, name, blockstore, opts) + ]) + + log('Adding %c as "%s" to %c', source, name, target) + + const result = await addLink(directory, pblink, blockstore, { + allowOverwriting: opts.force, + cidVersion: target.version, + ...opts + }) + + return result.cid +} diff --git a/src/commands/ls.ts b/src/commands/ls.ts new file mode 100644 index 00000000..ca52acfd --- /dev/null +++ b/src/commands/ls.ts @@ -0,0 +1,37 @@ +import { exporter, UnixFSEntry } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' +import type { LsOptions } from '../index.js' +import { resolve } from './utils/resolve.js' +import mergeOpts from 'merge-options' +import type { Blockstore } from 'interface-blockstore' +import { NoContentError, NotADirectoryError } from './utils/errors.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) + +const defaultOptions: LsOptions = { + +} + +export async function * ls (cid: CID, blockstore: Blockstore, options: Partial = {}): AsyncIterable { + const opts: LsOptions = mergeOptions(defaultOptions, options) + const resolved = await resolve(cid, opts.path, blockstore, opts) + const result = await exporter(resolved.cid, blockstore) + + if (result.type === 'file' || result.type === 'raw') { + yield result + return + } + + if (result.content == null) { + throw new NoContentError() + } + + if (result.type !== 'directory') { + throw new NotADirectoryError() + } + + yield * result.content({ + offset: options.offset, + length: options.length + }) +} diff --git a/src/commands/mkdir.ts b/src/commands/mkdir.ts new file mode 100644 index 00000000..6b96905b --- /dev/null +++ b/src/commands/mkdir.ts @@ -0,0 +1,73 @@ +import { CID } from 'multiformats/cid' +import mergeOpts from 'merge-options' +import { logger } from '@libp2p/logger' +import type { MkdirOptions } from '../index.js' +import * as dagPB from '@ipld/dag-pb' +import { addLink } from './utils/add-link.js' +import type { Blockstore } from 'interface-blockstore' +import { UnixFS } from 'ipfs-unixfs' +import { sha256 } from 'multiformats/hashes/sha2' +import { exporter } from 'ipfs-unixfs-exporter' +import { cidToDirectory } from './utils/cid-to-directory.js' +import { cidToPBLink } from './utils/cid-to-pblink.js' +import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' +import { InvalidParametersError, NotADirectoryError } from './utils/errors.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) +const log = logger('helia:unixfs:mkdir') + +const defaultOptions: MkdirOptions = { + cidVersion: 1, + force: false, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES +} + +export async function mkdir (parentCid: CID, dirname: string, blockstore: Blockstore, options: Partial = {}): Promise { + const opts: MkdirOptions = mergeOptions(defaultOptions, options) + + if (dirname.includes('/')) { + throw new InvalidParametersError('Path must not have slashes') + } + + const entry = await exporter(parentCid, blockstore, options) + + if (entry.type !== 'directory') { + throw new NotADirectoryError(`${parentCid.toString()} was not a UnixFS directory`) + } + + log('creating %s', dirname) + + const metadata = new UnixFS({ + type: 'directory', + mode: opts.mode, + mtime: opts.mtime + }) + + // Persist the new parent PBNode + const node = { + Data: metadata.marshal(), + Links: [] + } + const buf = dagPB.encode(node) + const hash = await sha256.digest(buf) + const emptyDirCid = CID.create(opts.cidVersion, dagPB.code, hash) + + await blockstore.put(emptyDirCid, buf) + + const [ + directory, + pblink + ] = await Promise.all([ + cidToDirectory(parentCid, blockstore, opts), + cidToPBLink(emptyDirCid, dirname, blockstore, opts) + ]) + + log('adding empty dir called %s to %c', dirname, parentCid) + + const result = await addLink(directory, pblink, blockstore, { + ...opts, + allowOverwriting: opts.force + }) + + return result.cid +} diff --git a/src/commands/rm.ts b/src/commands/rm.ts new file mode 100644 index 00000000..28f75325 --- /dev/null +++ b/src/commands/rm.ts @@ -0,0 +1,35 @@ +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' +import type { RmOptions } from '../index.js' +import mergeOpts from 'merge-options' +import { logger } from '@libp2p/logger' +import { removeLink } from './utils/remove-link.js' +import { cidToDirectory } from './utils/cid-to-directory.js' +import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' +import { InvalidParametersError } from './utils/errors.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) +const log = logger('helia:unixfs:rm') + +const defaultOptions: RmOptions = { + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES +} + +export async function rm (target: CID, name: string, blockstore: Blockstore, options: Partial = {}): Promise { + const opts: RmOptions = mergeOptions(defaultOptions, options) + + if (name.includes('/')) { + throw new InvalidParametersError('Name must not have slashes') + } + + const directory = await cidToDirectory(target, blockstore, opts) + + log('Removing %s from %c', name, target) + + const result = await removeLink(directory, name, blockstore, { + ...opts, + cidVersion: target.version + }) + + return result.cid +} diff --git a/src/commands/stat.ts b/src/commands/stat.ts new file mode 100644 index 00000000..9564ecad --- /dev/null +++ b/src/commands/stat.ts @@ -0,0 +1,142 @@ +import { exporter } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' +import type { StatOptions, UnixFSStats } from '../index.js' +import mergeOpts from 'merge-options' +import { logger } from '@libp2p/logger' +import { UnixFS } from 'ipfs-unixfs' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' +import * as dagPb from '@ipld/dag-pb' +import type { AbortOptions } from '@libp2p/interfaces' +import type { Mtime } from 'ipfs-unixfs' +import { resolve } from './utils/resolve.js' +import * as raw from 'multiformats/codecs/raw' +import type { Blockstore } from 'interface-blockstore' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) +const log = logger('helia:unixfs:stat') + +const defaultOptions: StatOptions = { + +} + +export async function stat (cid: CID, blockstore: Blockstore, options: Partial = {}): Promise { + const opts: StatOptions = mergeOptions(defaultOptions, options) + const resolved = await resolve(cid, options.path, blockstore, opts) + + log('stat %c', resolved.cid) + + const result = await exporter(resolved.cid, blockstore, opts) + + if (result.type !== 'file' && result.type !== 'directory' && result.type !== 'raw') { + throw new NotUnixFSError() + } + + let fileSize: bigint = 0n + let dagSize: bigint = 0n + let localFileSize: bigint = 0n + let localDagSize: bigint = 0n + let blocks: number = 0 + let mode: number | undefined + let mtime: Mtime | undefined + const type = result.type + let unixfs: UnixFS | undefined + + if (result.type === 'raw') { + fileSize = BigInt(result.node.byteLength) + dagSize = BigInt(result.node.byteLength) + localFileSize = BigInt(result.node.byteLength) + localDagSize = BigInt(result.node.byteLength) + blocks = 1 + } + + if (result.type === 'directory') { + fileSize = 0n + dagSize = BigInt(result.unixfs.marshal().byteLength) + localFileSize = 0n + localDagSize = dagSize + blocks = 1 + mode = result.unixfs.mode + mtime = result.unixfs.mtime + unixfs = result.unixfs + } + + if (result.type === 'file') { + const results = await inspectDag(resolved.cid, blockstore, opts) + + fileSize = result.unixfs.fileSize() + dagSize = BigInt((result.node.Data?.byteLength ?? 0) + result.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0)) + localFileSize = BigInt(results.localFileSize) + localDagSize = BigInt(results.localDagSize) + blocks = results.blocks + mode = result.unixfs.mode + mtime = result.unixfs.mtime + unixfs = result.unixfs + } + + return { + cid: resolved.cid, + mode, + mtime, + fileSize, + dagSize, + localFileSize, + localDagSize, + blocks, + type, + unixfs + } +} + +interface InspectDagResults { + localFileSize: number + localDagSize: number + blocks: number +} + +async function inspectDag (cid: CID, blockstore: Blockstore, options: AbortOptions): Promise { + const results = { + localFileSize: 0, + localDagSize: 0, + blocks: 0 + } + + if (await blockstore.has(cid, options)) { + const block = await blockstore.get(cid, options) + results.blocks++ + results.localDagSize += block.byteLength + + if (cid.code === raw.code) { + results.localFileSize += block.byteLength + } else if (cid.code === dagPb.code) { + const pbNode = dagPb.decode(block) + + if (pbNode.Links.length > 0) { + // intermediate node + for (const link of pbNode.Links) { + const linkResult = await inspectDag(link.Hash, blockstore, options) + + results.localFileSize += linkResult.localFileSize + results.localDagSize += linkResult.localDagSize + results.blocks += linkResult.blocks + } + } else { + // leaf node + if (pbNode.Data == null) { + throw new InvalidPBNodeError(`PBNode ${cid.toString()} had no data`) + } + + const unixfs = UnixFS.unmarshal(pbNode.Data) + + if (unixfs.data == null) { + throw new InvalidPBNodeError(`UnixFS node ${cid.toString()} had no data`) + } + + results.localFileSize += unixfs.data.byteLength ?? 0 + } + } else { + throw new UnknownError(`${cid.toString()} was neither DAG_PB nor RAW`) + } + } + + return results +} diff --git a/src/commands/touch.ts b/src/commands/touch.ts new file mode 100644 index 00000000..379778fb --- /dev/null +++ b/src/commands/touch.ts @@ -0,0 +1,141 @@ +import { recursive } from 'ipfs-unixfs-exporter' +import { CID } from 'multiformats/cid' +import type { TouchOptions } from '../index.js' +import mergeOpts from 'merge-options' +import { logger } from '@libp2p/logger' +import { UnixFS } from 'ipfs-unixfs' +import { pipe } from 'it-pipe' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' +import * as dagPB from '@ipld/dag-pb' +import type { PBNode, PBLink } from '@ipld/dag-pb' +import { importer } from 'ipfs-unixfs-importer' +import { persist } from './utils/persist.js' +import type { Blockstore } from 'interface-blockstore' +import last from 'it-last' +import { sha256 } from 'multiformats/hashes/sha2' +import { resolve, updatePathCids } from './utils/resolve.js' +import * as raw from 'multiformats/codecs/raw' +import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' + +const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) +const log = logger('helia:unixfs:touch') + +const defaultOptions: TouchOptions = { + recursive: false, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES +} + +export async function touch (cid: CID, blockstore: Blockstore, options: Partial = {}): Promise { + const opts: TouchOptions = mergeOptions(defaultOptions, options) + const resolved = await resolve(cid, opts.path, blockstore, opts) + const mtime = opts.mtime ?? { + secs: BigInt(Math.round(Date.now() / 1000)), + nsecs: 0 + } + + log('touch %c %o', resolved.cid, mtime) + + if (opts.recursive) { + // recursively export from root CID, change perms of each entry then reimport + // but do not reimport files, only manipulate dag-pb nodes + const root = await pipe( + async function * () { + for await (const entry of recursive(resolved.cid, blockstore)) { + let metadata: UnixFS + let links: PBLink[] + + if (entry.type === 'raw') { + metadata = new UnixFS({ data: entry.node }) + links = [] + } else if (entry.type === 'file' || entry.type === 'directory') { + metadata = entry.unixfs + links = entry.node.Links + } else { + throw new NotUnixFSError() + } + + metadata.mtime = mtime + + const node = { + Data: metadata.marshal(), + Links: links + } + + yield { + path: entry.path, + content: node + } + } + }, + // @ts-expect-error we account for the incompatible source type with our custom dag builder below + (source) => importer(source, blockstore, { + ...opts, + pin: false, + dagBuilder: async function * (source, block) { + for await (const entry of source) { + yield async function () { + // @ts-expect-error cannot derive type + const node: PBNode = entry.content + + const buf = dagPB.encode(node) + const updatedCid = await persist(buf, block, { + ...opts, + cidVersion: cid.version + }) + + if (node.Data == null) { + throw new InvalidPBNodeError(`${updatedCid} had no data`) + } + + const unixfs = UnixFS.unmarshal(node.Data) + + return { + cid: updatedCid, + size: buf.length, + path: entry.path, + unixfs + } + } + } + } + }), + async (nodes) => await last(nodes) + ) + + if (root == null) { + throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`) + } + + return await updatePathCids(root.cid, resolved, blockstore, opts) + } + + const block = await blockstore.get(resolved.cid) + let metadata: UnixFS + let links: PBLink[] = [] + + if (resolved.cid.code === raw.code) { + metadata = new UnixFS({ data: block }) + } else { + const node = dagPB.decode(block) + links = node.Links + + if (node.Data == null) { + throw new InvalidPBNodeError(`${resolved.cid.toString()} had no data`) + } + + metadata = UnixFS.unmarshal(node.Data) + } + + metadata.mtime = mtime + const updatedBlock = dagPB.encode({ + Data: metadata.marshal(), + Links: links + }) + + const hash = await sha256.digest(updatedBlock) + const updatedCid = CID.create(resolved.cid.version, dagPB.code, hash) + + await blockstore.put(updatedCid, updatedBlock) + + return await updatePathCids(updatedCid, resolved, blockstore, opts) +} diff --git a/src/commands/utils/add-link.ts b/src/commands/utils/add-link.ts new file mode 100644 index 00000000..b8f15101 --- /dev/null +++ b/src/commands/utils/add-link.ts @@ -0,0 +1,320 @@ +import * as dagPB from '@ipld/dag-pb' +import { CID, Version } from 'multiformats/cid' +import { logger } from '@libp2p/logger' +import { UnixFS } from 'ipfs-unixfs' +import { DirSharded } from './dir-sharded.js' +import { + updateHamtDirectory, + recreateHamtLevel, + recreateInitialHamtLevel, + createShard, + toPrefix, + addLinksToHamtBucket +} from './hamt-utils.js' +import last from 'it-last' +import type { PBNode, PBLink } from '@ipld/dag-pb/interface' +import { sha256 } from 'multiformats/hashes/sha2' +import type { Bucket } from 'hamt-sharding' +import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from './errors.js' +import type { ImportResult } from 'ipfs-unixfs-importer' +import type { AbortOptions } from '@libp2p/interfaces' +import type { Directory } from './cid-to-directory.js' +import type { Blockstore } from 'interface-blockstore' +import { isOverShardThreshold } from './is-over-shard-threshold.js' + +const log = logger('helia:unixfs:components:utils:add-link') + +export interface AddLinkResult { + node: PBNode + cid: CID + size: number +} + +export interface AddLinkOptions extends AbortOptions { + allowOverwriting: boolean + shardSplitThresholdBytes: number + cidVersion: Version +} + +export async function addLink (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise { + if (parent.node.Data == null) { + throw new InvalidParametersError('Invalid parent passed to addLink') + } + + const meta = UnixFS.unmarshal(parent.node.Data) + + if (meta.type === 'hamt-sharded-directory') { + log('adding link to sharded directory') + + return await addToShardedDirectory(parent, child, blockstore, options) + } + + log(`adding ${child.Name} (${child.Hash}) to regular directory`) + + const result = await addToDirectory(parent, child, blockstore, options) + + if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) { + log('converting directory to sharded directory') + + const converted = await convertToShardedDirectory(result, blockstore) + result.cid = converted.cid + result.node = dagPB.decode(await blockstore.get(converted.cid)) + } + + return result +} + +const convertToShardedDirectory = async (parent: Directory, blockstore: Blockstore): Promise => { + if (parent.node.Data == null) { + throw new InvalidParametersError('Invalid parent passed to convertToShardedDirectory') + } + + const unixfs = UnixFS.unmarshal(parent.node.Data) + + const result = await createShard(blockstore, parent.node.Links.map(link => ({ + name: (link.Name ?? ''), + size: BigInt(link.Tsize ?? 0), + cid: link.Hash + })), { + mode: unixfs.mode, + mtime: unixfs.mtime, + cidVersion: parent.cid.version + }) + + log(`Converted directory to sharded directory ${result.cid}`) + + return result +} + +const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Blockstore, options: AddLinkOptions): Promise => { + // Remove existing link if it exists + const parentLinks = parent.node.Links.filter((link) => { + const matches = link.Name === child.Name + + if (matches && !options.allowOverwriting) { + throw new AlreadyExistsError() + } + + return !matches + }) + parentLinks.push(child) + + if (parent.node.Data == null) { + throw new InvalidPBNodeError('Parent node with no data passed to addToDirectory') + } + + const node = UnixFS.unmarshal(parent.node.Data) + + let data + if (node.mtime != null) { + // Update mtime if previously set + const ms = Date.now() + const secs = Math.floor(ms / 1000) + + node.mtime = { + secs: BigInt(secs), + nsecs: (ms - (secs * 1000)) * 1000 + } + + data = node.marshal() + } else { + data = parent.node.Data + } + parent.node = dagPB.prepare({ + Data: data, + Links: parentLinks + }) + + // Persist the new parent PbNode + const buf = dagPB.encode(parent.node) + const hash = await sha256.digest(buf) + const cid = CID.create(parent.cid.version, dagPB.code, hash) + + await blockstore.put(cid, buf) + + return { + node: parent.node, + cid, + size: buf.length + } +} + +const addToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise => { + const { + shard, path + } = await addFileToShardedDirectory(parent, child, blockstore, options) + const result = await last(shard.flush(blockstore)) + + if (result == null) { + throw new Error('No result from flushing shard') + } + + const block = await blockstore.get(result.cid) + const node = dagPB.decode(block) + + // we have written out the shard, but only one sub-shard will have been written so replace it in the original shard + const parentLinks = parent.node.Links.filter((link) => { + return (link.Name ?? '').substring(0, 2) !== path[0].prefix + }) + + const newLink = node.Links + .find(link => (link.Name ?? '').substring(0, 2) === path[0].prefix) + + if (newLink == null) { + throw new Error(`No link found with prefix ${path[0].prefix}`) + } + + parentLinks.push(newLink) + + return await updateHamtDirectory({ + Data: parent.node.Data, + Links: parentLinks + }, blockstore, path[0].bucket, options) +} + +const addFileToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise<{ shard: DirSharded, path: BucketPath[] }> => { + if (parent.node.Data == null) { + throw new InvalidPBNodeError('Parent node with no data passed to addFileToShardedDirectory') + } + + // start at the root bucket and descend, loading nodes as we go + const rootBucket = await recreateInitialHamtLevel(parent.node.Links) + const node = UnixFS.unmarshal(parent.node.Data) + + const shard = new DirSharded({ + root: true, + dir: true, + parent: undefined, + parentKey: undefined, + path: '', + dirty: true, + flat: false, + mode: node.mode + }, { + ...options, + cidVersion: parent.cid.version + }) + shard._bucket = rootBucket + + if (node.mtime != null) { + // update mtime if previously set + shard.mtime = { + secs: BigInt(Math.round(Date.now() / 1000)) + } + } + + // load subshards until the bucket & position no longer changes + const position = await rootBucket._findNewBucketAndPos(child.Name) + const path = toBucketPath(position) + path[0].node = parent.node + let index = 0 + + while (index < path.length) { + const segment = path[index] + index++ + const node = segment.node + + if (node == null) { + throw new Error('Segment had no node') + } + + const link = node.Links + .find(link => (link.Name ?? '').substring(0, 2) === segment.prefix) + + if (link == null) { + // prefix is new, file will be added to the current bucket + log(`Link ${segment.prefix}${child.Name} will be added`) + index = path.length + + break + } + + if (link.Name === `${segment.prefix}${child.Name}`) { + if (!options.allowOverwriting) { + throw new AlreadyExistsError() + } + + // file already existed, file will be added to the current bucket + log(`Link ${segment.prefix}${child.Name} will be replaced`) + index = path.length + + break + } + + if ((link.Name ?? '').length > 2) { + // another file had the same prefix, will be replaced with a subshard + log(`Link ${link.Name} ${link.Hash} will be replaced with a subshard`) + index = path.length + + break + } + + // load sub-shard + log(`Found subshard ${segment.prefix}`) + const block = await blockstore.get(link.Hash) + const subShard = dagPB.decode(block) + + // subshard hasn't been loaded, descend to the next level of the HAMT + if (path[index] == null) { + log(`Loaded new subshard ${segment.prefix}`) + await recreateHamtLevel(blockstore, subShard.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options) + + const position = await rootBucket._findNewBucketAndPos(child.Name) + + path.push({ + bucket: position.bucket, + prefix: toPrefix(position.pos), + node: subShard + }) + + break + } + + const nextSegment = path[index] + + // add next levels worth of links to bucket + await addLinksToHamtBucket(blockstore, subShard.Links, nextSegment.bucket, rootBucket, options) + + nextSegment.node = subShard + } + + // finally add the new file into the shard + await shard._bucket.put(child.Name, { + size: BigInt(child.Tsize), + cid: child.Hash + }) + + return { + shard, path + } +} + +export interface BucketPath { + bucket: Bucket + prefix: string + node?: PBNode +} + +const toBucketPath = (position: { pos: number, bucket: Bucket }): BucketPath[] => { + const path = [{ + bucket: position.bucket, + prefix: toPrefix(position.pos) + }] + + let bucket = position.bucket._parent + let positionInBucket = position.bucket._posAtParent + + while (bucket != null) { + path.push({ + bucket, + prefix: toPrefix(positionInBucket) + }) + + positionInBucket = bucket._posAtParent + bucket = bucket._parent + } + + path.reverse() + + return path +} diff --git a/src/commands/utils/cid-to-directory.ts b/src/commands/utils/cid-to-directory.ts new file mode 100644 index 00000000..8d01526b --- /dev/null +++ b/src/commands/utils/cid-to-directory.ts @@ -0,0 +1,23 @@ +import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' +import type { PBNode } from '@ipld/dag-pb' +import type { Blockstore } from 'interface-blockstore' +import { NotADirectoryError } from './errors.js' + +export interface Directory { + cid: CID + node: PBNode +} + +export async function cidToDirectory (cid: CID, blockstore: Blockstore, options: ExporterOptions = {}): Promise { + const entry = await exporter(cid, blockstore, options) + + if (entry.type !== 'directory') { + throw new NotADirectoryError(`${cid.toString()} was not a UnixFS directory`) + } + + return { + cid, + node: entry.node + } +} diff --git a/src/commands/utils/cid-to-pblink.ts b/src/commands/utils/cid-to-pblink.ts new file mode 100644 index 00000000..e819e92c --- /dev/null +++ b/src/commands/utils/cid-to-pblink.ts @@ -0,0 +1,26 @@ +import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' +import { NotUnixFSError } from './errors.js' +import * as dagPb from '@ipld/dag-pb' +import type { PBNode, PBLink } from '@ipld/dag-pb' +import type { Blockstore } from 'interface-blockstore' + +export async function cidToPBLink (cid: CID, name: string, blockstore: Blockstore, options?: ExporterOptions): Promise> { + const sourceEntry = await exporter(cid, blockstore, options) + + if (sourceEntry.type !== 'directory' && sourceEntry.type !== 'file' && sourceEntry.type !== 'raw') { + throw new NotUnixFSError(`${cid.toString()} was not a UnixFS node`) + } + + return { + Name: name, + Tsize: sourceEntry.node instanceof Uint8Array ? sourceEntry.node.byteLength : dagNodeTsize(sourceEntry.node), + Hash: cid + } +} + +function dagNodeTsize (node: PBNode): number { + const linkSizes = node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), 0) + + return dagPb.encode(node).byteLength + linkSizes +} diff --git a/src/commands/utils/constants.ts b/src/commands/utils/constants.ts new file mode 100644 index 00000000..919f10d9 --- /dev/null +++ b/src/commands/utils/constants.ts @@ -0,0 +1,2 @@ + +export const SHARD_SPLIT_THRESHOLD_BYTES = 262144 diff --git a/src/commands/utils/dir-sharded.ts b/src/commands/utils/dir-sharded.ts new file mode 100644 index 00000000..2f896772 --- /dev/null +++ b/src/commands/utils/dir-sharded.ts @@ -0,0 +1,318 @@ +import { encode, PBLink, prepare } from '@ipld/dag-pb' +import { UnixFS } from 'ipfs-unixfs' +import { persist, PersistOptions } from './persist.js' +import { createHAMT, Bucket, BucketChild } from 'hamt-sharding' +import { + hamtHashCode, + hamtHashFn +} from './hamt-constants.js' +import { CID } from 'multiformats/cid' +import type { Mtime } from 'ipfs-unixfs' +import type { Blockstore } from 'interface-blockstore' + +interface InProgressImportResult extends ImportResult { + single?: boolean + originalPath?: string +} + +interface ImportResult { + cid: CID + size: bigint + path?: string + unixfs?: UnixFS +} + +interface DirProps { + root: boolean + dir: boolean + path: string + dirty: boolean + flat: boolean + parent?: Dir + parentKey?: string + unixfs?: UnixFS + mode?: number + mtime?: Mtime +} + +abstract class Dir { + public options: PersistOptions + public root: boolean + public dir: boolean + public path: string + public dirty: boolean + public flat: boolean + public parent?: Dir + public parentKey?: string + public unixfs?: UnixFS + public mode?: number + public mtime?: Mtime + public cid?: CID + public size?: number + public nodeSize?: number + + constructor (props: DirProps, options: PersistOptions) { + this.options = options ?? {} + + this.root = props.root + this.dir = props.dir + this.path = props.path + this.dirty = props.dirty + this.flat = props.flat + this.parent = props.parent + this.parentKey = props.parentKey + this.unixfs = props.unixfs + this.mode = props.mode + this.mtime = props.mtime + } + + abstract put (name: string, value: InProgressImportResult | Dir): Promise + abstract get (name: string): Promise + abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }> + abstract flush (blockstore: Blockstore): AsyncGenerator + abstract estimateNodeSize (): number + abstract childCount (): number +} + +export class DirSharded extends Dir { + public _bucket: Bucket + + constructor (props: DirProps, options: PersistOptions) { + super(props, options) + + this._bucket = createHAMT({ + hashFn: hamtHashFn, + bits: 8 + }) + } + + async put (name: string, value: InProgressImportResult | Dir): Promise { + this.cid = undefined + this.size = undefined + this.nodeSize = undefined + + await this._bucket.put(name, value) + } + + async get (name: string): Promise { + return await this._bucket.get(name) + } + + childCount (): number { + return this._bucket.leafCount() + } + + directChildrenCount (): number { + return this._bucket.childrenCount() + } + + onlyChild (): Bucket | BucketChild { + return this._bucket.onlyChild() + } + + async * eachChildSeries (): AsyncGenerator<{ key: string, child: InProgressImportResult | Dir }> { + for await (const { key, value } of this._bucket.eachLeafSeries()) { + yield { + key, + child: value + } + } + } + + estimateNodeSize (): number { + if (this.nodeSize !== undefined) { + return this.nodeSize + } + + this.nodeSize = calculateSize(this._bucket, this, this.options) + + return this.nodeSize + } + + async * flush (blockstore: Blockstore): AsyncGenerator { + for await (const entry of flush(this._bucket, blockstore, this, this.options)) { + yield { + ...entry, + path: this.path + } + } + } +} + +async function * flush (bucket: Bucket, blockstore: Blockstore, shardRoot: DirSharded | null, options: PersistOptions): AsyncIterable { + const children = bucket._children + const links: PBLink[] = [] + let childrenSize = 0n + + for (let i = 0; i < children.length; i++) { + const child = children.get(i) + + if (child == null) { + continue + } + + const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0') + + if (child instanceof Bucket) { + let shard + + for await (const subShard of flush(child, blockstore, null, options)) { + shard = subShard + } + + if (shard == null) { + throw new Error('Could not flush sharded directory, no subshard found') + } + + links.push({ + Name: labelPrefix, + Tsize: Number(shard.size), + Hash: shard.cid + }) + childrenSize += shard.size + } else if (isDir(child.value)) { + const dir = child.value + let flushedDir: ImportResult | undefined + + for await (const entry of dir.flush(blockstore)) { + flushedDir = entry + + yield flushedDir + } + + if (flushedDir == null) { + throw new Error('Did not flush dir') + } + + const label = labelPrefix + child.key + links.push({ + Name: label, + Tsize: Number(flushedDir.size), + Hash: flushedDir.cid + }) + + childrenSize += flushedDir.size + } else { + const value = child.value + + if (value.cid == null) { + continue + } + + const label = labelPrefix + child.key + const size = value.size + + links.push({ + Name: label, + Tsize: Number(size), + Hash: value.cid + }) + childrenSize += BigInt(size ?? 0) + } + } + + // go-ipfs uses little endian, that's why we have to + // reverse the bit field before storing it + const data = Uint8Array.from(children.bitField().reverse()) + const dir = new UnixFS({ + type: 'hamt-sharded-directory', + data, + fanout: BigInt(bucket.tableSize()), + hashType: hamtHashCode, + mtime: shardRoot?.mtime, + mode: shardRoot?.mode + }) + + const node = { + Data: dir.marshal(), + Links: links + } + const buffer = encode(prepare(node)) + const cid = await persist(buffer, blockstore, options) + const size = BigInt(buffer.byteLength) + childrenSize + + yield { + cid, + unixfs: dir, + size + } +} + +function isDir (obj: any): obj is Dir { + return typeof obj.flush === 'function' +} + +function calculateSize (bucket: Bucket, shardRoot: DirSharded | null, options: PersistOptions): number { + const children = bucket._children + const links: PBLink[] = [] + + for (let i = 0; i < children.length; i++) { + const child = children.get(i) + + if (child == null) { + continue + } + + const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0') + + if (child instanceof Bucket) { + const size = calculateSize(child, null, options) + + links.push({ + Name: labelPrefix, + Tsize: Number(size), + Hash: options.cidVersion === 0 ? CID_V0 : CID_V1 + }) + } else if (typeof child.value.flush === 'function') { + const dir = child.value + const size = dir.nodeSize() + + links.push({ + Name: labelPrefix + child.key, + Tsize: Number(size), + Hash: options.cidVersion === 0 ? CID_V0 : CID_V1 + }) + } else { + const value = child.value + + if (value.cid == null) { + continue + } + + const label = labelPrefix + child.key + const size = value.size + + links.push({ + Name: label, + Tsize: Number(size), + Hash: value.cid + }) + } + } + + // go-ipfs uses little endian, that's why we have to + // reverse the bit field before storing it + const data = Uint8Array.from(children.bitField().reverse()) + const dir = new UnixFS({ + type: 'hamt-sharded-directory', + data, + fanout: BigInt(bucket.tableSize()), + hashType: hamtHashCode, + mtime: shardRoot?.mtime, + mode: shardRoot?.mode + }) + + const buffer = encode(prepare({ + Data: dir.marshal(), + Links: links + })) + + return buffer.length +} + +// we use these to calculate the node size to use as a check for whether a directory +// should be sharded or not. Since CIDs have a constant length and We're only +// interested in the data length and not the actual content identifier we can use +// any old CID instead of having to hash the data which is expensive. +export const CID_V0 = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') +export const CID_V1 = CID.parse('zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi') diff --git a/src/commands/utils/errors.ts b/src/commands/utils/errors.ts new file mode 100644 index 00000000..22614c6b --- /dev/null +++ b/src/commands/utils/errors.ts @@ -0,0 +1,65 @@ +export abstract class UnixFSError extends Error { + public readonly name: string + public readonly code: string + + constructor (message: string, name: string, code: string) { + super(message) + + this.name = name + this.code = code + } +} + +export class NotUnixFSError extends UnixFSError { + constructor (message = 'not a Unixfs node') { + super(message, 'NotUnixFSError', 'ERR_NOT_UNIXFS') + } +} + +export class InvalidPBNodeError extends UnixFSError { + constructor (message = 'invalid PBNode') { + super(message, 'InvalidPBNodeError', 'ERR_INVALID_PBNODE') + } +} + +export class UnknownError extends UnixFSError { + constructor (message = 'unknown error') { + super(message, 'InvalidPBNodeError', 'ERR_UNKNOWN_ERROR') + } +} + +export class AlreadyExistsError extends UnixFSError { + constructor (message = 'path already exists') { + super(message, 'AlreadyExistsError', 'ERR_ALREADY_EXISTS') + } +} + +export class DoesNotExistError extends UnixFSError { + constructor (message = 'path does not exist') { + super(message, 'DoesNotExistError', 'ERR_DOES_NOT_EXIST') + } +} + +export class NoContentError extends UnixFSError { + constructor (message = 'no content') { + super(message, 'NoContentError', 'ERR_NO_CONTENT') + } +} + +export class NotAFileError extends UnixFSError { + constructor (message = 'not a file') { + super(message, 'NotAFileError', 'ERR_NOT_A_FILE') + } +} + +export class NotADirectoryError extends UnixFSError { + constructor (message = 'not a directory') { + super(message, 'NotADirectoryError', 'ERR_NOT_A_DIRECTORY') + } +} + +export class InvalidParametersError extends UnixFSError { + constructor (message = 'invalid parameters') { + super(message, 'InvalidParametersError', 'ERR_INVALID_PARAMETERS') + } +} diff --git a/src/commands/utils/hamt-constants.ts b/src/commands/utils/hamt-constants.ts new file mode 100644 index 00000000..66d9dcfd --- /dev/null +++ b/src/commands/utils/hamt-constants.ts @@ -0,0 +1,14 @@ +import { murmur3128 } from '@multiformats/murmur3' + +export const hamtHashCode = BigInt(murmur3128.code) +export const hamtBucketBits = 8 + +export async function hamtHashFn (buf: Uint8Array): Promise { + return (await murmur3128.encode(buf)) + // Murmur3 outputs 128 bit but, accidentally, IPFS Go's + // implementation only uses the first 64, so we must do the same + // for parity.. + .subarray(0, 8) + // Invert buffer because that's how Go impl does it + .reverse() +} diff --git a/src/commands/utils/hamt-utils.ts b/src/commands/utils/hamt-utils.ts new file mode 100644 index 00000000..90147ba5 --- /dev/null +++ b/src/commands/utils/hamt-utils.ts @@ -0,0 +1,294 @@ +import * as dagPB from '@ipld/dag-pb' +import { + Bucket, + createHAMT +} from 'hamt-sharding' +import { DirSharded } from './dir-sharded.js' +import { logger } from '@libp2p/logger' +import { UnixFS } from 'ipfs-unixfs' +import last from 'it-last' +import type { CID, Version } from 'multiformats/cid' +import { + hamtHashCode, + hamtHashFn, + hamtBucketBits +} from './hamt-constants.js' +import type { PBLink, PBNode } from '@ipld/dag-pb/interface' +import type { Blockstore } from 'interface-blockstore' +import type { Mtime } from 'ipfs-unixfs' +import type { Directory } from './cid-to-directory.js' +import type { AbortOptions } from '@libp2p/interfaces' +import type { ImportResult } from 'ipfs-unixfs-importer' +import { persist } from './persist.js' + +const log = logger('helia:unixfs:commands:utils:hamt-utils') + +export interface UpdateHamtResult { + node: PBNode + cid: CID + size: number +} + +export interface UpdateHamtDirectoryOptions extends AbortOptions { + cidVersion: Version +} + +export const updateHamtDirectory = async (pbNode: PBNode, blockstore: Blockstore, bucket: Bucket, options: UpdateHamtDirectoryOptions): Promise => { + if (pbNode.Data == null) { + throw new Error('Could not update HAMT directory because parent had no data') + } + + // update parent with new bit field + const node = UnixFS.unmarshal(pbNode.Data) + const dir = new UnixFS({ + type: 'hamt-sharded-directory', + data: Uint8Array.from(bucket._children.bitField().reverse()), + fanout: BigInt(bucket.tableSize()), + hashType: hamtHashCode, + mode: node.mode, + mtime: node.mtime + }) + + const updatedPbNode = { + Data: dir.marshal(), + Links: pbNode.Links + } + + const buf = dagPB.encode(dagPB.prepare(updatedPbNode)) + const cid = await persist(buf, blockstore, options) + + return { + node: updatedPbNode, + cid, + size: pbNode.Links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.byteLength) + } +} + +export const recreateHamtLevel = async (blockstore: Blockstore, links: PBLink[], rootBucket: Bucket, parentBucket: Bucket, positionAtParent: number, options: AbortOptions): Promise> => { + // recreate this level of the HAMT + const bucket = new Bucket({ + hash: rootBucket._options.hash, + bits: rootBucket._options.bits + }, parentBucket, positionAtParent) + parentBucket._putObjectAt(positionAtParent, bucket) + + await addLinksToHamtBucket(blockstore, links, bucket, rootBucket, options) + + return bucket +} + +export const recreateInitialHamtLevel = async (links: PBLink[]): Promise> => { + const bucket = createHAMT({ + hashFn: hamtHashFn, + bits: hamtBucketBits + }) + + // populate sub bucket but do not recurse as we do not want to load the whole shard + await Promise.all( + links.map(async link => { + const linkName = (link.Name ?? '') + + if (linkName.length === 2) { + const pos = parseInt(linkName, 16) + const subBucket = new Bucket({ + hash: bucket._options.hash, + bits: bucket._options.bits + }, bucket, pos) + + bucket._putObjectAt(pos, subBucket) + } + + await bucket.put(linkName.substring(2), { + size: link.Tsize, + cid: link.Hash + }) + }) + ) + + return bucket +} + +export const addLinksToHamtBucket = async (blockstore: Blockstore, links: PBLink[], bucket: Bucket, rootBucket: Bucket, options: AbortOptions): Promise => { + await Promise.all( + links.map(async link => { + const linkName = (link.Name ?? '') + + if (linkName.length === 2) { + log('Populating sub bucket', linkName) + const pos = parseInt(linkName, 16) + const block = await blockstore.get(link.Hash, options) + const node = dagPB.decode(block) + + const subBucket = new Bucket({ + hash: rootBucket._options.hash, + bits: rootBucket._options.bits + }, bucket, pos) + bucket._putObjectAt(pos, subBucket) + + await addLinksToHamtBucket(blockstore, node.Links, subBucket, rootBucket, options) + } + + await rootBucket.put(linkName.substring(2), { + size: link.Tsize, + cid: link.Hash + }) + }) + ) +} + +export const toPrefix = (position: number): string => { + return position + .toString(16) + .toUpperCase() + .padStart(2, '0') + .substring(0, 2) +} + +export interface HamtPathSegment { + bucket?: Bucket + prefix?: string + node?: PBNode + cid?: CID + size?: number +} + +export const generatePath = async (root: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise => { + // start at the root bucket and descend, loading nodes as we go + const rootBucket = await recreateInitialHamtLevel(root.node.Links) + const position = await rootBucket._findNewBucketAndPos(name) + const path: HamtPathSegment[] = [{ + bucket: position.bucket, + prefix: toPrefix(position.pos) + }] + let currentBucket = position.bucket + + while (currentBucket !== rootBucket) { + path.push({ + bucket: currentBucket, + prefix: toPrefix(currentBucket._posAtParent) + }) + + if (currentBucket._parent == null) { + break + } + + currentBucket = currentBucket._parent + } + + // add the root bucket to the path + path.push({ + bucket: rootBucket, + node: root.node + }) + + path.reverse() + + // load PbNode for each path segment + for (let i = 1; i < path.length; i++) { + const segment = path[i] + const previousSegment = path[i - 1] + + if (previousSegment.node == null) { + throw new Error('Could not generate HAMT path') + } + + // find prefix in links + const link = previousSegment.node.Links + .filter(link => (link.Name ?? '').substring(0, 2) === segment.prefix) + .pop() + + // entry was not in shard + if (link == null) { + // reached bottom of tree, file will be added to the current bucket + log(`Link ${segment.prefix}${name} will be added`) + // return path + continue + } + + const linkName = link.Name ?? '' + + // found entry + if (linkName === `${segment.prefix}${name}`) { + log(`Link ${segment.prefix}${name} will be replaced`) + // file already existed, file will be added to the current bucket + // return path + continue + } + + // found subshard + log(`Found subshard ${segment.prefix}`) + const block = await blockstore.get(link.Hash) + const node = dagPB.decode(block) + + // subshard hasn't been loaded, descend to the next level of the HAMT + if (path[i + 1] == null) { + log(`Loaded new subshard ${segment.prefix}`) + + if (segment.bucket == null || segment.prefix == null) { + throw new Error('Shard was invalid') + } + + await recreateHamtLevel(blockstore, node.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options) + const position = await rootBucket._findNewBucketAndPos(name) + + // i-- + path.push({ + bucket: position.bucket, + prefix: toPrefix(position.pos), + node + }) + + continue + } + + if (segment.bucket == null) { + throw new Error('Shard was invalid') + } + + // add intermediate links to bucket + await addLinksToHamtBucket(blockstore, node.Links, segment.bucket, rootBucket, options) + + segment.node = node + } + + await rootBucket.put(name, true) + + path.reverse() + + return path +} + +export interface CreateShardOptions { + mtime?: Mtime + mode?: number + cidVersion: Version +} + +export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise => { + const shard = new DirSharded({ + root: true, + dir: true, + parent: undefined, + parentKey: undefined, + path: '', + dirty: true, + flat: false, + mtime: options.mtime, + mode: options.mode + }, options) + + for (let i = 0; i < contents.length; i++) { + await shard._bucket.put(contents[i].name, { + size: contents[i].size, + cid: contents[i].cid + }) + } + + const res = await last(shard.flush(blockstore)) + + if (res == null) { + throw new Error('Flushing shard yielded no result') + } + + return res +} diff --git a/src/commands/utils/is-over-shard-threshold.ts b/src/commands/utils/is-over-shard-threshold.ts new file mode 100644 index 00000000..9902eb90 --- /dev/null +++ b/src/commands/utils/is-over-shard-threshold.ts @@ -0,0 +1,78 @@ +import type { PBNode } from '@ipld/dag-pb' +import type { Blockstore } from 'interface-blockstore' +import { UnixFS } from 'ipfs-unixfs' +import * as dagPb from '@ipld/dag-pb' +import { CID_V0, CID_V1 } from './dir-sharded.js' + +/** + * Estimate node size only based on DAGLink name and CID byte lengths + * https://github.com/ipfs/go-unixfsnode/blob/37b47f1f917f1b2f54c207682f38886e49896ef9/data/builder/directory.go#L81-L96 + * + * If the node is a hamt sharded directory the calculation is based on if it was a regular directory. + */ +export async function isOverShardThreshold (node: PBNode, blockstore: Blockstore, threshold: number): Promise { + if (node.Data == null) { + throw new Error('DagPB node had no data') + } + + const unixfs = UnixFS.unmarshal(node.Data) + let size: number + + if (unixfs.type === 'directory') { + size = estimateNodeSize(node) + } else if (unixfs.type === 'hamt-sharded-directory') { + size = await estimateShardSize(node, 0, threshold, blockstore) + } else { + throw new Error('Can only estimate the size of directories or shards') + } + + return size > threshold +} + +function estimateNodeSize (node: PBNode): number { + let size = 0 + + // estimate size only based on DAGLink name and CID byte lengths + // https://github.com/ipfs/go-unixfsnode/blob/37b47f1f917f1b2f54c207682f38886e49896ef9/data/builder/directory.go#L81-L96 + for (const link of node.Links) { + size += (link.Name ?? '').length + size += link.Hash.version === 1 ? CID_V1.bytes.byteLength : CID_V0.bytes.byteLength + } + + return size +} + +async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blockstore): Promise { + if (current > max) { + return max + } + + if (node.Data == null) { + return current + } + + const unixfs = UnixFS.unmarshal(node.Data) + + if (!unixfs.isDirectory()) { + return current + } + + for (const link of node.Links) { + let name = link.Name ?? '' + + // remove hamt hash prefix from name + name = name.substring(2) + + current += name.length + current += link.Hash.bytes.byteLength + + if (link.Hash.code === dagPb.code) { + const block = await blockstore.get(link.Hash) + const node = dagPb.decode(block) + + current += await estimateShardSize(node, current, max, blockstore) + } + } + + return current +} diff --git a/src/commands/utils/persist.ts b/src/commands/utils/persist.ts new file mode 100644 index 00000000..15f12752 --- /dev/null +++ b/src/commands/utils/persist.ts @@ -0,0 +1,27 @@ +import { CID } from 'multiformats/cid' +import * as dagPb from '@ipld/dag-pb' +import { sha256 } from 'multiformats/hashes/sha2' +import type { Blockstore } from 'interface-blockstore' +import type { BlockCodec } from 'multiformats/codecs/interface' +import type { Version as CIDVersion } from 'multiformats/cid' + +export interface PersistOptions { + codec?: BlockCodec + cidVersion: CIDVersion + signal?: AbortSignal +} + +export const persist = async (buffer: Uint8Array, blockstore: Blockstore, options: PersistOptions): Promise => { + if (options.codec == null) { + options.codec = dagPb + } + + const multihash = await sha256.digest(buffer) + const cid = CID.create(options.cidVersion, options.codec.code, multihash) + + await blockstore.put(cid, buffer, { + signal: options.signal + }) + + return cid +} diff --git a/src/commands/utils/remove-link.ts b/src/commands/utils/remove-link.ts new file mode 100644 index 00000000..e9765d39 --- /dev/null +++ b/src/commands/utils/remove-link.ts @@ -0,0 +1,252 @@ + +import * as dagPB from '@ipld/dag-pb' +import type { CID, Version } from 'multiformats/cid' +import { logger } from '@libp2p/logger' +import { UnixFS } from 'ipfs-unixfs' +import { + generatePath, + HamtPathSegment, + updateHamtDirectory, + UpdateHamtDirectoryOptions +} from './hamt-utils.js' +import type { PBNode } from '@ipld/dag-pb' +import type { Blockstore } from 'interface-blockstore' +import type { Directory } from './cid-to-directory.js' +import type { AbortOptions } from '@libp2p/interfaces' +import { InvalidParametersError, InvalidPBNodeError } from './errors.js' +import { exporter } from 'ipfs-unixfs-exporter' +import { persist } from './persist.js' +import { isOverShardThreshold } from './is-over-shard-threshold.js' + +const log = logger('helia:unixfs:utils:remove-link') + +export interface RmLinkOptions extends AbortOptions { + shardSplitThresholdBytes: number + cidVersion: Version +} + +export interface RemoveLinkResult { + node: PBNode + cid: CID +} + +export async function removeLink (parent: Directory, name: string, blockstore: Blockstore, options: RmLinkOptions): Promise { + if (parent.node.Data == null) { + throw new InvalidPBNodeError('Parent node had no data') + } + + const meta = UnixFS.unmarshal(parent.node.Data) + + if (meta.type === 'hamt-sharded-directory') { + log(`removing ${name} from sharded directory`) + + const result = await removeFromShardedDirectory(parent, name, blockstore, options) + + if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) { + log('converting shard to flat directory %c', parent.cid) + + return await convertToFlatDirectory(result, blockstore, options) + } + + return result + } + + log(`removing link ${name} regular directory`) + + return await removeFromDirectory(parent, name, blockstore, options) +} + +const removeFromDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise => { + // Remove existing link if it exists + parent.node.Links = parent.node.Links.filter((link) => { + return link.Name !== name + }) + + const parentBlock = dagPB.encode(parent.node) + const parentCid = await persist(parentBlock, blockstore, { + ...options, + cidVersion: parent.cid.version + }) + + log(`Updated regular directory ${parentCid}`) + + return { + node: parent.node, + cid: parentCid + } +} + +const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ cid: CID, node: PBNode }> => { + const path = await generatePath(parent, name, blockstore, options) + + // remove file from root bucket + const rootBucket = path[path.length - 1].bucket + + if (rootBucket == null) { + throw new Error('Could not generate HAMT path') + } + + await rootBucket.del(name) + + // update all nodes in the shard path + return await updateShard(path, name, blockstore, options) +} + +/** + * The `path` param is a list of HAMT path segments starting with th + */ +const updateShard = async (path: HamtPathSegment[], name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ node: PBNode, cid: CID }> => { + const fileName = `${path[0].prefix}${name}` + + // skip first path segment as it is the file to remove + for (let i = 1; i < path.length; i++) { + const lastPrefix = path[i - 1].prefix + const segment = path[i] + + if (segment.node == null) { + throw new InvalidParametersError('Path segment had no associated PBNode') + } + + const link = segment.node.Links + .find(link => (link.Name ?? '').substring(0, 2) === lastPrefix) + + if (link == null) { + throw new InvalidParametersError(`No link found with prefix ${lastPrefix} for file ${name}`) + } + + if (link.Name == null) { + throw new InvalidParametersError(`${lastPrefix} link had no name`) + } + + if (link.Name === fileName) { + log(`removing existing link ${link.Name}`) + + const links = segment.node.Links.filter((nodeLink) => { + return nodeLink.Name !== link.Name + }) + + if (segment.bucket == null) { + throw new Error('Segment bucket was missing') + } + + await segment.bucket.del(name) + + const result = await updateHamtDirectory({ + Data: segment.node.Data, + Links: links + }, blockstore, segment.bucket, options) + + segment.node = result.node + segment.cid = result.cid + segment.size = result.size + } + + if (link.Name === lastPrefix) { + log(`updating subshard with prefix ${lastPrefix}`) + + const lastSegment = path[i - 1] + + if (lastSegment.node?.Links.length === 1) { + log(`removing subshard for ${lastPrefix}`) + + // convert subshard back to normal file entry + const link = lastSegment.node.Links[0] + link.Name = `${lastPrefix}${(link.Name ?? '').substring(2)}` + + // remove existing prefix + segment.node.Links = segment.node.Links.filter((link) => { + return link.Name !== lastPrefix + }) + + // add new child + segment.node.Links.push(link) + } else { + // replace subshard entry + log(`replacing subshard for ${lastPrefix}`) + + // remove existing prefix + segment.node.Links = segment.node.Links.filter((link) => { + return link.Name !== lastPrefix + }) + + if (lastSegment.cid == null) { + throw new Error('Did not persist previous segment') + } + + // add new child + segment.node.Links.push({ + Name: lastPrefix, + Hash: lastSegment.cid, + Tsize: lastSegment.size + }) + } + + if (segment.bucket == null) { + throw new Error('Segment bucket was missing') + } + + const result = await updateHamtDirectory(segment.node, blockstore, segment.bucket, options) + segment.node = result.node + segment.cid = result.cid + segment.size = result.size + } + } + + const rootSegment = path[path.length - 1] + + if (rootSegment == null || rootSegment.cid == null || rootSegment.node == null) { + throw new InvalidParametersError('Failed to update shard') + } + + return { + cid: rootSegment.cid, + node: rootSegment.node + } +} + +const convertToFlatDirectory = async (parent: Directory, blockstore: Blockstore, options: RmLinkOptions): Promise => { + if (parent.node.Data == null) { + throw new InvalidParametersError('Invalid parent passed to convertToFlatDirectory') + } + + const rootNode: PBNode = { + Links: [] + } + const dir = await exporter(parent.cid, blockstore) + + if (dir.type !== 'directory') { + throw new Error('Unexpected node type') + } + + for await (const entry of dir.content()) { + let tsize = 0 + + if (entry.node instanceof Uint8Array) { + tsize = entry.node.byteLength + } else { + tsize = dagPB.encode(entry.node).length + } + + rootNode.Links.push({ + Hash: entry.cid, + Name: entry.name, + Tsize: tsize + }) + } + + // copy mode/mtime over if set + const oldUnixfs = UnixFS.unmarshal(parent.node.Data) + rootNode.Data = new UnixFS({ type: 'directory', mode: oldUnixfs.mode, mtime: oldUnixfs.mtime }).marshal() + const block = dagPB.encode(dagPB.prepare(rootNode)) + + const cid = await persist(block, blockstore, { + codec: dagPB, + cidVersion: parent.cid.version, + signal: options.signal + }) + + return { + cid, + node: rootNode + } +} diff --git a/src/commands/utils/resolve.ts b/src/commands/utils/resolve.ts new file mode 100644 index 00000000..33530d83 --- /dev/null +++ b/src/commands/utils/resolve.ts @@ -0,0 +1,135 @@ +import type { CID } from 'multiformats/cid' +import { exporter } from 'ipfs-unixfs-exporter' +import type { AbortOptions } from '@libp2p/interfaces' +import { logger } from '@libp2p/logger' +import { DoesNotExistError, InvalidParametersError } from './errors.js' +import { addLink } from './add-link.js' +import { cidToDirectory } from './cid-to-directory.js' +import { cidToPBLink } from './cid-to-pblink.js' +import type { Blockstore } from 'interface-blockstore' + +const log = logger('helia:unixfs:components:utils:add-link') + +export interface Segment { + name: string + cid: CID + size: bigint +} + +export interface ResolveResult { + /** + * The CID at the end of the path + */ + cid: CID + + path?: string + + /** + * If present, these are the CIDs and path segments that were traversed through to reach the final CID + * + * If not present, there was no path passed or the path was an empty string + */ + segments?: Segment[] +} + +export async function resolve (cid: CID, path: string | undefined, blockstore: Blockstore, options: AbortOptions): Promise { + log('resolve "%s" under %c', path, cid) + + if (path == null || path === '') { + return { cid } + } + + const parts = path.split('/').filter(Boolean) + const segments: Segment[] = [{ + name: '', + cid, + size: 0n + }] + + for (let i = 0; i < parts.length; i++) { + const part = parts[i] + const result = await exporter(cid, blockstore, options) + + if (result.type === 'file') { + if (i < parts.length - 1) { + throw new InvalidParametersError('Path was invalid') + } + + cid = result.cid + } else if (result.type === 'directory') { + let dirCid: CID | undefined + + for await (const entry of result.content()) { + if (entry.name === part) { + dirCid = entry.cid + } + } + + if (dirCid == null) { + throw new DoesNotExistError('Could not find path in directory') + } + + cid = dirCid + + segments.push({ + name: part, + cid, + size: result.size + }) + } else { + throw new InvalidParametersError('Could not resolve path') + } + } + + return { + cid, + path, + segments + } +} + +export interface UpdatePathCidsOptions extends AbortOptions { + shardSplitThresholdBytes: number +} + +/** + * Where we have descended into a DAG to update a child node, ascend up the DAG creating + * new hashes and blocks for the changed content + */ +export async function updatePathCids (cid: CID, result: ResolveResult, blockstore: Blockstore, options: UpdatePathCidsOptions): Promise { + if (result.segments == null || result.segments.length === 0) { + return cid + } + + let child = result.segments.pop() + + if (child == null) { + throw new Error('Insufficient segments') + } + + child.cid = cid + + result.segments.reverse() + + for (const parent of result.segments) { + const [ + directory, + pblink + ] = await Promise.all([ + cidToDirectory(parent.cid, blockstore, options), + cidToPBLink(child.cid, child.name, blockstore, options) + ]) + + const result = await addLink(directory, pblink, blockstore, { + ...options, + allowOverwriting: true, + cidVersion: cid.version + }) + + cid = result.cid + parent.cid = cid + child = parent + } + + return cid +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 00000000..a2c6d733 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,172 @@ +import type { CID, Version } from 'multiformats/cid' +import type { Blockstore } from 'interface-blockstore' +import type { AbortOptions } from '@libp2p/interfaces' +import { cat } from './commands/cat.js' +import { mkdir } from './commands/mkdir.js' +import type { Mtime } from 'ipfs-unixfs' +import { cp } from './commands/cp.js' +import { rm } from './commands/rm.js' +import { stat } from './commands/stat.js' +import { touch } from './commands/touch.js' +import { chmod } from './commands/chmod.js' +import type { UnixFSEntry } from 'ipfs-unixfs-exporter' +import { ls } from './commands/ls.js' + +export interface UnixFSComponents { + blockstore: Blockstore +} + +export interface CatOptions extends AbortOptions { + offset?: number + length?: number + path?: string +} + +export interface ChmodOptions extends AbortOptions { + recursive: boolean + path?: string + shardSplitThresholdBytes: number +} + +export interface CpOptions extends AbortOptions { + force: boolean + shardSplitThresholdBytes: number +} + +export interface LsOptions extends AbortOptions { + path?: string + offset?: number + length?: number +} + +export interface MkdirOptions extends AbortOptions { + cidVersion: Version + force: boolean + mode?: number + mtime?: Mtime + shardSplitThresholdBytes: number +} + +export interface RmOptions extends AbortOptions { + shardSplitThresholdBytes: number +} + +export interface StatOptions extends AbortOptions { + path?: string +} + +export interface UnixFSStats { + /** + * The file or directory CID + */ + cid: CID + + /** + * The file or directory mode + */ + mode?: number + + /** + * The file or directory mtime + */ + mtime?: Mtime + + /** + * The size of the file in bytes + */ + fileSize: bigint + + /** + * The size of the DAG that holds the file in bytes + */ + dagSize: bigint + + /** + * How much of the file is in the local block store + */ + localFileSize: bigint + + /** + * How much of the DAG that holds the file is in the local blockstore + */ + localDagSize: bigint + + /** + * How many blocks make up the DAG - nb. this will only be accurate + * if all blocks are present in the local blockstore + */ + blocks: number + + /** + * The type of file + */ + type: 'file' | 'directory' | 'raw' + + /** + * UnixFS metadata about this file or directory. Will not be present + * if the node is a `raw` type. + */ + unixfs?: import('ipfs-unixfs').UnixFS +} + +export interface TouchOptions extends AbortOptions { + mtime?: Mtime + path?: string + recursive: boolean + shardSplitThresholdBytes: number +} + +export interface UnixFS { + cat: (cid: CID, options?: Partial) => AsyncIterable + chmod: (source: CID, mode: number, options?: Partial) => Promise + cp: (source: CID, target: CID, name: string, options?: Partial) => Promise + ls: (cid: CID, options?: Partial) => AsyncIterable + mkdir: (cid: CID, dirname: string, options?: Partial) => Promise + rm: (cid: CID, path: string, options?: Partial) => Promise + stat: (cid: CID, options?: Partial) => Promise + touch: (cid: CID, options?: Partial) => Promise +} + +class DefaultUnixFS implements UnixFS { + private readonly components: UnixFSComponents + + constructor (components: UnixFSComponents) { + this.components = components + } + + async * cat (cid: CID, options: Partial = {}): AsyncIterable { + yield * cat(cid, this.components.blockstore, options) + } + + async chmod (source: CID, mode: number, options: Partial = {}): Promise { + return await chmod(source, mode, this.components.blockstore, options) + } + + async cp (source: CID, target: CID, name: string, options: Partial = {}): Promise { + return await cp(source, target, name, this.components.blockstore, options) + } + + async * ls (cid: CID, options: Partial = {}): AsyncIterable { + yield * ls(cid, this.components.blockstore, options) + } + + async mkdir (cid: CID, dirname: string, options: Partial = {}): Promise { + return await mkdir(cid, dirname, this.components.blockstore, options) + } + + async rm (cid: CID, path: string, options: Partial = {}): Promise { + return await rm(cid, path, this.components.blockstore, options) + } + + async stat (cid: CID, options: Partial = {}): Promise { + return await stat(cid, this.components.blockstore, options) + } + + async touch (cid: CID, options: Partial = {}): Promise { + return await touch(cid, this.components.blockstore, options) + } +} + +export function unixfs (helia: { blockstore: Blockstore }): UnixFS { + return new DefaultUnixFS(helia) +} diff --git a/test/cat.spec.ts b/test/cat.spec.ts new file mode 100644 index 00000000..3f670010 --- /dev/null +++ b/test/cat.spec.ts @@ -0,0 +1,87 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { CID } from 'multiformats/cid' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import toBuffer from 'it-to-buffer' +import drain from 'it-drain' +import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' + +const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) + +describe('cat', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('reads a small file', async () => { + const { cid } = await importBytes(smallFile, blockstore) + const bytes = await toBuffer(fs.cat(cid)) + + expect(bytes).to.equalBytes(smallFile) + }) + + it('reads a file with an offset', async () => { + const offset = 10 + const { cid } = await importBytes(smallFile, blockstore) + const bytes = await toBuffer(fs.cat(cid, { + offset + })) + + expect(bytes).to.equalBytes(smallFile.subarray(offset)) + }) + + it('reads a file with a length', async () => { + const length = 10 + const { cid } = await importBytes(smallFile, blockstore) + const bytes = await toBuffer(fs.cat(cid, { + length + })) + + expect(bytes).to.equalBytes(smallFile.subarray(0, length)) + }) + + it('reads a file with an offset and a length', async () => { + const offset = 2 + const length = 5 + const { cid } = await importBytes(smallFile, blockstore) + const bytes = await toBuffer(fs.cat(cid, { + offset, + length + })) + + expect(bytes).to.equalBytes(smallFile.subarray(offset, offset + length)) + }) + + it('refuses to read a directory', async () => { + await expect(drain(fs.cat(emptyDirCid))).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_A_FILE') + }) + + it('reads file from inside a sharded directory', async () => { + const content = Uint8Array.from([0, 1, 2, 3, 4]) + const dirCid = await createShardedDirectory(blockstore) + const { cid: fileCid } = await importBytes(content, blockstore) + const path = 'new-file.txt' + + const updatedCid = await fs.cp(fileCid, dirCid, path) + + const bytes = await toBuffer(fs.cat(updatedCid, { + path + })) + + expect(bytes).to.deep.equal(content) + }) +}) diff --git a/test/chmod.spec.ts b/test/chmod.spec.ts new file mode 100644 index 00000000..e9ce3cfc --- /dev/null +++ b/test/chmod.spec.ts @@ -0,0 +1,95 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { MemoryBlockstore } from 'blockstore-core' +import { UnixFS, unixfs } from '../src/index.js' +import type { CID } from 'multiformats/cid' +import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' + +const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) + +describe('chmod', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('should update the mode for a raw node', async () => { + const { cid } = await importBytes(smallFile, blockstore) + const originalMode = (await fs.stat(cid)).mode + const updatedCid = await fs.chmod(cid, 0o777) + + const updatedMode = (await fs.stat(updatedCid)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update the mode for a file', async () => { + const { cid } = await importBytes(smallFile, blockstore, { + rawLeaves: false + }) + const originalMode = (await fs.stat(cid)).mode + const updatedCid = await fs.chmod(cid, 0o777) + + const updatedMode = (await fs.stat(updatedCid)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update the mode for a directory', async () => { + const path = `foo-${Math.random()}` + + const dirCid = await fs.mkdir(emptyDirCid, path) + const originalMode = (await fs.stat(dirCid, { + path + })).mode + const updatedCid = await fs.chmod(dirCid, 0o777, { + path + }) + + const updatedMode = (await fs.stat(updatedCid, { + path + })).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update mode recursively', async () => { + const path = 'path' + const { cid } = await importBytes(smallFile, blockstore) + const dirCid = await fs.cp(cid, emptyDirCid, path) + const originalMode = (await fs.stat(dirCid, { + path + })).mode + const updatedCid = await fs.chmod(dirCid, 0o777, { + recursive: true + }) + + const updatedMode = (await fs.stat(updatedCid, { + path + })).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update the mode for a hamt-sharded-directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + + const originalMode = (await fs.stat(shardedDirCid)).mode + const updatedShardCid = await fs.chmod(shardedDirCid, 0o777) + + const updatedMode = (await fs.stat(updatedShardCid)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) +}) diff --git a/test/cp.spec.ts b/test/cp.spec.ts new file mode 100644 index 00000000..b849325e --- /dev/null +++ b/test/cp.spec.ts @@ -0,0 +1,169 @@ +/* eslint-env mocha */ + +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { expect } from 'aegir/chai' +import { identity } from 'multiformats/hashes/identity' +import { CID } from 'multiformats/cid' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import toBuffer from 'it-to-buffer' +import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import first from 'it-first' + +describe('cp', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('refuses to copy files without a source', async () => { + // @ts-expect-error invalid args + await expect(fs.cp()).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files without a source, even with options', async () => { + // @ts-expect-error invalid args + await expect(fs.cp({})).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files without a destination', async () => { + // @ts-expect-error invalid args + await expect(fs.cp('/source')).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files without a destination, even with options', async () => { + // @ts-expect-error invalid args + await expect(fs.cp('/source', {})).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files to an unreadable node', async () => { + const hash = identity.digest(uint8ArrayFromString('derp')) + const { cid: source } = await importBytes(Uint8Array.from([0, 1, 3, 4]), blockstore) + const target = CID.createV1(identity.code, hash) + + await expect(fs.cp(source, target, 'foo')).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_A_DIRECTORY') + }) + + it('refuses to copy files from an unreadable node', async () => { + const hash = identity.digest(uint8ArrayFromString('derp')) + const source = CID.createV1(identity.code, hash) + + await expect(fs.cp(source, emptyDirCid, 'foo')).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_UNIXFS') + }) + + it('refuses to copy files to an existing file', async () => { + const path = 'path' + const { cid: source } = await importBytes(Uint8Array.from([0, 1, 3, 4]), blockstore) + const target = await fs.cp(source, emptyDirCid, path) + + await expect(fs.cp(source, target, path)).to.eventually.be.rejected + .with.property('code', 'ERR_ALREADY_EXISTS') + + // should succeed with force option + await expect(fs.cp(source, target, path, { + force: true + })).to.eventually.be.ok() + }) + + it('copies a file to new location', async () => { + const data = Uint8Array.from([0, 1, 3, 4]) + const path = 'path' + const { cid: source } = await importBytes(data, blockstore) + const dirCid = await fs.cp(source, emptyDirCid, path) + + const bytes = await toBuffer(fs.cat(dirCid, { + path + })) + + expect(bytes).to.deep.equal(data) + }) + + it('copies directories', async () => { + const path = 'path' + const dirCid = await fs.cp(emptyDirCid, emptyDirCid, path) + + await expect(fs.stat(dirCid, { + path + })).to.eventually.include({ + type: 'directory' + }) + }) + + it('copies a sharded directory to a normal directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const path = 'sharded-dir' + const containingDirCid = await fs.cp(shardedDirCid, emptyDirCid, path) + + // should still be a regular directory + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory') + + const subDirStats = await fs.stat(containingDirCid, { + path + }) + expect(subDirStats).to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + expect(subDirStats.cid).to.eql(shardedDirCid) + }) + + it('copies a normal directory to a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const path = 'normal-dir' + const containingDirCid = await fs.cp(emptyDirCid, shardedDirCid, path) + + // should still be a sharded directory + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const subDirStats = await fs.stat(containingDirCid, { + path + }) + expect(subDirStats).to.have.nested.property('unixfs.type', 'directory') + expect(subDirStats.cid).to.eql(emptyDirCid) + }) + + it('copies a file from a normal directory to a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const path = `file-${Math.random()}.txt` + const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore, { + rawLeaves: false + }) + + const containingDirCid = await fs.cp(fileCid, shardedDirCid, path) + + // should still be a sharded directory + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const fileInDirStats = await fs.stat(containingDirCid, { + path + }) + expect(fileInDirStats).to.have.nested.property('unixfs.type', 'file') + expect(fileInDirStats.cid).to.eql(fileCid) + }) + + it('refuses to copy files to an existing file in a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const file = await first(fs.ls(shardedDirCid)) + + if (file == null) { + throw new Error('No files listed') + } + + await expect(fs.cp(file.cid, shardedDirCid, file.name)).to.eventually.be.rejected + .with.property('code', 'ERR_ALREADY_EXISTS') + + // should succeed with force option + await expect(fs.cp(file.cid, shardedDirCid, file.name, { + force: true + })).to.eventually.be.ok() + }) +}) diff --git a/test/fixtures/create-sharded-directory.ts b/test/fixtures/create-sharded-directory.ts new file mode 100644 index 00000000..ae5f5e43 --- /dev/null +++ b/test/fixtures/create-sharded-directory.ts @@ -0,0 +1,27 @@ +import { expect } from 'aegir/chai' +import last from 'it-last' +import { importer } from 'ipfs-unixfs-importer' +import type { CID } from 'multiformats/cid' +import type { Blockstore } from 'interface-blockstore' + +export async function createShardedDirectory (blockstore: Blockstore, files = 1001): Promise { + const result = await last(importer((function * () { + for (let i = 0; i < files; i++) { + yield { + path: `./file-${i}`, + content: Uint8Array.from([0, 1, 2, 3, 4, 5, i]) + } + } + }()), blockstore, { + shardSplitThresholdBytes: 1, + wrapWithDirectory: true + })) + + if (result == null) { + throw new Error('No result received from ipfs.addAll') + } + + expect(result).to.have.nested.property('unixfs.type', 'hamt-sharded-directory', 'tried to create a shared directory but the result was not a shard') + + return result.cid +} diff --git a/test/fixtures/create-subsharded-directory.ts b/test/fixtures/create-subsharded-directory.ts new file mode 100644 index 00000000..64bcd963 --- /dev/null +++ b/test/fixtures/create-subsharded-directory.ts @@ -0,0 +1,93 @@ +import type { Blockstore } from 'interface-blockstore' +import { importBytes, importer } from 'ipfs-unixfs-importer' +import { CID } from 'multiformats/cid' +import { unixfs } from '../../src/index.js' +import * as dagPb from '@ipld/dag-pb' +import last from 'it-last' + +export async function createSubshardedDirectory (blockstore: Blockstore, depth: number = 1, files: number = 5000): Promise<{ + importerCid: CID + containingDirCid: CID + fileName: string +}> { + const fs = unixfs({ blockstore }) + + const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + let containingDirCid = CID.parse('bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354') + let fileName: string | undefined + let count = 0 + + for (let i = 0; i < files; i++) { + fileName = `file-${i}-${new Array(512).fill('0').join('')}.txt` + + containingDirCid = await fs.cp(fileCid, containingDirCid, fileName, { + shardSplitThresholdBytes: 1 + }) + + if (await searchCIDForSubshards(containingDirCid, blockstore, depth)) { + count = i + + break + } + } + + if (fileName == null) { + throw new Error('could not find file that would create a subshard') + } + + // create a shard with the importer that is the same as the directory after we delete the file that causes a sub-shard to be created + const importResult = await last(importer( + new Array(count).fill(0).map((_, i) => { + return { + path: `file-${i}-${new Array(512).fill('0').join('')}.txt`, + content: Uint8Array.from([0, 1, 2, 3, 4]) + } + }), blockstore, { + wrapWithDirectory: true, + shardSplitThresholdBytes: 1 + })) + + if (importResult == null) { + throw new Error('Nothing imported') + } + + const { cid: importerCid } = importResult + + return { + importerCid, + containingDirCid, + fileName + } +} + +async function searchCIDForSubshards (cid: CID, blockstore: Blockstore, depth: number = 1): Promise { + const block = await blockstore.get(cid) + const node = dagPb.decode(block) + + // search links for subshard + for (const link of node.Links) { + if (link.Name?.length === 2) { + const block = await blockstore.get(link.Hash) + const node = dagPb.decode(block) + const firstLink = node.Links[1] + + if (firstLink == null) { + throw new Error('Subshard had no child links') + } + + if (firstLink.Name == null) { + throw new Error('Subshard child had no name') + } + + if (depth === 1) { + return true + } + + if (await searchCIDForSubshards(link.Hash, blockstore, depth - 1)) { + return true + } + } + } + + return false +} diff --git a/test/fixtures/print-tree.ts b/test/fixtures/print-tree.ts new file mode 100644 index 00000000..e2cb75a4 --- /dev/null +++ b/test/fixtures/print-tree.ts @@ -0,0 +1,33 @@ +import type { Blockstore } from 'interface-blockstore' +import { UnixFS } from 'ipfs-unixfs' +import type { CID } from 'multiformats/cid' +import * as dagPb from '@ipld/dag-pb' +import * as raw from 'multiformats/codecs/raw' + +export async function printTree (cid: CID, blockstore: Blockstore, name: string = '', indent: string = ''): Promise { + const block = await blockstore.get(cid) + + if (cid.code === dagPb.code) { + const node = dagPb.decode(block) + + if (node.Data == null) { + return + } + + const unixfs = UnixFS.unmarshal(node.Data) + + console.info(indent, cid.toString(), name, unixfs.type) // eslint-disable-line no-console + + for (const link of node.Links) { + let name = link.Name ?? '' + + if (name.length > 12) { + name = name.substring(0, 12) + '...' + } + + await printTree(link.Hash, blockstore, name, `${indent} `) + } + } else if (cid.code === raw.code) { + console.info(indent, cid.toString(), name, 'dag-raw') // eslint-disable-line no-console + } +} diff --git a/test/ls.spec.ts b/test/ls.spec.ts new file mode 100644 index 00000000..6c316ae3 --- /dev/null +++ b/test/ls.spec.ts @@ -0,0 +1,123 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { CID } from 'multiformats/cid' +import all from 'it-all' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' + +describe('ls', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('should require a path', async () => { + // @ts-expect-error invalid args + await expect(all(fs.ls())).to.eventually.be.rejected() + }) + + it('lists files in a directory', async () => { + const path = 'path' + const data = Uint8Array.from([0, 1, 2, 3]) + const { cid: fileCid } = await importBytes(data, blockstore) + const dirCid = await fs.cp(fileCid, emptyDirCid, path) + const files = await all(fs.ls(dirCid)) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileCid, + name: path, + size: BigInt(data.byteLength), + type: 'raw' + }]) + }) + + it('lists a file', async () => { + const path = 'path' + const data = Uint8Array.from([0, 1, 2, 3]) + const { cid: fileCid } = await importBytes(data, blockstore, { + rawLeaves: false + }) + const dirCid = await fs.cp(fileCid, emptyDirCid, path) + const files = await all(fs.ls(dirCid, { + path + })) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileCid, + size: BigInt(data.byteLength), + type: 'file' + }]) + }) + + it('lists a raw node', async () => { + const path = 'path' + const data = Uint8Array.from([0, 1, 2, 3]) + const { cid: fileCid } = await importBytes(data, blockstore) + const dirCid = await fs.cp(fileCid, emptyDirCid, path) + const files = await all(fs.ls(dirCid, { + path + })) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileCid, + size: BigInt(data.byteLength), + type: 'raw' + }]) + }) + + it('lists a sharded directory contents', async () => { + const fileCount = 1001 + const shardedDirCid = await createShardedDirectory(blockstore, fileCount) + const files = await all(fs.ls(shardedDirCid)) + + expect(files.length).to.equal(fileCount) + + files.forEach(file => { + // should be a file + expect(file.type).to.equal('raw') + }) + }) + + it('lists a file inside a sharded directory directly', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const files = await all(fs.ls(shardedDirCid)) + const fileName = files[0].name + + // should be able to ls new file directly + const directFiles = await all(fs.ls(shardedDirCid, { + path: fileName + })) + + expect(directFiles.length).to.equal(1) + expect(directFiles.filter(file => file.name === fileName)).to.be.ok() + }) + + it('lists the contents of a directory inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const dirName = `subdir-${Math.random()}` + const fileName = `small-file-${Math.random()}.txt` + + const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore) + const containingDirectoryCid = await fs.cp(fileCid, emptyDirCid, fileName) + const updatedShardCid = await fs.cp(containingDirectoryCid, shardedDirCid, dirName) + + const files = await all(fs.ls(updatedShardCid, { + path: dirName + })) + + expect(files.length).to.equal(1) + expect(files.filter(file => file.name === fileName)).to.be.ok() + }) +}) diff --git a/test/mkdir.spec.ts b/test/mkdir.spec.ts new file mode 100644 index 00000000..eb94437b --- /dev/null +++ b/test/mkdir.spec.ts @@ -0,0 +1,123 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import all from 'it-all' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import type { CID } from 'multiformats/cid' +import type { Mtime } from 'ipfs-unixfs' +import { importContent } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' + +describe('mkdir', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + let emptyDirCidV0: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + + const importedV0 = await importContent({ path: 'empty' }, blockstore, { + cidVersion: 0 + }) + emptyDirCidV0 = importedV0.cid + }) + + async function testMode (mode: number | undefined, expectedMode: number): Promise { + const path = 'sub-directory' + const dirCid = await fs.mkdir(emptyDirCid, path, { + mode + }) + + await expect(fs.stat(dirCid, { + path + })).to.eventually.have.property('mode', expectedMode) + } + + async function testMtime (mtime: Mtime, expectedMtime: Mtime): Promise { + const path = 'sub-directory' + const dirCid = await fs.mkdir(emptyDirCid, path, { + mtime + }) + + await expect(fs.stat(dirCid, { + path + })).to.eventually.have.deep.property('mtime', expectedMtime) + } + + it('requires a directory', async () => { + // @ts-expect-error not enough arguments + await expect(fs.mkdir(emptyDirCid)).to.eventually.be.rejected() + }) + + it('creates a directory', async () => { + const path = 'foo' + const dirCid = await fs.mkdir(emptyDirCid, path) + + const stats = await fs.stat(dirCid) + expect(stats.type).to.equal('directory') + + const files = await all(fs.ls(dirCid)) + + expect(files.length).to.equal(1) + expect(files).to.have.nested.property('[0].name', path) + }) + + it('refuses to create a directory that already exists', async () => { + const path = 'qux' + const dirCid = await fs.mkdir(emptyDirCid, path) + + await expect(fs.mkdir(dirCid, path)).to.eventually.be.rejected() + .with.property('code', 'ERR_ALREADY_EXISTS') + }) + + it('creates a nested directory with a different CID version to the parent', async () => { + const subDirectory = 'sub-dir' + + expect(emptyDirCidV0).to.have.property('version', 0) + + const dirCid = await fs.mkdir(emptyDirCidV0, subDirectory, { + cidVersion: 1 + }) + + await expect(fs.stat(dirCid)).to.eventually.have.nested.property('cid.version', 0) + await expect(fs.stat(dirCid, { + path: subDirectory + })).to.eventually.have.nested.property('cid.version', 1) + }) + + it('should make directory and have default mode', async function () { + await testMode(undefined, parseInt('0755', 8)) + }) + + it('should make directory and specify mtime as { nsecs, secs }', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + + await testMtime(mtime, mtime) + }) + + it('makes a directory inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const dirName = `subdir-${Math.random()}` + + const updatedShardCid = await fs.mkdir(shardedDirCid, dirName) + + // should still be a sharded directory + await expect(fs.stat(updatedShardCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + // subdir should be a regular directory + await expect(fs.stat(updatedShardCid, { + path: dirName + })).to.eventually.have.nested.property('unixfs.type', 'directory') + }) +}) diff --git a/test/rm.spec.ts b/test/rm.spec.ts new file mode 100644 index 00000000..5b14c35e --- /dev/null +++ b/test/rm.spec.ts @@ -0,0 +1,219 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import type { CID } from 'multiformats/cid' +import { importContent, importBytes, importer } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import last from 'it-last' +import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' + +const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) + +describe('rm', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('refuses to remove files without arguments', async () => { + // @ts-expect-error invalid args + await expect(fs.rm()).to.eventually.be.rejected() + }) + + it('removes a file', async () => { + const path = 'foo' + const { cid: fileCid } = await importBytes(smallFile, blockstore) + const dirCid = await fs.cp(fileCid, emptyDirCid, path) + const updatedDirCid = await fs.rm(dirCid, path) + + await expect(fs.stat(updatedDirCid, { + path + })).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('removes a directory', async () => { + const path = 'foo' + const dirCid = await fs.cp(emptyDirCid, emptyDirCid, path) + const updatedDirCid = await fs.rm(dirCid, path) + + await expect(fs.stat(updatedDirCid, { + path + })).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('removes a sharded directory inside a normal directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const dirName = `subdir-${Math.random()}` + const containingDirCid = await fs.cp(shardedDirCid, emptyDirCid, dirName) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory') + await expect(fs.stat(containingDirCid, { + path: dirName + })).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const updatedContainingDirCid = await fs.rm(containingDirCid, dirName) + + await expect(fs.stat(updatedContainingDirCid, { + path: dirName + })).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('removes a sharded directory inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const otherShardedDirCid = await createShardedDirectory(blockstore) + const dirName = `subdir-${Math.random()}` + const containingDirCid = await fs.cp(shardedDirCid, otherShardedDirCid, dirName) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + await expect(fs.stat(containingDirCid, { + path: dirName + })).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const updatedContainingDirCid = await fs.rm(containingDirCid, dirName) + + await expect(fs.stat(updatedContainingDirCid, { + path: dirName + })).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('results in the same hash as a sharded directory created by the importer when removing a file and removing the file means the root node is below the shard threshold', async function () { + const shardSplitThresholdBytes = 55 + + // create a shard with the importer + const importResult = await last(importer([{ + path: 'file-1.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }], blockstore, { + wrapWithDirectory: true, + shardSplitThresholdBytes + })) + + if (importResult == null) { + throw new Error('Nothing imported') + } + + const { cid: importerCid } = importResult + await expect(fs.stat(importerCid)).to.eventually.have.nested.property('unixfs.type', 'directory') + + // create the same shard with unixfs command + const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + let containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', { + shardSplitThresholdBytes + }) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory') + + containingDirCid = await fs.cp(fileCid, containingDirCid, 'file-2.txt', { + shardSplitThresholdBytes + }) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + containingDirCid = await fs.rm(containingDirCid, 'file-2.txt', { + shardSplitThresholdBytes + }) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory') + + expect(containingDirCid).to.eql(importerCid) + }) + + it('results in the same hash as a sharded directory created by the importer when removing a file', async function () { + const shardSplitThresholdBytes = 1 + + // create a shard with the importer + const importResult = await last(importer([{ + path: 'file-1.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }], blockstore, { + wrapWithDirectory: true, + shardSplitThresholdBytes + })) + + if (importResult == null) { + throw new Error('Nothing imported') + } + + const { cid: importerCid } = importResult + await expect(fs.stat(importerCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + // create the same shard with unixfs command + const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + let containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', { + shardSplitThresholdBytes + }) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + containingDirCid = await fs.cp(fileCid, containingDirCid, 'file-2.txt', { + shardSplitThresholdBytes + }) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + containingDirCid = await fs.rm(containingDirCid, 'file-2.txt', { + shardSplitThresholdBytes + }) + + await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + expect(containingDirCid).to.eql(importerCid) + }) + + it.skip('results in the same hash as a sharded directory created by the importer when removing a subshard', async function () { + let { + containingDirCid, + fileName, + importerCid + } = await createSubshardedDirectory(blockstore) + + await expect(fs.stat(importerCid)) + .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + // remove the file that caused the subshard to be created and the CID should be the same as the importer + containingDirCid = await fs.rm(containingDirCid, fileName, { + shardSplitThresholdBytes: 1 + }) + + await expect(fs.stat(containingDirCid)) + .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + expect(containingDirCid).to.eql(importerCid) + }) + + it.skip('results in the same hash as a sharded directory created by the importer when removing a subshard of a subshard', async function () { + let { + containingDirCid, + fileName, + importerCid + } = await createSubshardedDirectory(blockstore, 2) + + await expect(fs.stat(importerCid)) + .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + // remove the file that caused the subshard to be created and the CID should be the same as the importer + containingDirCid = await fs.rm(containingDirCid, fileName, { + shardSplitThresholdBytes: 1 + }) + + await expect(fs.stat(containingDirCid)) + .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + expect(containingDirCid).to.eql(importerCid) + }) +}) diff --git a/test/stat.spec.ts b/test/stat.spec.ts new file mode 100644 index 00000000..041768b5 --- /dev/null +++ b/test/stat.spec.ts @@ -0,0 +1,204 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import type { CID } from 'multiformats/cid' +import * as dagPb from '@ipld/dag-pb' +import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' + +const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) +const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100)) + +describe('stat', function () { + this.timeout(120 * 1000) + + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('stats an empty directory', async () => { + await expect(fs.stat(emptyDirCid)).to.eventually.include({ + fileSize: 0n, + dagSize: 2n, + blocks: 1, + type: 'directory' + }) + }) + + it('computes how much of the DAG is local', async () => { + const { cid: largeFileCid } = await importBytes(largeFile, blockstore) + const block = await blockstore.get(largeFileCid) + const node = dagPb.decode(block) + + expect(node.Links).to.have.lengthOf(2) + + await expect(fs.stat(largeFileCid)).to.eventually.include({ + fileSize: 490668n, + blocks: 3, + localDagSize: 490776n + }) + + // remove one of the blocks so we now have an incomplete DAG + await blockstore.delete(node.Links[0].Hash) + + // block count and local file/dag sizes should be smaller + await expect(fs.stat(largeFileCid)).to.eventually.include({ + fileSize: 490668n, + blocks: 2, + localFileSize: 228524n, + localDagSize: 228632n + }) + }) + + it('stats a raw node', async () => { + const { cid: fileCid } = await importBytes(smallFile, blockstore) + + await expect(fs.stat(fileCid)).to.eventually.include({ + fileSize: BigInt(smallFile.length), + dagSize: 13n, + blocks: 1, + type: 'raw' + }) + }) + + it('stats a small file', async () => { + const { cid: fileCid } = await importBytes(smallFile, blockstore, { + cidVersion: 0, + rawLeaves: false + }) + + await expect(fs.stat(fileCid)).to.eventually.include({ + fileSize: BigInt(smallFile.length), + dagSize: 19n, + blocks: 1, + type: 'file' + }) + }) + + it('stats a large file', async () => { + const { cid } = await importBytes(largeFile, blockstore) + + await expect(fs.stat(cid)).to.eventually.include({ + fileSize: BigInt(largeFile.length), + dagSize: 490682n, + blocks: 3, + type: 'file' + }) + }) + + it('should stat file with mode', async () => { + const mode = 0o644 + const { cid } = await importContent({ + content: smallFile, + mode + }, blockstore) + + await expect(fs.stat(cid)).to.eventually.include({ + mode + }) + }) + + it('should stat file with mtime', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + const { cid } = await importContent({ + content: smallFile, + mtime + }, blockstore) + + await expect(fs.stat(cid)).to.eventually.deep.include({ + mtime + }) + }) + + it('should stat a directory', async function () { + await expect(fs.stat(emptyDirCid)).to.eventually.include({ + type: 'directory', + blocks: 1, + fileSize: 0n + }) + }) + + it('should stat dir with mode', async function () { + const mode = 0o755 + const path = 'test-dir' + const dirCid = await fs.mkdir(emptyDirCid, path, { + mode + }) + + await expect(fs.stat(dirCid, { + path + })).to.eventually.include({ + mode + }) + }) + + it('should stat dir with mtime', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + + const path = 'test-dir' + const dirCid = await fs.mkdir(emptyDirCid, path, { + mtime + }) + + await expect(fs.stat(dirCid, { + path + })).to.eventually.deep.include({ + mtime + }) + }) + + it('sstats a sharded directory', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + const shardedDirCid = await createShardedDirectory(blockstore) + const updatedShardCid = await fs.touch(shardedDirCid, { + mtime + }) + + const stat = await fs.stat(updatedShardCid) + expect(stat).to.have.property('type', 'directory') + expect(stat).to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + expect(stat).to.include({ + mode: 0o755 + }) + expect(stat).to.deep.include({ + mtime + }) + }) + + it('stats a file inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore, { + rawLeaves: false + }) + const fileName = `small-file-${Math.random()}.txt` + const updatedShardCid = await fs.cp(fileCid, shardedDirCid, fileName) + + const stats = await fs.stat(updatedShardCid, { + path: fileName + }) + + expect(stats.type).to.equal('file') + expect(stats.fileSize).to.equal(4n) + }) +}) diff --git a/test/touch.spec.ts b/test/touch.spec.ts new file mode 100644 index 00000000..52a75dcb --- /dev/null +++ b/test/touch.spec.ts @@ -0,0 +1,150 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import type { CID } from 'multiformats/cid' +import delay from 'delay' +import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' + +describe('.files.touch', () => { + let blockstore: Blockstore + let fs: UnixFS + let emptyDirCid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + + const imported = await importContent({ path: 'empty' }, blockstore) + emptyDirCid = imported.cid + }) + + it('should have default mtime', async () => { + const { cid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + + await expect(fs.stat(cid)).to.eventually.have.property('mtime') + .that.is.undefined() + + const updatedCid = await fs.touch(cid) + + await expect(fs.stat(updatedCid)).to.eventually.have.property('mtime') + .that.is.not.undefined().and.does.not.deep.equal({ + secs: 0, + nsecs: 0 + }) + }) + + it('should update file mtime', async function () { + this.slow(5 * 1000) + const mtime = new Date() + const seconds = BigInt(Math.floor(mtime.getTime() / 1000)) + + const { cid } = await importContent({ + content: Uint8Array.from([0, 1, 2, 3, 4]), + mtime: { + secs: seconds + } + }, blockstore) + + await delay(2000) + const updatedCid = await fs.touch(cid) + + await expect(fs.stat(updatedCid)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + }) + + it('should update directory mtime', async function () { + this.slow(5 * 1000) + const path = 'path' + const mtime = new Date() + const seconds = BigInt(Math.floor(mtime.getTime() / 1000)) + + const cid = await fs.mkdir(emptyDirCid, path, { + mtime: { + secs: seconds + } + }) + await delay(2000) + const updateCid = await fs.touch(cid) + + await expect(fs.stat(updateCid)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + }) + + it('should update mtime recursively', async function () { + this.slow(5 * 1000) + const path = 'path' + const mtime = new Date() + const seconds = Math.floor(mtime.getTime() / 1000) + + const { cid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + const dirCid = await fs.cp(cid, emptyDirCid, path) + + await delay(2000) + + const updatedCid = await fs.touch(dirCid, { + recursive: true + }) + + await expect(fs.stat(updatedCid)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + + await expect(fs.stat(updatedCid, { + path + })).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + }) + + it('should update the mtime for a hamt-sharded-directory', async function () { + this.slow(5 * 1000) + const shardedDirCid = await createShardedDirectory(blockstore) + let updatedShardCid = await fs.touch(shardedDirCid) + const originalMtime = (await fs.stat(updatedShardCid)).mtime + + if (originalMtime == null) { + throw new Error('No originalMtime found') + } + + await delay(2000) + updatedShardCid = await fs.touch(shardedDirCid) + const updatedMtime = (await fs.stat(updatedShardCid)).mtime + + if (updatedMtime == null) { + throw new Error('No updatedMtime found') + } + + // no bigint support + expect(updatedMtime.secs).to.satisfy((s: bigint) => s > originalMtime.secs) + }) + + it('should update mtime recursively for a hamt-sharded-directory', async function () { + this.slow(5 * 1000) + const mtime = new Date() + const seconds = Math.floor(mtime.getTime() / 1000) + const shardedDirCid = await createShardedDirectory(blockstore) + + await delay(2000) + + const updatedCid = await fs.touch(shardedDirCid, { + recursive: true + }) + + await expect(fs.stat(updatedCid)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + + for await (const file of fs.ls(updatedCid)) { + expect(file).to.have.nested.property('unixfs.mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + } + }) +}) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} From 83e5e0e0ccfd27f9371c9a8940c237e398e9b68f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 16 Feb 2023 11:25:49 +0100 Subject: [PATCH 013/253] docs: update readme --- README.md | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index dc448fc9..239e4e49 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,11 @@ ## Table of contents -- - [Install](#install) - - [Browser ` ``` -

- - Helia logo - -

- -# @helia/unixfs - -[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) -[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) - ## API Docs - From d893bc26aa231f3b27e9b2072dfe579cecc7f65e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 16 Feb 2023 10:34:42 +0000 Subject: [PATCH 014/253] chore(release): 1.0.0 [skip ci] ## 1.0.0 (2023-02-16) ### Features * initial implementation ([#1](https://github.com/ipfs/helia-unixfs/issues/1)) ([adb6b0e](https://github.com/ipfs/helia-unixfs/commit/adb6b0e2626a3bdd08cdc4445e3367f104bc5bb8)) ### Trivial Changes * initial commit ([35e8a8f](https://github.com/ipfs/helia-unixfs/commit/35e8a8fd7c1ca68b21320b95211304bf01b30086)) * Update .github/workflows/stale.yml [skip ci] ([bcb060d](https://github.com/ipfs/helia-unixfs/commit/bcb060d880175ab885479388049a1ca2e5873629)) ### Documentation * update readme ([83e5e0e](https://github.com/ipfs/helia-unixfs/commit/83e5e0e0ccfd27f9371c9a8940c237e398e9b68f)) --- CHANGELOG.md | 17 +++++++++++++++++ package.json | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..b1fc3432 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +## 1.0.0 (2023-02-16) + + +### Features + +* initial implementation ([#1](https://github.com/ipfs/helia-unixfs/issues/1)) ([adb6b0e](https://github.com/ipfs/helia-unixfs/commit/adb6b0e2626a3bdd08cdc4445e3367f104bc5bb8)) + + +### Trivial Changes + +* initial commit ([35e8a8f](https://github.com/ipfs/helia-unixfs/commit/35e8a8fd7c1ca68b21320b95211304bf01b30086)) +* Update .github/workflows/stale.yml [skip ci] ([bcb060d](https://github.com/ipfs/helia-unixfs/commit/bcb060d880175ab885479388049a1ca2e5873629)) + + +### Documentation + +* update readme ([83e5e0e](https://github.com/ipfs/helia-unixfs/commit/83e5e0e0ccfd27f9371c9a8940c237e398e9b68f)) diff --git a/package.json b/package.json index eb974593..6a2f5da3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "0.0.0", + "version": "1.0.0", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", From f6edeca471da4aaa2171b0b3f2d2ea91d527a00e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 17 Feb 2023 08:18:35 +0100 Subject: [PATCH 015/253] fix: update unixfs importer --- package.json | 2 +- src/commands/chmod.ts | 4 +--- src/commands/touch.ts | 4 +--- test/cat.spec.ts | 4 ++-- test/chmod.spec.ts | 4 ++-- test/cp.spec.ts | 4 ++-- test/ls.spec.ts | 4 ++-- test/mkdir.spec.ts | 6 +++--- test/rm.spec.ts | 4 ++-- test/stat.spec.ts | 8 ++++---- test/touch.spec.ts | 6 +++--- 11 files changed, 23 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 6a2f5da3..2ba3fa78 100644 --- a/package.json +++ b/package.json @@ -148,7 +148,7 @@ "interface-blockstore": "^4.0.1", "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^12.0.0", - "ipfs-unixfs-importer": "^14.0.0", + "ipfs-unixfs-importer": "^14.0.1", "it-last": "^2.0.0", "it-pipe": "^2.0.5", "merge-options": "^3.0.4", diff --git a/src/commands/chmod.ts b/src/commands/chmod.ts index 2f3f6105..ea17a0af 100644 --- a/src/commands/chmod.ts +++ b/src/commands/chmod.ts @@ -63,10 +63,8 @@ export async function chmod (cid: CID, mode: number, blockstore: Blockstore, opt } } }, - // @ts-expect-error we account for the incompatible source type with our custom dag builder below (source) => importer(source, blockstore, { ...opts, - pin: false, dagBuilder: async function * (source, block) { for await (const entry of source) { yield async function () { @@ -87,7 +85,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blockstore, opt return { cid: updatedCid, - size: buf.length, + size: BigInt(buf.length), path: entry.path, unixfs } diff --git a/src/commands/touch.ts b/src/commands/touch.ts index 379778fb..cbed2557 100644 --- a/src/commands/touch.ts +++ b/src/commands/touch.ts @@ -67,10 +67,8 @@ export async function touch (cid: CID, blockstore: Blockstore, options: Partial< } } }, - // @ts-expect-error we account for the incompatible source type with our custom dag builder below (source) => importer(source, blockstore, { ...opts, - pin: false, dagBuilder: async function * (source, block) { for await (const entry of source) { yield async function () { @@ -91,7 +89,7 @@ export async function touch (cid: CID, blockstore: Blockstore, options: Partial< return { cid: updatedCid, - size: buf.length, + size: BigInt(buf.length), path: entry.path, unixfs } diff --git a/test/cat.spec.ts b/test/cat.spec.ts index 3f670010..68134b12 100644 --- a/test/cat.spec.ts +++ b/test/cat.spec.ts @@ -7,7 +7,7 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import toBuffer from 'it-to-buffer' import drain from 'it-drain' -import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) @@ -22,7 +22,7 @@ describe('cat', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) diff --git a/test/chmod.spec.ts b/test/chmod.spec.ts index e9ce3cfc..165c5ecb 100644 --- a/test/chmod.spec.ts +++ b/test/chmod.spec.ts @@ -5,7 +5,7 @@ import type { Blockstore } from 'interface-blockstore' import { MemoryBlockstore } from 'blockstore-core' import { UnixFS, unixfs } from '../src/index.js' import type { CID } from 'multiformats/cid' -import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) @@ -20,7 +20,7 @@ describe('chmod', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) diff --git a/test/cp.spec.ts b/test/cp.spec.ts index b849325e..2d11301d 100644 --- a/test/cp.spec.ts +++ b/test/cp.spec.ts @@ -8,7 +8,7 @@ import type { Blockstore } from 'interface-blockstore' import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import toBuffer from 'it-to-buffer' -import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import first from 'it-first' @@ -22,7 +22,7 @@ describe('cp', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) diff --git a/test/ls.spec.ts b/test/ls.spec.ts index 6c316ae3..2eb55abe 100644 --- a/test/ls.spec.ts +++ b/test/ls.spec.ts @@ -6,7 +6,7 @@ import all from 'it-all' import type { Blockstore } from 'interface-blockstore' import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' -import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' describe('ls', () => { @@ -19,7 +19,7 @@ describe('ls', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) diff --git a/test/mkdir.spec.ts b/test/mkdir.spec.ts index eb94437b..1caa94ec 100644 --- a/test/mkdir.spec.ts +++ b/test/mkdir.spec.ts @@ -7,7 +7,7 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' import type { Mtime } from 'ipfs-unixfs' -import { importContent } from 'ipfs-unixfs-importer' +import { importDirectory } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' describe('mkdir', () => { @@ -21,10 +21,10 @@ describe('mkdir', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid - const importedV0 = await importContent({ path: 'empty' }, blockstore, { + const importedV0 = await importDirectory({ path: 'empty' }, blockstore, { cidVersion: 0 }) emptyDirCidV0 = importedV0.cid diff --git a/test/rm.spec.ts b/test/rm.spec.ts index 5b14c35e..b67c7463 100644 --- a/test/rm.spec.ts +++ b/test/rm.spec.ts @@ -5,7 +5,7 @@ import type { Blockstore } from 'interface-blockstore' import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' -import { importContent, importBytes, importer } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes, importer } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import last from 'it-last' import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' @@ -22,7 +22,7 @@ describe('rm', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) diff --git a/test/stat.spec.ts b/test/stat.spec.ts index 041768b5..b55fae25 100644 --- a/test/stat.spec.ts +++ b/test/stat.spec.ts @@ -6,7 +6,7 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' -import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes, importFile } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) @@ -24,7 +24,7 @@ describe('stat', function () { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) @@ -100,7 +100,7 @@ describe('stat', function () { it('should stat file with mode', async () => { const mode = 0o644 - const { cid } = await importContent({ + const { cid } = await importFile({ content: smallFile, mode }, blockstore) @@ -115,7 +115,7 @@ describe('stat', function () { secs: 5n, nsecs: 0 } - const { cid } = await importContent({ + const { cid } = await importFile({ content: smallFile, mtime }, blockstore) diff --git a/test/touch.spec.ts b/test/touch.spec.ts index 52a75dcb..bd124e0d 100644 --- a/test/touch.spec.ts +++ b/test/touch.spec.ts @@ -6,7 +6,7 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' import delay from 'delay' -import { importContent, importBytes } from 'ipfs-unixfs-importer' +import { importDirectory, importBytes, importFile } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' describe('.files.touch', () => { @@ -19,7 +19,7 @@ describe('.files.touch', () => { fs = unixfs({ blockstore }) - const imported = await importContent({ path: 'empty' }, blockstore) + const imported = await importDirectory({ path: 'empty' }, blockstore) emptyDirCid = imported.cid }) @@ -43,7 +43,7 @@ describe('.files.touch', () => { const mtime = new Date() const seconds = BigInt(Math.floor(mtime.getTime() / 1000)) - const { cid } = await importContent({ + const { cid } = await importFile({ content: Uint8Array.from([0, 1, 2, 3, 4]), mtime: { secs: seconds From 08eccf94aa3475957b0b3426264437e007a1308f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Feb 2023 07:26:35 +0000 Subject: [PATCH 016/253] chore(release): 1.0.1 [skip ci] ## [1.0.1](https://github.com/ipfs/helia-unixfs/compare/v1.0.0...v1.0.1) (2023-02-17) ### Bug Fixes * update unixfs importer ([f6edeca](https://github.com/ipfs/helia-unixfs/commit/f6edeca471da4aaa2171b0b3f2d2ea91d527a00e)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1fc3432..973e1f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.0.1](https://github.com/ipfs/helia-unixfs/compare/v1.0.0...v1.0.1) (2023-02-17) + + +### Bug Fixes + +* update unixfs importer ([f6edeca](https://github.com/ipfs/helia-unixfs/commit/f6edeca471da4aaa2171b0b3f2d2ea91d527a00e)) + ## 1.0.0 (2023-02-16) diff --git a/package.json b/package.json index 2ba3fa78..5ed76948 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.0.0", + "version": "1.0.1", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", From 917a564c0d990dfd35d4615436fc8e3609c72a76 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 24 Feb 2023 13:48:06 +0000 Subject: [PATCH 017/253] fix: add methods to import data (#3) In order to have a more consistent developer experience, wrap the `ipfs-unixfs-importer` methods to allow adding data from the `UnixFS` interface. --- src/commands/add.ts | 34 ++++ src/index.ts | 384 +++++++++++++++++++++++++++++++++++++++++++- test/add.spec.ts | 110 +++++++++++++ 3 files changed, 525 insertions(+), 3 deletions(-) create mode 100644 src/commands/add.ts create mode 100644 test/add.spec.ts diff --git a/src/commands/add.ts b/src/commands/add.ts new file mode 100644 index 00000000..e37e95ea --- /dev/null +++ b/src/commands/add.ts @@ -0,0 +1,34 @@ +import type { CID } from 'multiformats/cid' +import type { Blockstore } from 'interface-blockstore' +import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer' + +export async function * addAll (source: ImportCandidateStream, blockstore: Blockstore, options: Partial = {}): AsyncGenerator { + yield * importer(source, blockstore, options) +} + +export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importBytes(bytes, blockstore, options) + + return cid +} + +export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importByteStream(bytes, blockstore, options) + + return cid +} + +export async function addFile (file: FileCandidate, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importFile(file, blockstore, options) + + return cid +} + +export async function addDirectory (dir: Partial, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importDirectory({ + ...dir, + path: dir.path ?? '-' + }, blockstore, options) + + return cid +} diff --git a/src/index.ts b/src/index.ts index a2c6d733..23b05d36 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,40 @@ +/** + * @packageDocumentation + * + * `@helia/unixfs` is an implementation of a {@link https://github.com/ipfs/specs/blob/main/UNIXFS.md UnixFS filesystem} compatible with {@link https://github.com/ipfs/helia Helia}. + * + * See the {@link UnixFS UnixFS interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { unixfs } from '@helia/unixfs' + * + * const helia = createHelia({ + * // ... helia config + * }) + * const fs = unixfs(helia) + * + * // create an empty dir and a file, then add the file to the dir + * const emptyDirCid = await fs.addDirectory() + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const updateDirCid = await fs.cp(fileCid, emptyDirCid, 'foo.txt') + * + * // or doing the same thing as a stream + * for await (const entry of fs.addAll([{ + * path: 'foo.txt', + * content: Uint8Array.from([0, 1, 2, 3]) + * }])) { + * console.info(entry) + * } + * ``` + */ + import type { CID, Version } from 'multiformats/cid' import type { Blockstore } from 'interface-blockstore' import type { AbortOptions } from '@libp2p/interfaces' +import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js' import { cat } from './commands/cat.js' import { mkdir } from './commands/mkdir.js' import type { Mtime } from 'ipfs-unixfs' @@ -11,50 +45,147 @@ import { touch } from './commands/touch.js' import { chmod } from './commands/chmod.js' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' import { ls } from './commands/ls.js' +import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportResult } from 'ipfs-unixfs-importer' export interface UnixFSComponents { blockstore: Blockstore } +/** + * Options to pass to the cat command + */ export interface CatOptions extends AbortOptions { + /** + * Start reading the file at this offset + */ offset?: number + + /** + * Stop reading the file after this many bytes + */ length?: number + + /** + * An optional path to allow reading files inside directories + */ path?: string } +/** + * Options to pass to the chmod command + */ export interface ChmodOptions extends AbortOptions { + /** + * If the target of the operation is a directory and this is true, + * apply the new mode to all directory contents + */ recursive: boolean + + /** + * Optional path to set the mode on directory contents + */ path?: string + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the cp command + */ export interface CpOptions extends AbortOptions { + /** + * If true, allow overwriting existing directory entries (default: false) + */ force: boolean + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the ls command + */ export interface LsOptions extends AbortOptions { + /** + * Optional path to list subdirectory contents if the target CID resolves to + * a directory + */ path?: string + + /** + * Start reading the directory entries at this offset + */ offset?: number + + /** + * Stop reading the directory contents after this many directory entries + */ length?: number } +/** + * Options to pass to the mkdir command + */ export interface MkdirOptions extends AbortOptions { + /** + * The CID version to create the new directory with - defaults to the same + * version as the containing directory + */ cidVersion: Version + + /** + * If true, allow overwriting existing directory entries (default: false) + */ force: boolean + + /** + * An optional mode to set on the new directory + */ mode?: number + + /** + * An optional mtime to set on the new directory + */ mtime?: Mtime + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the rm command + */ export interface RmOptions extends AbortOptions { + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the stat command + */ export interface StatOptions extends AbortOptions { + /** + * An optional path to allow statting paths inside directories + */ path?: string } +/** + * Statistics relating to a UnixFS DAG + */ export interface UnixFSStats { /** * The file or directory CID @@ -109,21 +240,245 @@ export interface UnixFSStats { unixfs?: import('ipfs-unixfs').UnixFS } +/** + * Options to pass to the touch command + */ export interface TouchOptions extends AbortOptions { + /** + * Optional mtime to set on the DAG root, defaults to the current time + */ mtime?: Mtime + + /** + * Optional path to set mtime on directory contents + */ path?: string + + /** + * If the DAG is a directory and this is true, update the mtime on all contents + */ recursive: boolean + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * The UnixFS interface provides familiar filesystem operations to make working with + * UnixFS DAGs simple and intuitive. + */ export interface UnixFS { + /** + * Add all files and directories from the passed stream. This method wraps the + * `importer` export from the `ipfs-unixfs-importer` module - please see the docs + * for input/output types. + * + * @example + * + * ```typescript + * const source = [{ + * path: './foo.txt', + * content: Uint8Array.from([0, 1, 2, 3]) + * }, { + * path: './bar.txt', + * content: Uint8Array.from([4, 5, 6, 7]) + * }] + * + * for await (const entry of fs.import(source)) { + * console.info(entry) + * } + * ``` + */ + addAll: (source: ImportCandidateStream, options?: Partial) => AsyncIterable + + /** + * Add a single `Uint8Array` to your Helia node as a file. + * + * @example + * + * ```typescript + * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * + * console.info(cid) + * ``` + */ + addBytes: (bytes: Uint8Array, options?: Partial) => Promise + + /** + * Add a stream of `Uint8Array` to your Helia node as a file. + * + * @example + * + * ```typescript + * import fs from 'node:fs' + * + * const stream = fs.createReadStream('./foo.txt') + * const cid = await fs.addByteStream(stream) + * + * console.info(cid) + * ``` + */ + addByteStream: (bytes: ByteStream, options?: Partial) => Promise + + /** + * Add a file to your Helia node with optional metadata. + * + * @example + * + * ```typescript + * const cid = await fs.addFile({ + * path: './foo.txt' + * content: Uint8Array.from([0, 1, 2, 3]), + * mode: 0x755, + * mtime: { + * secs: 10n, + * nsecs: 0 + * } + * }) + * + * console.info(cid) + * ``` + */ + addFile: (file: FileCandidate, options?: Partial) => Promise + + /** + * Add a directory to your Helia node. + * + * @example + * + * ```typescript + * const cid = await fs.addDirectory() + * + * console.info(cid) + * ``` + */ + addDirectory: (dir?: Partial, options?: Partial) => Promise + + /** + * Retrieve the contents of a file from your Helia node. + * + * @example + * + * ```typescript + * for await (const buf of fs.cat(cid)) { + * console.info(buf) + * } + * ``` + */ cat: (cid: CID, options?: Partial) => AsyncIterable - chmod: (source: CID, mode: number, options?: Partial) => Promise + + /** + * Change the permissions on a file or directory in a DAG + * + * @example + * + * ```typescript + * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const beforeStats = await fs.stat(beforeCid) + * + * const afterCid = await fs.chmod(cid, 0x755) + * const afterStats = await fs.stat(afterCid) + * + * console.info(beforeCid, beforeStats) + * console.info(afterCid, afterStats) + * ``` + */ + chmod: (cid: CID, mode: number, options?: Partial) => Promise + + /** + * Add a file or directory to a target directory. + * + * @example + * + * ```typescript + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const directoryCid = await fs.addDirectory() + * + * const updatedCid = await fs.cp(fileCid, directoryCid, 'foo.txt') + * + * console.info(updatedCid) + * ``` + */ cp: (source: CID, target: CID, name: string, options?: Partial) => Promise + + /** + * List directory contents. + * + * @example + * + * ```typescript + * for await (const entry of fs.ls(directoryCid)) { + * console.info(etnry) + * } + * ``` + */ ls: (cid: CID, options?: Partial) => AsyncIterable + + /** + * Make a new directory under an existing directory. + * + * @example + * + * ```typescript + * const directoryCid = await fs.addDirectory() + * + * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') + * + * console.info(updatedCid) + * ``` + */ mkdir: (cid: CID, dirname: string, options?: Partial) => Promise + + /** + * Remove a file or directory from an existing directory. + * + * @example + * + * ```typescript + * const directoryCid = await fs.addDirectory() + * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') + * + * const finalCid = await fs.rm(updatedCid, 'new-dir') + * + * console.info(finalCid) + * ``` + */ rm: (cid: CID, path: string, options?: Partial) => Promise + + /** + * Return statistics about a UnixFS DAG. + * + * @example + * + * ```typescript + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * + * const stats = await fs.stat(fileCid) + * + * console.info(stats) + * ``` + */ stat: (cid: CID, options?: Partial) => Promise + + /** + * Update the mtime of a UnixFS DAG + * + * @example + * + * ```typescript + * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const beforeStats = await fs.stat(beforeCid) + * + * const afterCid = await fs.touch(beforeCid) + * const afterStats = await fs.stat(afterCid) + * + * console.info(beforeCid, beforeStats) + * console.info(afterCid, afterStats) + * ``` + */ touch: (cid: CID, options?: Partial) => Promise } @@ -134,12 +489,32 @@ class DefaultUnixFS implements UnixFS { this.components = components } + async * addAll (source: ImportCandidateStream, options: Partial = {}): AsyncIterable { + yield * addAll(source, this.components.blockstore, options) + } + + async addBytes (bytes: Uint8Array, options: Partial = {}): Promise { + return await addBytes(bytes, this.components.blockstore, options) + } + + async addByteStream (bytes: ByteStream, options: Partial = {}): Promise { + return await addByteStream(bytes, this.components.blockstore, options) + } + + async addFile (file: FileCandidate, options: Partial = {}): Promise { + return await addFile(file, this.components.blockstore, options) + } + + async addDirectory (dir: Partial = {}, options: Partial = {}): Promise { + return await addDirectory(dir, this.components.blockstore, options) + } + async * cat (cid: CID, options: Partial = {}): AsyncIterable { yield * cat(cid, this.components.blockstore, options) } - async chmod (source: CID, mode: number, options: Partial = {}): Promise { - return await chmod(source, mode, this.components.blockstore, options) + async chmod (cid: CID, mode: number, options: Partial = {}): Promise { + return await chmod(cid, mode, this.components.blockstore, options) } async cp (source: CID, target: CID, name: string, options: Partial = {}): Promise { @@ -167,6 +542,9 @@ class DefaultUnixFS implements UnixFS { } } +/** + * Create a {@link UnixFS} instance for use with {@link https://github.com/ipfs/helia Helia} + */ export function unixfs (helia: { blockstore: Blockstore }): UnixFS { return new DefaultUnixFS(helia) } diff --git a/test/add.spec.ts b/test/add.spec.ts new file mode 100644 index 00000000..ebc4ba84 --- /dev/null +++ b/test/add.spec.ts @@ -0,0 +1,110 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import all from 'it-all' + +describe('addAll', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds a stream of files', async () => { + const output = await all(fs.addAll([{ + path: './foo.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + path: './bar.txt', + content: Uint8Array.from([5, 4, 3, 2, 1]) + }])) + + expect(output).to.have.lengthOf(2) + expect(output[0].cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + expect(output[1].cid.toString()).to.equal('bafkreidmuy2n45xj3cdknzprtzo2uvgm3hak6mzy5sllxty457agsftd34') + }) +}) + +describe('addBytes', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds bytes', async () => { + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) + + expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + }) +}) + +describe('addByteStream', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds bytes', async () => { + const cid = await fs.addByteStream([Uint8Array.from([0, 1, 2, 3, 4])]) + + expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + }) +}) + +describe('addFile', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds a file', async () => { + const cid = await fs.addFile({ + content: Uint8Array.from([0, 1, 2, 3, 4]) + }) + + expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + }) +}) + +describe('addDirectory', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds an empty directory with cidv0', async () => { + const cid = await fs.addDirectory({}, { + cidVersion: 0 + }) + + expect(cid.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + }) + + it('adds an empty directory with no args', async () => { + const cid = await fs.addDirectory() + + expect(cid.toString()).to.equal('bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354') + }) +}) From e973120b0b1266cf43e096beaa4a5ed90cb88c32 Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Fri, 24 Feb 2023 14:50:09 +0100 Subject: [PATCH 018/253] ci: update CI config files (#2) --- .github/workflows/automerge.yml | 3 +++ .github/workflows/js-test-and-release.yml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index d57c2a02..3833fc22 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -1,3 +1,6 @@ +# File managed by web3-bot. DO NOT EDIT. +# See https://github.com/protocol/.github/ for details. + name: Automerge on: [ pull_request ] diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 081b272a..0c396fbd 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,3 +1,6 @@ +# File managed by web3-bot. DO NOT EDIT. +# See https://github.com/protocol/.github/ for details. + name: test & maybe release on: push: @@ -139,5 +142,5 @@ jobs: docker-username: ${{ secrets.DOCKER_USERNAME }} - run: npm run --if-present release env: - GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} From 1b760847a18b7b7c1e3fa8c871fd75acb298480b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 24 Feb 2023 14:49:09 +0000 Subject: [PATCH 019/253] test: move test file data to fixtures --- test/cat.spec.ts | 3 +-- test/chmod.spec.ts | 3 +-- test/fixtures/files.ts | 4 ++++ test/rm.spec.ts | 3 +-- test/stat.spec.ts | 4 +--- 5 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/files.ts diff --git a/test/cat.spec.ts b/test/cat.spec.ts index 68134b12..3f938ce4 100644 --- a/test/cat.spec.ts +++ b/test/cat.spec.ts @@ -9,8 +9,7 @@ import toBuffer from 'it-to-buffer' import drain from 'it-drain' import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' - -const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) +import { smallFile } from './fixtures/files.js' describe('cat', () => { let blockstore: Blockstore diff --git a/test/chmod.spec.ts b/test/chmod.spec.ts index 165c5ecb..53652a61 100644 --- a/test/chmod.spec.ts +++ b/test/chmod.spec.ts @@ -7,8 +7,7 @@ import { UnixFS, unixfs } from '../src/index.js' import type { CID } from 'multiformats/cid' import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' - -const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) +import { smallFile } from './fixtures/files.js' describe('chmod', () => { let blockstore: Blockstore diff --git a/test/fixtures/files.ts b/test/fixtures/files.ts new file mode 100644 index 00000000..ba4f28b0 --- /dev/null +++ b/test/fixtures/files.ts @@ -0,0 +1,4 @@ + +export const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100)) + +export const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) diff --git a/test/rm.spec.ts b/test/rm.spec.ts index b67c7463..6052347c 100644 --- a/test/rm.spec.ts +++ b/test/rm.spec.ts @@ -9,8 +9,7 @@ import { importDirectory, importBytes, importer } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import last from 'it-last' import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' - -const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) +import { smallFile } from './fixtures/files.js' describe('rm', () => { let blockstore: Blockstore diff --git a/test/stat.spec.ts b/test/stat.spec.ts index b55fae25..d2e5d71b 100644 --- a/test/stat.spec.ts +++ b/test/stat.spec.ts @@ -8,9 +8,7 @@ import type { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' import { importDirectory, importBytes, importFile } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' - -const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) -const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100)) +import { largeFile, smallFile } from './fixtures/files.js' describe('stat', function () { this.timeout(120 * 1000) From 9d4799bb9e19f1137463261af6cb57256769c930 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 24 Feb 2023 14:54:43 +0000 Subject: [PATCH 020/253] chore(release): 1.0.2 [skip ci] ## [1.0.2](https://github.com/ipfs/helia-unixfs/compare/v1.0.1...v1.0.2) (2023-02-24) ### Bug Fixes * add methods to import data ([#3](https://github.com/ipfs/helia-unixfs/issues/3)) ([917a564](https://github.com/ipfs/helia-unixfs/commit/917a564c0d990dfd35d4615436fc8e3609c72a76)) ### Tests * move test file data to fixtures ([1b76084](https://github.com/ipfs/helia-unixfs/commit/1b760847a18b7b7c1e3fa8c871fd75acb298480b)) --- CHANGELOG.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 973e1f4d..5703e9ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## [1.0.2](https://github.com/ipfs/helia-unixfs/compare/v1.0.1...v1.0.2) (2023-02-24) + + +### Bug Fixes + +* add methods to import data ([#3](https://github.com/ipfs/helia-unixfs/issues/3)) ([917a564](https://github.com/ipfs/helia-unixfs/commit/917a564c0d990dfd35d4615436fc8e3609c72a76)) + + +### Tests + +* move test file data to fixtures ([1b76084](https://github.com/ipfs/helia-unixfs/commit/1b760847a18b7b7c1e3fa8c871fd75acb298480b)) + ## [1.0.1](https://github.com/ipfs/helia-unixfs/compare/v1.0.0...v1.0.1) (2023-02-17) diff --git a/package.json b/package.json index 5ed76948..168e7211 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.0.1", + "version": "1.0.2", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", From 97744606d6da2e61a1aefa6af8f0f3b68f8827ab Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Sat, 25 Feb 2023 15:21:07 +0000 Subject: [PATCH 021/253] fix: enable last shard tests (#4) Fixes up traversing deep shards --- src/commands/utils/hamt-utils.ts | 6 +++--- test/rm.spec.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands/utils/hamt-utils.ts b/src/commands/utils/hamt-utils.ts index 90147ba5..431c283f 100644 --- a/src/commands/utils/hamt-utils.ts +++ b/src/commands/utils/hamt-utils.ts @@ -96,6 +96,7 @@ export const recreateInitialHamtLevel = async (links: PBLink[]): Promise { expect(containingDirCid).to.eql(importerCid) }) - it.skip('results in the same hash as a sharded directory created by the importer when removing a subshard', async function () { + it('results in the same hash as a sharded directory created by the importer when removing a subshard', async function () { let { containingDirCid, fileName, @@ -195,7 +195,7 @@ describe('rm', () => { expect(containingDirCid).to.eql(importerCid) }) - it.skip('results in the same hash as a sharded directory created by the importer when removing a subshard of a subshard', async function () { + it('results in the same hash as a sharded directory created by the importer when removing a subshard of a subshard', async function () { let { containingDirCid, fileName, From 3f552085a94f3f46c7991feb5f24bcfcde4d0f08 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 25 Feb 2023 15:26:10 +0000 Subject: [PATCH 022/253] chore(release): 1.0.3 [skip ci] ## [1.0.3](https://github.com/ipfs/helia-unixfs/compare/v1.0.2...v1.0.3) (2023-02-25) ### Bug Fixes * enable last shard tests ([#4](https://github.com/ipfs/helia-unixfs/issues/4)) ([9774460](https://github.com/ipfs/helia-unixfs/commit/97744606d6da2e61a1aefa6af8f0f3b68f8827ab)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5703e9ff..65b9aa27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.0.3](https://github.com/ipfs/helia-unixfs/compare/v1.0.2...v1.0.3) (2023-02-25) + + +### Bug Fixes + +* enable last shard tests ([#4](https://github.com/ipfs/helia-unixfs/issues/4)) ([9774460](https://github.com/ipfs/helia-unixfs/commit/97744606d6da2e61a1aefa6af8f0f3b68f8827ab)) + ## [1.0.2](https://github.com/ipfs/helia-unixfs/compare/v1.0.1...v1.0.2) (2023-02-24) diff --git a/package.json b/package.json index 168e7211..e50ce1e4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.0.2", + "version": "1.0.3", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", From 52d4786831c3b2b60322de562b752ecfbc8791bb Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 27 Feb 2023 16:00:21 +0000 Subject: [PATCH 023/253] fix: simplify shard handling (#5) Reduces duplication between adding and removing links to sharded directories by making them use the same code to trace a path to the operation target without loading the entire shard. --- package.json | 4 +- src/commands/utils/add-link.ts | 277 ++++++-------- src/commands/utils/consumable-hash.ts | 174 +++++++++ src/commands/utils/hamt-utils.ts | 366 +++++++------------ src/commands/utils/remove-link.ts | 148 ++------ src/commands/utils/resolve.ts | 11 +- test/cat.spec.ts | 17 +- test/chmod.spec.ts | 10 +- test/cp.spec.ts | 27 +- test/fixtures/create-sharded-directory.ts | 2 +- test/fixtures/create-subsharded-directory.ts | 9 +- test/ls.spec.ts | 12 +- test/mkdir.spec.ts | 8 +- test/rm.spec.ts | 34 +- test/stat.spec.ts | 22 +- test/touch.spec.ts | 12 +- 16 files changed, 545 insertions(+), 588 deletions(-) create mode 100644 src/commands/utils/consumable-hash.ts diff --git a/package.json b/package.json index e50ce1e4..9cbceffb 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,6 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "next", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", "@libp2p/logger": "^2.0.5", @@ -152,7 +151,8 @@ "it-last": "^2.0.0", "it-pipe": "^2.0.5", "merge-options": "^3.0.4", - "multiformats": "^11.0.1" + "multiformats": "^11.0.1", + "sparse-array": "^1.3.2" }, "devDependencies": { "aegir": "^38.1.0", diff --git a/src/commands/utils/add-link.ts b/src/commands/utils/add-link.ts index b8f15101..0eb7566f 100644 --- a/src/commands/utils/add-link.ts +++ b/src/commands/utils/add-link.ts @@ -2,32 +2,31 @@ import * as dagPB from '@ipld/dag-pb' import { CID, Version } from 'multiformats/cid' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' -import { DirSharded } from './dir-sharded.js' import { - updateHamtDirectory, - recreateHamtLevel, - recreateInitialHamtLevel, createShard, + recreateShardedDirectory, toPrefix, - addLinksToHamtBucket + updateShardedDirectory } from './hamt-utils.js' -import last from 'it-last' import type { PBNode, PBLink } from '@ipld/dag-pb/interface' import { sha256 } from 'multiformats/hashes/sha2' -import type { Bucket } from 'hamt-sharding' import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from './errors.js' import type { ImportResult } from 'ipfs-unixfs-importer' import type { AbortOptions } from '@libp2p/interfaces' import type { Directory } from './cid-to-directory.js' import type { Blockstore } from 'interface-blockstore' import { isOverShardThreshold } from './is-over-shard-threshold.js' +import { hamtBucketBits, hamtHashFn } from './hamt-constants.js' +// @ts-expect-error no types +import SparseArray from 'sparse-array' +import { wrapHash } from './consumable-hash.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' const log = logger('helia:unixfs:components:utils:add-link') export interface AddLinkResult { node: PBNode cid: CID - size: number } export interface AddLinkOptions extends AbortOptions { @@ -81,7 +80,7 @@ const convertToShardedDirectory = async (parent: Directory, blockstore: Blocksto cidVersion: parent.cid.version }) - log(`Converted directory to sharded directory ${result.cid}`) + log(`converted directory to sharded directory ${result.cid}`) return result } @@ -134,187 +133,125 @@ const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Bloc return { node: parent.node, - cid, - size: buf.length + cid } } const addToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise => { - const { - shard, path - } = await addFileToShardedDirectory(parent, child, blockstore, options) - const result = await last(shard.flush(blockstore)) + const { path, hash } = await recreateShardedDirectory(parent.cid, child.Name, blockstore, options) + const finalSegment = path[path.length - 1] - if (result == null) { - throw new Error('No result from flushing shard') + if (finalSegment == null) { + throw new Error('Invalid HAMT, could not generate path') } - const block = await blockstore.get(result.cid) - const node = dagPB.decode(block) + // find the next prefix + // const index = await hash.take(hamtBucketBits) + const prefix = finalSegment.prefix + const index = parseInt(prefix, 16) - // we have written out the shard, but only one sub-shard will have been written so replace it in the original shard - const parentLinks = parent.node.Links.filter((link) => { - return (link.Name ?? '').substring(0, 2) !== path[0].prefix - }) - - const newLink = node.Links - .find(link => (link.Name ?? '').substring(0, 2) === path[0].prefix) - - if (newLink == null) { - throw new Error(`No link found with prefix ${path[0].prefix}`) - } - - parentLinks.push(newLink) - - return await updateHamtDirectory({ - Data: parent.node.Data, - Links: parentLinks - }, blockstore, path[0].bucket, options) -} + log('next prefix for %s is %s', child.Name, prefix) -const addFileToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise<{ shard: DirSharded, path: BucketPath[] }> => { - if (parent.node.Data == null) { - throw new InvalidPBNodeError('Parent node with no data passed to addFileToShardedDirectory') - } + const linkName = `${prefix}${child.Name}` + const existingLink = finalSegment.node.Links.find(l => (l.Name ?? '').startsWith(prefix)) - // start at the root bucket and descend, loading nodes as we go - const rootBucket = await recreateInitialHamtLevel(parent.node.Links) - const node = UnixFS.unmarshal(parent.node.Data) - - const shard = new DirSharded({ - root: true, - dir: true, - parent: undefined, - parentKey: undefined, - path: '', - dirty: true, - flat: false, - mode: node.mode - }, { - ...options, - cidVersion: parent.cid.version - }) - shard._bucket = rootBucket - - if (node.mtime != null) { - // update mtime if previously set - shard.mtime = { - secs: BigInt(Math.round(Date.now() / 1000)) - } - } - - // load subshards until the bucket & position no longer changes - const position = await rootBucket._findNewBucketAndPos(child.Name) - const path = toBucketPath(position) - path[0].node = parent.node - let index = 0 - - while (index < path.length) { - const segment = path[index] - index++ - const node = segment.node - - if (node == null) { - throw new Error('Segment had no node') - } - - const link = node.Links - .find(link => (link.Name ?? '').substring(0, 2) === segment.prefix) - - if (link == null) { - // prefix is new, file will be added to the current bucket - log(`Link ${segment.prefix}${child.Name} will be added`) - index = path.length - - break - } + if (existingLink != null) { + log('link %s was present in shard', linkName) + // link is already present in shard - if (link.Name === `${segment.prefix}${child.Name}`) { + if (existingLink.Name === linkName) { + // file with same name is already present in shard if (!options.allowOverwriting) { throw new AlreadyExistsError() } - // file already existed, file will be added to the current bucket - log(`Link ${segment.prefix}${child.Name} will be replaced`) - index = path.length - - break - } - - if ((link.Name ?? '').length > 2) { - // another file had the same prefix, will be replaced with a subshard - log(`Link ${link.Name} ${link.Hash} will be replaced with a subshard`) - index = path.length - - break - } - - // load sub-shard - log(`Found subshard ${segment.prefix}`) - const block = await blockstore.get(link.Hash) - const subShard = dagPB.decode(block) - - // subshard hasn't been loaded, descend to the next level of the HAMT - if (path[index] == null) { - log(`Loaded new subshard ${segment.prefix}`) - await recreateHamtLevel(blockstore, subShard.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options) - - const position = await rootBucket._findNewBucketAndPos(child.Name) - - path.push({ - bucket: position.bucket, - prefix: toPrefix(position.pos), - node: subShard + log('overwriting %s in subshard', child.Name) + finalSegment.node.Links = finalSegment.node.Links.filter(l => l.Name !== linkName) + finalSegment.node.Links.push({ + Name: linkName, + Hash: child.Hash, + Tsize: child.Tsize }) + } else if (existingLink.Name?.length === 2) { + throw new Error('Existing link was subshard?!') + } else { + // conflict, add a new HAMT segment + log('prefix %s already exists, creating new subshard', prefix) + // find the sibling we are going to replace + const index = finalSegment.node.Links.findIndex(l => l.Name?.startsWith(prefix)) + const sibling = finalSegment.node.Links.splice(index, 1)[0] + + // give the sibling a new HAMT prefix + const siblingName = (sibling.Name ?? '').substring(2) + const wrapped = wrapHash(hamtHashFn) + const siblingHash = wrapped(uint8ArrayFromString(siblingName)) + + // discard hash bits until we reach the subshard depth + for (let i = 0; i < path.length; i++) { + await siblingHash.take(hamtBucketBits) + } - break + while (true) { + const siblingIndex = await siblingHash.take(hamtBucketBits) + const siblingPrefix = toPrefix(siblingIndex) + sibling.Name = `${siblingPrefix}${siblingName}` + + // calculate the target file's HAMT prefix in the new sub-shard + const newIndex = await hash.take(hamtBucketBits) + const newPrefix = toPrefix(newIndex) + + if (siblingPrefix === newPrefix) { + // the two sibling names have caused another conflict - add an intermediate node to + // the HAMT and try again + + // create the child locations + const children = new SparseArray() + children.set(newIndex, true) + + path.push({ + prefix: newPrefix, + children, + node: { + Links: [] + } + }) + + continue + } + + // create the child locations + const children = new SparseArray() + children.set(newIndex, true) + children.set(siblingIndex, true) + + // add our new segment + path.push({ + prefix, + children, + node: { + Links: [ + sibling, { + Name: `${newPrefix}${child.Name}`, + Hash: child.Hash, + Tsize: child.Tsize + } + ] + } + }) + + break + } } + } else { + log('link %s was not present in sub-shard', linkName) - const nextSegment = path[index] - - // add next levels worth of links to bucket - await addLinksToHamtBucket(blockstore, subShard.Links, nextSegment.bucket, rootBucket, options) - - nextSegment.node = subShard - } - - // finally add the new file into the shard - await shard._bucket.put(child.Name, { - size: BigInt(child.Tsize), - cid: child.Hash - }) - - return { - shard, path - } -} - -export interface BucketPath { - bucket: Bucket - prefix: string - node?: PBNode -} - -const toBucketPath = (position: { pos: number, bucket: Bucket }): BucketPath[] => { - const path = [{ - bucket: position.bucket, - prefix: toPrefix(position.pos) - }] - - let bucket = position.bucket._parent - let positionInBucket = position.bucket._posAtParent - - while (bucket != null) { - path.push({ - bucket, - prefix: toPrefix(positionInBucket) - }) + // add new link to shard + child.Name = linkName + finalSegment.node.Links.push(child) + finalSegment.children.set(index, true) - positionInBucket = bucket._posAtParent - bucket = bucket._parent + log('adding %s to existing sub-shard', linkName) } - path.reverse() - - return path + return await updateShardedDirectory(path, blockstore, options) } diff --git a/src/commands/utils/consumable-hash.ts b/src/commands/utils/consumable-hash.ts new file mode 100644 index 00000000..b7363bfa --- /dev/null +++ b/src/commands/utils/consumable-hash.ts @@ -0,0 +1,174 @@ +import { concat as uint8ArrayConcat } from 'uint8arrays/concat' + +export function wrapHash (hashFn: (value: Uint8Array) => Promise): (value: InfiniteHash | Uint8Array) => InfiniteHash { + function hashing (value: InfiniteHash | Uint8Array): InfiniteHash { + if (value instanceof InfiniteHash) { + // already a hash. return it + return value + } else { + return new InfiniteHash(value, hashFn) + } + } + + return hashing +} + +export class InfiniteHash { + _value: Uint8Array + _hashFn: (value: Uint8Array) => Promise + _depth: number + _availableBits: number + _currentBufferIndex: number + _buffers: ConsumableBuffer[] + + constructor (value: Uint8Array, hashFn: (value: Uint8Array) => Promise) { + if (!(value instanceof Uint8Array)) { + throw new Error('can only hash Uint8Arrays') + } + + this._value = value + this._hashFn = hashFn + this._depth = -1 + this._availableBits = 0 + this._currentBufferIndex = 0 + this._buffers = [] + } + + async take (bits: number): Promise { + let pendingBits = bits + + while (this._availableBits < pendingBits) { + await this._produceMoreBits() + } + + let result = 0 + + while (pendingBits > 0) { + const hash = this._buffers[this._currentBufferIndex] + const available = Math.min(hash.availableBits(), pendingBits) + const took = hash.take(available) + result = (result << available) + took + pendingBits -= available + this._availableBits -= available + + if (hash.availableBits() === 0) { + this._currentBufferIndex++ + } + } + + return result + } + + untake (bits: number): void { + let pendingBits = bits + + while (pendingBits > 0) { + const hash = this._buffers[this._currentBufferIndex] + const availableForUntake = Math.min(hash.totalBits() - hash.availableBits(), pendingBits) + hash.untake(availableForUntake) + pendingBits -= availableForUntake + this._availableBits += availableForUntake + + if (this._currentBufferIndex > 0 && hash.totalBits() === hash.availableBits()) { + this._depth-- + this._currentBufferIndex-- + } + } + } + + async _produceMoreBits (): Promise { + this._depth++ + + const value = this._depth > 0 ? uint8ArrayConcat([this._value, Uint8Array.from([this._depth])]) : this._value + const hashValue = await this._hashFn(value) + const buffer = new ConsumableBuffer(hashValue) + + this._buffers.push(buffer) + this._availableBits += buffer.availableBits() + } +} + +const START_MASKS = [ + 0b11111111, + 0b11111110, + 0b11111100, + 0b11111000, + 0b11110000, + 0b11100000, + 0b11000000, + 0b10000000 +] + +const STOP_MASKS = [ + 0b00000001, + 0b00000011, + 0b00000111, + 0b00001111, + 0b00011111, + 0b00111111, + 0b01111111, + 0b11111111 +] + +export class ConsumableBuffer { + _value: Uint8Array + _currentBytePos: number + _currentBitPos: number + + constructor (value: Uint8Array) { + this._value = value + this._currentBytePos = value.length - 1 + this._currentBitPos = 7 + } + + availableBits (): number { + return this._currentBitPos + 1 + this._currentBytePos * 8 + } + + totalBits (): number { + return this._value.length * 8 + } + + take (bits: number): number { + let pendingBits = bits + let result = 0 + while (pendingBits > 0 && this._haveBits()) { + const byte = this._value[this._currentBytePos] + const availableBits = this._currentBitPos + 1 + const taking = Math.min(availableBits, pendingBits) + const value = byteBitsToInt(byte, availableBits - taking, taking) + result = (result << taking) + value + + pendingBits -= taking + + this._currentBitPos -= taking + if (this._currentBitPos < 0) { + this._currentBitPos = 7 + this._currentBytePos-- + } + } + + return result + } + + untake (bits: number): void { + this._currentBitPos += bits + while (this._currentBitPos > 7) { + this._currentBitPos -= 8 + this._currentBytePos += 1 + } + } + + _haveBits (): boolean { + return this._currentBytePos >= 0 + } +} + +function byteBitsToInt (byte: number, start: number, length: number): number { + const mask = maskFor(start, length) + return (byte & mask) >>> start +} + +function maskFor (start: number, length: number): number { + return START_MASKS[start] & STOP_MASKS[Math.min(length + start - 1, 7)] +} diff --git a/src/commands/utils/hamt-utils.ts b/src/commands/utils/hamt-utils.ts index 431c283f..7df9a771 100644 --- a/src/commands/utils/hamt-utils.ts +++ b/src/commands/utils/hamt-utils.ts @@ -1,8 +1,4 @@ import * as dagPB from '@ipld/dag-pb' -import { - Bucket, - createHAMT -} from 'hamt-sharding' import { DirSharded } from './dir-sharded.js' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' @@ -13,131 +9,23 @@ import { hamtHashFn, hamtBucketBits } from './hamt-constants.js' -import type { PBLink, PBNode } from '@ipld/dag-pb/interface' import type { Blockstore } from 'interface-blockstore' import type { Mtime } from 'ipfs-unixfs' -import type { Directory } from './cid-to-directory.js' import type { AbortOptions } from '@libp2p/interfaces' import type { ImportResult } from 'ipfs-unixfs-importer' import { persist } from './persist.js' +import { InfiniteHash, wrapHash } from './consumable-hash.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +// @ts-expect-error no types +import SparseArray from 'sparse-array' +import type { PersistOptions } from './persist.js' const log = logger('helia:unixfs:commands:utils:hamt-utils') -export interface UpdateHamtResult { - node: PBNode - cid: CID - size: number -} - export interface UpdateHamtDirectoryOptions extends AbortOptions { cidVersion: Version } -export const updateHamtDirectory = async (pbNode: PBNode, blockstore: Blockstore, bucket: Bucket, options: UpdateHamtDirectoryOptions): Promise => { - if (pbNode.Data == null) { - throw new Error('Could not update HAMT directory because parent had no data') - } - - // update parent with new bit field - const node = UnixFS.unmarshal(pbNode.Data) - const dir = new UnixFS({ - type: 'hamt-sharded-directory', - data: Uint8Array.from(bucket._children.bitField().reverse()), - fanout: BigInt(bucket.tableSize()), - hashType: hamtHashCode, - mode: node.mode, - mtime: node.mtime - }) - - const updatedPbNode = { - Data: dir.marshal(), - Links: pbNode.Links - } - - const buf = dagPB.encode(dagPB.prepare(updatedPbNode)) - const cid = await persist(buf, blockstore, options) - - return { - node: updatedPbNode, - cid, - size: pbNode.Links.reduce((sum, link) => sum + (link.Tsize ?? 0), buf.byteLength) - } -} - -export const recreateHamtLevel = async (blockstore: Blockstore, links: PBLink[], rootBucket: Bucket, parentBucket: Bucket, positionAtParent: number, options: AbortOptions): Promise> => { - // recreate this level of the HAMT - const bucket = new Bucket({ - hash: rootBucket._options.hash, - bits: rootBucket._options.bits - }, parentBucket, positionAtParent) - parentBucket._putObjectAt(positionAtParent, bucket) - - await addLinksToHamtBucket(blockstore, links, bucket, rootBucket, options) - - return bucket -} - -export const recreateInitialHamtLevel = async (links: PBLink[]): Promise> => { - const bucket = createHAMT({ - hashFn: hamtHashFn, - bits: hamtBucketBits - }) - - // populate sub bucket but do not recurse as we do not want to load the whole shard - await Promise.all( - links.map(async link => { - const linkName = (link.Name ?? '') - - if (linkName.length === 2) { - const pos = parseInt(linkName, 16) - const subBucket = new Bucket({ - hash: bucket._options.hash, - bits: bucket._options.bits - }, bucket, pos) - - bucket._putObjectAt(pos, subBucket) - return - } - - await bucket.put(linkName.substring(2), { - size: link.Tsize, - cid: link.Hash - }) - }) - ) - - return bucket -} - -export const addLinksToHamtBucket = async (blockstore: Blockstore, links: PBLink[], bucket: Bucket, rootBucket: Bucket, options: AbortOptions): Promise => { - await Promise.all( - links.map(async link => { - const linkName = (link.Name ?? '') - - if (linkName.length === 2) { - log('Populating sub bucket', linkName) - const pos = parseInt(linkName, 16) - const block = await blockstore.get(link.Hash, options) - const node = dagPB.decode(block) - - const subBucket = new Bucket({ - hash: rootBucket._options.hash, - bits: rootBucket._options.bits - }, bucket, pos) - bucket._putObjectAt(pos, subBucket) - - await addLinksToHamtBucket(blockstore, node.Links, subBucket, rootBucket, options) - return - } - - await rootBucket.put(linkName.substring(2), { - size: link.Tsize, - cid: link.Hash - }) - }) - ) -} - export const toPrefix = (position: number): string => { return position .toString(16) @@ -146,149 +34,175 @@ export const toPrefix = (position: number): string => { .substring(0, 2) } -export interface HamtPathSegment { - bucket?: Bucket - prefix?: string - node?: PBNode - cid?: CID - size?: number +export interface CreateShardOptions { + mtime?: Mtime + mode?: number + cidVersion: Version } -export const generatePath = async (root: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise => { - // start at the root bucket and descend, loading nodes as we go - const rootBucket = await recreateInitialHamtLevel(root.node.Links) - const position = await rootBucket._findNewBucketAndPos(name) - const path: HamtPathSegment[] = [{ - bucket: position.bucket, - prefix: toPrefix(position.pos) - }] - let currentBucket = position.bucket - - while (currentBucket !== rootBucket) { - path.push({ - bucket: currentBucket, - prefix: toPrefix(currentBucket._posAtParent) +export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise => { + const shard = new DirSharded({ + root: true, + dir: true, + parent: undefined, + parentKey: undefined, + path: '', + dirty: true, + flat: false, + mtime: options.mtime, + mode: options.mode + }, options) + + for (let i = 0; i < contents.length; i++) { + await shard._bucket.put(contents[i].name, { + size: contents[i].size, + cid: contents[i].cid }) + } - if (currentBucket._parent == null) { - break - } + const res = await last(shard.flush(blockstore)) - currentBucket = currentBucket._parent + if (res == null) { + throw new Error('Flushing shard yielded no result') } - // add the root bucket to the path - path.push({ - bucket: rootBucket, - node: root.node - }) + return res +} + +export interface HAMTPath { + prefix: string + children: SparseArray + node: dagPB.PBNode +} + +export const updateShardedDirectory = async (path: HAMTPath[], blockstore: Blockstore, options: PersistOptions): Promise<{ cid: CID, node: dagPB.PBNode }> => { + // persist any metadata on the shard root + const shardRoot = UnixFS.unmarshal(path[0].node.Data ?? new Uint8Array(0)) + // this is always the same + const fanout = BigInt(Math.pow(2, hamtBucketBits)) + + // start from the leaf and ascend to the root path.reverse() - // load PbNode for each path segment - for (let i = 1; i < path.length; i++) { - const segment = path[i] - const previousSegment = path[i - 1] + let cid: CID | undefined + let node: dagPB.PBNode | undefined - if (previousSegment.node == null) { - throw new Error('Could not generate HAMT path') - } + for (let i = 0; i < path.length; i++) { + const isRoot = i === path.length - 1 + const segment = path[i] - // find prefix in links - const link = previousSegment.node.Links - .filter(link => (link.Name ?? '').substring(0, 2) === segment.prefix) - .pop() + // go-ipfs uses little endian, that's why we have to + // reverse the bit field before storing it + const data = Uint8Array.from(segment.children.bitField().reverse()) + const dir = new UnixFS({ + type: 'hamt-sharded-directory', + data, + fanout, + hashType: hamtHashCode + }) - // entry was not in shard - if (link == null) { - // reached bottom of tree, file will be added to the current bucket - log(`Link ${segment.prefix}${name} will be added`) - // return path - continue + if (isRoot) { + dir.mtime = shardRoot.mtime + dir.mode = shardRoot.mode } - const linkName = link.Name ?? '' - - // found entry - if (linkName === `${segment.prefix}${name}`) { - log(`Link ${segment.prefix}${name} will be replaced`) - // file already existed, file will be added to the current bucket - // return path - continue + node = { + Data: dir.marshal(), + Links: segment.node.Links } - // found subshard - log(`Found subshard ${segment.prefix}`) - const block = await blockstore.get(link.Hash) - const node = segment.node = dagPB.decode(block) + const block = dagPB.encode(dagPB.prepare(node)) - // subshard hasn't been loaded, descend to the next level of the HAMT - if (path[i + 1] == null) { - log(`Loaded new subshard ${segment.prefix}`) + cid = await persist(block, blockstore, options) - if (segment.bucket == null || segment.prefix == null) { - throw new Error('Shard was invalid') + if (!isRoot) { + // update link in parent sub-shard + const nextSegment = path[i + 1] + + if (nextSegment == null) { + throw new Error('Was not operating on shard root but also had no parent?') } - await recreateHamtLevel(blockstore, node.Links, rootBucket, segment.bucket, parseInt(segment.prefix, 16), options) - const position = await rootBucket._findNewBucketAndPos(name) + log('updating link in parent sub-shard with prefix %s', nextSegment.prefix) - // i-- - path.push({ - bucket: position.bucket, - prefix: toPrefix(position.pos), - node + nextSegment.node.Links = nextSegment.node.Links.filter(l => l.Name !== nextSegment.prefix) + nextSegment.node.Links.push({ + Name: nextSegment.prefix, + Hash: cid, + Tsize: segment.node.Links.reduce((acc, curr) => acc + (curr.Tsize ?? 0), block.byteLength) }) - - continue - } - - if (segment.bucket == null) { - throw new Error('Shard was invalid') } - - // add intermediate links to bucket - await addLinksToHamtBucket(blockstore, node.Links, segment.bucket, rootBucket, options) } - await rootBucket.put(name, true) - - path.reverse() + if (cid == null || node == null) { + throw new Error('Noting persisted') + } - return path + return { cid, node } } -export interface CreateShardOptions { - mtime?: Mtime - mode?: number - cidVersion: Version -} +export const recreateShardedDirectory = async (cid: CID, fileName: string, blockstore: Blockstore, options: AbortOptions): Promise<{ path: HAMTPath[], hash: InfiniteHash }> => { + const wrapped = wrapHash(hamtHashFn) + const hash = wrapped(uint8ArrayFromString(fileName)) + const path: HAMTPath[] = [] -export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise => { - const shard = new DirSharded({ - root: true, - dir: true, - parent: undefined, - parentKey: undefined, - path: '', - dirty: true, - flat: false, - mtime: options.mtime, - mode: options.mode - }, options) + // descend the HAMT, loading each layer as we head towards the target child + while (true) { + const block = await blockstore.get(cid, options) + const node = dagPB.decode(block) + const children = new SparseArray() + const index = await hash.take(hamtBucketBits) + const prefix = toPrefix(index) - for (let i = 0; i < contents.length; i++) { - await shard._bucket.put(contents[i].name, { - size: contents[i].size, - cid: contents[i].cid + path.push({ + prefix, + children, + node }) - } - const res = await last(shard.flush(blockstore)) + let childLink: dagPB.PBLink | undefined - if (res == null) { - throw new Error('Flushing shard yielded no result') + // update sparsearray child layout - the bitfield is used as the data field for the + // intermediate DAG node so this is required to generate consistent hashes + for (const link of node.Links) { + const linkName = link.Name ?? '' + + if (linkName.length < 2) { + throw new Error('Invalid HAMT - link name was too short') + } + + const position = parseInt(linkName.substring(0, 2), 16) + children.set(position, true) + + // we found the child we are looking for + if (linkName.startsWith(prefix)) { + childLink = link + } + } + + if (childLink == null) { + log('no link found with prefix %s for %s', prefix, fileName) + // hash.untake(hamtBucketBits) + break + } + + const linkName = childLink.Name ?? '' + + if (linkName.length < 2) { + throw new Error('Invalid HAMT - link name was too short') + } + + if (linkName.length === 2) { + // found sub-shard + cid = childLink.Hash + log('descend into sub-shard with prefix %s', linkName) + + continue + } + + break } - return res + return { path, hash } } diff --git a/src/commands/utils/remove-link.ts b/src/commands/utils/remove-link.ts index e9765d39..8b2a9a57 100644 --- a/src/commands/utils/remove-link.ts +++ b/src/commands/utils/remove-link.ts @@ -4,10 +4,9 @@ import type { CID, Version } from 'multiformats/cid' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' import { - generatePath, - HamtPathSegment, - updateHamtDirectory, - UpdateHamtDirectoryOptions + recreateShardedDirectory, + UpdateHamtDirectoryOptions, + updateShardedDirectory } from './hamt-utils.js' import type { PBNode } from '@ipld/dag-pb' import type { Blockstore } from 'interface-blockstore' @@ -77,131 +76,60 @@ const removeFromDirectory = async (parent: Directory, name: string, blockstore: } const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ cid: CID, node: PBNode }> => { - const path = await generatePath(parent, name, blockstore, options) + const { path } = await recreateShardedDirectory(parent.cid, name, blockstore, options) + const finalSegment = path[path.length - 1] - // remove file from root bucket - const rootBucket = path[path.length - 1].bucket - - if (rootBucket == null) { - throw new Error('Could not generate HAMT path') + if (finalSegment == null) { + throw new Error('Invalid HAMT, could not generate path') } - await rootBucket.del(name) - - // update all nodes in the shard path - return await updateShard(path, name, blockstore, options) -} - -/** - * The `path` param is a list of HAMT path segments starting with th - */ -const updateShard = async (path: HamtPathSegment[], name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ node: PBNode, cid: CID }> => { - const fileName = `${path[0].prefix}${name}` - - // skip first path segment as it is the file to remove - for (let i = 1; i < path.length; i++) { - const lastPrefix = path[i - 1].prefix - const segment = path[i] - - if (segment.node == null) { - throw new InvalidParametersError('Path segment had no associated PBNode') - } - - const link = segment.node.Links - .find(link => (link.Name ?? '').substring(0, 2) === lastPrefix) + const linkName = finalSegment.node.Links.filter(l => (l.Name ?? '').substring(2) === name).map(l => l.Name).pop() - if (link == null) { - throw new InvalidParametersError(`No link found with prefix ${lastPrefix} for file ${name}`) - } + if (linkName == null) { + throw new Error('File not found') + } - if (link.Name == null) { - throw new InvalidParametersError(`${lastPrefix} link had no name`) - } + const prefix = linkName.substring(0, 2) + const index = parseInt(prefix, 16) - if (link.Name === fileName) { - log(`removing existing link ${link.Name}`) + // remove the file from the shard + finalSegment.node.Links = finalSegment.node.Links.filter(link => link.Name !== linkName) + finalSegment.children.unset(index) - const links = segment.node.Links.filter((nodeLink) => { - return nodeLink.Name !== link.Name - }) - - if (segment.bucket == null) { - throw new Error('Segment bucket was missing') + if (finalSegment.node.Links.length === 1) { + // replace the subshard with the last remaining file in the parent + while (true) { + if (path.length === 1) { + break } - await segment.bucket.del(name) - - const result = await updateHamtDirectory({ - Data: segment.node.Data, - Links: links - }, blockstore, segment.bucket, options) + const segment = path[path.length - 1] - segment.node = result.node - segment.cid = result.cid - segment.size = result.size - } - - if (link.Name === lastPrefix) { - log(`updating subshard with prefix ${lastPrefix}`) - - const lastSegment = path[i - 1] - - if (lastSegment.node?.Links.length === 1) { - log(`removing subshard for ${lastPrefix}`) - - // convert subshard back to normal file entry - const link = lastSegment.node.Links[0] - link.Name = `${lastPrefix}${(link.Name ?? '').substring(2)}` - - // remove existing prefix - segment.node.Links = segment.node.Links.filter((link) => { - return link.Name !== lastPrefix - }) - - // add new child - segment.node.Links.push(link) - } else { - // replace subshard entry - log(`replacing subshard for ${lastPrefix}`) + if (segment == null || segment.node.Links.length > 1) { + break + } - // remove existing prefix - segment.node.Links = segment.node.Links.filter((link) => { - return link.Name !== lastPrefix - }) + // remove final segment + path.pop() - if (lastSegment.cid == null) { - throw new Error('Did not persist previous segment') - } + const nextSegment = path[path.length - 1] - // add new child - segment.node.Links.push({ - Name: lastPrefix, - Hash: lastSegment.cid, - Tsize: lastSegment.size - }) + if (nextSegment == null) { + break } - if (segment.bucket == null) { - throw new Error('Segment bucket was missing') - } + const link = segment.node.Links[0] - const result = await updateHamtDirectory(segment.node, blockstore, segment.bucket, options) - segment.node = result.node - segment.cid = result.cid - segment.size = result.size + nextSegment.node.Links = nextSegment.node.Links.filter(l => !(l.Name ?? '').startsWith(nextSegment.prefix)) + nextSegment.node.Links.push({ + Hash: link.Hash, + Name: `${nextSegment.prefix}${(link.Name ?? '').substring(2)}`, + Tsize: link.Tsize + }) } } - const rootSegment = path[path.length - 1] - - if (rootSegment == null || rootSegment.cid == null || rootSegment.node == null) { - throw new InvalidParametersError('Failed to update shard') - } - - return { - cid: rootSegment.cid, - node: rootSegment.node - } + return await updateShardedDirectory(path, blockstore, options) } const convertToFlatDirectory = async (parent: Directory, blockstore: Blockstore, options: RmLinkOptions): Promise => { diff --git a/src/commands/utils/resolve.ts b/src/commands/utils/resolve.ts index 33530d83..55f9a79d 100644 --- a/src/commands/utils/resolve.ts +++ b/src/commands/utils/resolve.ts @@ -8,7 +8,7 @@ import { cidToDirectory } from './cid-to-directory.js' import { cidToPBLink } from './cid-to-pblink.js' import type { Blockstore } from 'interface-blockstore' -const log = logger('helia:unixfs:components:utils:add-link') +const log = logger('helia:unixfs:components:utils:resolve') export interface Segment { name: string @@ -33,12 +33,12 @@ export interface ResolveResult { } export async function resolve (cid: CID, path: string | undefined, blockstore: Blockstore, options: AbortOptions): Promise { - log('resolve "%s" under %c', path, cid) - if (path == null || path === '') { return { cid } } + log('resolve "%s" under %c', path, cid) + const parts = path.split('/').filter(Boolean) const segments: Segment[] = [{ name: '', @@ -50,6 +50,8 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B const part = parts[i] const result = await exporter(cid, blockstore, options) + log('resolving "%s"', part, result) + if (result.type === 'file') { if (i < parts.length - 1) { throw new InvalidParametersError('Path was invalid') @@ -62,6 +64,7 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B for await (const entry of result.content()) { if (entry.name === part) { dirCid = entry.cid + break } } @@ -81,6 +84,8 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B } } + log('resolved %s to %c', path, cid) + return { cid, path, diff --git a/test/cat.spec.ts b/test/cat.spec.ts index 3f938ce4..30b5d482 100644 --- a/test/cat.spec.ts +++ b/test/cat.spec.ts @@ -7,7 +7,6 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import toBuffer from 'it-to-buffer' import drain from 'it-drain' -import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { smallFile } from './fixtures/files.js' @@ -21,12 +20,11 @@ describe('cat', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('reads a small file', async () => { - const { cid } = await importBytes(smallFile, blockstore) + const cid = await fs.addBytes(smallFile) const bytes = await toBuffer(fs.cat(cid)) expect(bytes).to.equalBytes(smallFile) @@ -34,7 +32,7 @@ describe('cat', () => { it('reads a file with an offset', async () => { const offset = 10 - const { cid } = await importBytes(smallFile, blockstore) + const cid = await fs.addBytes(smallFile) const bytes = await toBuffer(fs.cat(cid, { offset })) @@ -44,7 +42,7 @@ describe('cat', () => { it('reads a file with a length', async () => { const length = 10 - const { cid } = await importBytes(smallFile, blockstore) + const cid = await fs.addBytes(smallFile) const bytes = await toBuffer(fs.cat(cid, { length })) @@ -55,7 +53,7 @@ describe('cat', () => { it('reads a file with an offset and a length', async () => { const offset = 2 const length = 5 - const { cid } = await importBytes(smallFile, blockstore) + const cid = await fs.addBytes(smallFile) const bytes = await toBuffer(fs.cat(cid, { offset, length @@ -70,9 +68,8 @@ describe('cat', () => { }) it('reads file from inside a sharded directory', async () => { - const content = Uint8Array.from([0, 1, 2, 3, 4]) const dirCid = await createShardedDirectory(blockstore) - const { cid: fileCid } = await importBytes(content, blockstore) + const fileCid = await fs.addBytes(smallFile) const path = 'new-file.txt' const updatedCid = await fs.cp(fileCid, dirCid, path) @@ -81,6 +78,6 @@ describe('cat', () => { path })) - expect(bytes).to.deep.equal(content) + expect(bytes).to.deep.equal(smallFile) }) }) diff --git a/test/chmod.spec.ts b/test/chmod.spec.ts index 53652a61..e76db519 100644 --- a/test/chmod.spec.ts +++ b/test/chmod.spec.ts @@ -5,7 +5,6 @@ import type { Blockstore } from 'interface-blockstore' import { MemoryBlockstore } from 'blockstore-core' import { UnixFS, unixfs } from '../src/index.js' import type { CID } from 'multiformats/cid' -import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { smallFile } from './fixtures/files.js' @@ -19,12 +18,11 @@ describe('chmod', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('should update the mode for a raw node', async () => { - const { cid } = await importBytes(smallFile, blockstore) + const cid = await fs.addBytes(smallFile) const originalMode = (await fs.stat(cid)).mode const updatedCid = await fs.chmod(cid, 0o777) @@ -34,7 +32,7 @@ describe('chmod', () => { }) it('should update the mode for a file', async () => { - const { cid } = await importBytes(smallFile, blockstore, { + const cid = await fs.addBytes(smallFile, { rawLeaves: false }) const originalMode = (await fs.stat(cid)).mode @@ -65,7 +63,7 @@ describe('chmod', () => { it('should update mode recursively', async () => { const path = 'path' - const { cid } = await importBytes(smallFile, blockstore) + const cid = await fs.addBytes(smallFile) const dirCid = await fs.cp(cid, emptyDirCid, path) const originalMode = (await fs.stat(dirCid, { path diff --git a/test/cp.spec.ts b/test/cp.spec.ts index 2d11301d..51fc9e9e 100644 --- a/test/cp.spec.ts +++ b/test/cp.spec.ts @@ -8,9 +8,9 @@ import type { Blockstore } from 'interface-blockstore' import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import toBuffer from 'it-to-buffer' -import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import first from 'it-first' +import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' describe('cp', () => { let blockstore: Blockstore @@ -22,8 +22,7 @@ describe('cp', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('refuses to copy files without a source', async () => { @@ -48,7 +47,7 @@ describe('cp', () => { it('refuses to copy files to an unreadable node', async () => { const hash = identity.digest(uint8ArrayFromString('derp')) - const { cid: source } = await importBytes(Uint8Array.from([0, 1, 3, 4]), blockstore) + const source = await fs.addBytes(Uint8Array.from([0, 1, 3, 4])) const target = CID.createV1(identity.code, hash) await expect(fs.cp(source, target, 'foo')).to.eventually.be.rejected @@ -65,7 +64,7 @@ describe('cp', () => { it('refuses to copy files to an existing file', async () => { const path = 'path' - const { cid: source } = await importBytes(Uint8Array.from([0, 1, 3, 4]), blockstore) + const source = await fs.addBytes(Uint8Array.from([0, 1, 3, 4])) const target = await fs.cp(source, emptyDirCid, path) await expect(fs.cp(source, target, path)).to.eventually.be.rejected @@ -80,7 +79,7 @@ describe('cp', () => { it('copies a file to new location', async () => { const data = Uint8Array.from([0, 1, 3, 4]) const path = 'path' - const { cid: source } = await importBytes(data, blockstore) + const source = await fs.addBytes(data) const dirCid = await fs.cp(source, emptyDirCid, path) const bytes = await toBuffer(fs.cat(dirCid, { @@ -134,7 +133,7 @@ describe('cp', () => { it('copies a file from a normal directory to a sharded directory', async () => { const shardedDirCid = await createShardedDirectory(blockstore) const path = `file-${Math.random()}.txt` - const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore, { + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 3, 4]), { rawLeaves: false }) @@ -166,4 +165,18 @@ describe('cp', () => { force: true })).to.eventually.be.ok() }) + + it('copies a file to a sharded directory that creates a subshard', async () => { + const { + containingDirCid, + fileName, + importerCid + } = await createSubshardedDirectory(blockstore) + + // adding a file to the importer CID should result in the shard with a subshard + const fileCid = CID.parse('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + const finalDirCid = await fs.cp(fileCid, importerCid, fileName) + + expect(finalDirCid).to.eql(containingDirCid, 'adding a file to the imported dir did not result in the same CID') + }) }) diff --git a/test/fixtures/create-sharded-directory.ts b/test/fixtures/create-sharded-directory.ts index ae5f5e43..0ba0d1b6 100644 --- a/test/fixtures/create-sharded-directory.ts +++ b/test/fixtures/create-sharded-directory.ts @@ -9,7 +9,7 @@ export async function createShardedDirectory (blockstore: Blockstore, files = 10 for (let i = 0; i < files; i++) { yield { path: `./file-${i}`, - content: Uint8Array.from([0, 1, 2, 3, 4, 5, i]) + content: Uint8Array.from([0, 1, 2, 3, 4]) } } }()), blockstore, { diff --git a/test/fixtures/create-subsharded-directory.ts b/test/fixtures/create-subsharded-directory.ts index 64bcd963..1ae1561c 100644 --- a/test/fixtures/create-subsharded-directory.ts +++ b/test/fixtures/create-subsharded-directory.ts @@ -1,6 +1,6 @@ import type { Blockstore } from 'interface-blockstore' -import { importBytes, importer } from 'ipfs-unixfs-importer' -import { CID } from 'multiformats/cid' +import { importer } from 'ipfs-unixfs-importer' +import type { CID } from 'multiformats/cid' import { unixfs } from '../../src/index.js' import * as dagPb from '@ipld/dag-pb' import last from 'it-last' @@ -11,9 +11,8 @@ export async function createSubshardedDirectory (blockstore: Blockstore, depth: fileName: string }> { const fs = unixfs({ blockstore }) - - const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) - let containingDirCid = CID.parse('bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354') + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) + let containingDirCid = await fs.addDirectory() let fileName: string | undefined let count = 0 diff --git a/test/ls.spec.ts b/test/ls.spec.ts index 2eb55abe..84ad860b 100644 --- a/test/ls.spec.ts +++ b/test/ls.spec.ts @@ -6,7 +6,6 @@ import all from 'it-all' import type { Blockstore } from 'interface-blockstore' import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' -import { importDirectory, importBytes } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' describe('ls', () => { @@ -19,8 +18,7 @@ describe('ls', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('should require a path', async () => { @@ -31,7 +29,7 @@ describe('ls', () => { it('lists files in a directory', async () => { const path = 'path' const data = Uint8Array.from([0, 1, 2, 3]) - const { cid: fileCid } = await importBytes(data, blockstore) + const fileCid = await fs.addBytes(data) const dirCid = await fs.cp(fileCid, emptyDirCid, path) const files = await all(fs.ls(dirCid)) @@ -46,7 +44,7 @@ describe('ls', () => { it('lists a file', async () => { const path = 'path' const data = Uint8Array.from([0, 1, 2, 3]) - const { cid: fileCid } = await importBytes(data, blockstore, { + const fileCid = await fs.addBytes(data, { rawLeaves: false }) const dirCid = await fs.cp(fileCid, emptyDirCid, path) @@ -64,7 +62,7 @@ describe('ls', () => { it('lists a raw node', async () => { const path = 'path' const data = Uint8Array.from([0, 1, 2, 3]) - const { cid: fileCid } = await importBytes(data, blockstore) + const fileCid = await fs.addBytes(data) const dirCid = await fs.cp(fileCid, emptyDirCid, path) const files = await all(fs.ls(dirCid, { path @@ -109,7 +107,7 @@ describe('ls', () => { const dirName = `subdir-${Math.random()}` const fileName = `small-file-${Math.random()}.txt` - const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore) + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) const containingDirectoryCid = await fs.cp(fileCid, emptyDirCid, fileName) const updatedShardCid = await fs.cp(containingDirectoryCid, shardedDirCid, dirName) diff --git a/test/mkdir.spec.ts b/test/mkdir.spec.ts index 1caa94ec..5d402fd1 100644 --- a/test/mkdir.spec.ts +++ b/test/mkdir.spec.ts @@ -7,7 +7,6 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' import type { Mtime } from 'ipfs-unixfs' -import { importDirectory } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' describe('mkdir', () => { @@ -21,13 +20,10 @@ describe('mkdir', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid - - const importedV0 = await importDirectory({ path: 'empty' }, blockstore, { + emptyDirCid = await fs.addDirectory() + emptyDirCidV0 = await fs.addDirectory({}, { cidVersion: 0 }) - emptyDirCidV0 = importedV0.cid }) async function testMode (mode: number | undefined, expectedMode: number): Promise { diff --git a/test/rm.spec.ts b/test/rm.spec.ts index e7b4cfd7..5cfa84fe 100644 --- a/test/rm.spec.ts +++ b/test/rm.spec.ts @@ -5,7 +5,7 @@ import type { Blockstore } from 'interface-blockstore' import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' -import { importDirectory, importBytes, importer } from 'ipfs-unixfs-importer' +import { importer } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import last from 'it-last' import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' @@ -21,8 +21,7 @@ describe('rm', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('refuses to remove files without arguments', async () => { @@ -32,7 +31,7 @@ describe('rm', () => { it('removes a file', async () => { const path = 'foo' - const { cid: fileCid } = await importBytes(smallFile, blockstore) + const fileCid = await fs.addBytes(smallFile) const dirCid = await fs.cp(fileCid, emptyDirCid, path) const updatedDirCid = await fs.rm(dirCid, path) @@ -88,6 +87,8 @@ describe('rm', () => { path: dirName })).to.eventually.be.rejected .with.property('code', 'ERR_DOES_NOT_EXIST') + + expect(updatedContainingDirCid.toString()).to.equal(shardedDirCid.toString(), 'adding and removing a file from a sharded directory did not result in the original sharded CID') }) it('results in the same hash as a sharded directory created by the importer when removing a file and removing the file means the root node is below the shard threshold', async function () { @@ -110,26 +111,26 @@ describe('rm', () => { await expect(fs.stat(importerCid)).to.eventually.have.nested.property('unixfs.type', 'directory') // create the same shard with unixfs command - const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) - let containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', { + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) + const containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', { shardSplitThresholdBytes }) await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory') - containingDirCid = await fs.cp(fileCid, containingDirCid, 'file-2.txt', { + const dirWithFileCid = await fs.cp(fileCid, containingDirCid, 'file-2.txt', { shardSplitThresholdBytes }) - await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + await expect(fs.stat(dirWithFileCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') - containingDirCid = await fs.rm(containingDirCid, 'file-2.txt', { + const dirWithoutFileCid = await fs.rm(dirWithFileCid, 'file-2.txt', { shardSplitThresholdBytes }) - await expect(fs.stat(containingDirCid)).to.eventually.have.nested.property('unixfs.type', 'directory') + await expect(fs.stat(dirWithoutFileCid)).to.eventually.have.nested.property('unixfs.type', 'directory') - expect(containingDirCid).to.eql(importerCid) + expect(dirWithoutFileCid).to.eql(importerCid) }) it('results in the same hash as a sharded directory created by the importer when removing a file', async function () { @@ -152,7 +153,7 @@ describe('rm', () => { await expect(fs.stat(importerCid)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') // create the same shard with unixfs command - const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) let containingDirCid = await fs.cp(fileCid, emptyDirCid, 'file-1.txt', { shardSplitThresholdBytes }) @@ -175,7 +176,7 @@ describe('rm', () => { }) it('results in the same hash as a sharded directory created by the importer when removing a subshard', async function () { - let { + const { containingDirCid, fileName, importerCid @@ -185,14 +186,15 @@ describe('rm', () => { .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') // remove the file that caused the subshard to be created and the CID should be the same as the importer - containingDirCid = await fs.rm(containingDirCid, fileName, { + const finalDirCid = await fs.rm(containingDirCid, fileName, { shardSplitThresholdBytes: 1 }) - await expect(fs.stat(containingDirCid)) + // should still be a shard + await expect(fs.stat(finalDirCid)) .to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') - expect(containingDirCid).to.eql(importerCid) + expect(finalDirCid).to.eql(importerCid, 'removing a file from the imported dir not result in the same CID') }) it('results in the same hash as a sharded directory created by the importer when removing a subshard of a subshard', async function () { diff --git a/test/stat.spec.ts b/test/stat.spec.ts index d2e5d71b..a3d3f10c 100644 --- a/test/stat.spec.ts +++ b/test/stat.spec.ts @@ -6,7 +6,6 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' -import { importDirectory, importBytes, importFile } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { largeFile, smallFile } from './fixtures/files.js' @@ -22,8 +21,7 @@ describe('stat', function () { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('stats an empty directory', async () => { @@ -36,7 +34,7 @@ describe('stat', function () { }) it('computes how much of the DAG is local', async () => { - const { cid: largeFileCid } = await importBytes(largeFile, blockstore) + const largeFileCid = await fs.addBytes(largeFile) const block = await blockstore.get(largeFileCid) const node = dagPb.decode(block) @@ -61,7 +59,7 @@ describe('stat', function () { }) it('stats a raw node', async () => { - const { cid: fileCid } = await importBytes(smallFile, blockstore) + const fileCid = await fs.addBytes(smallFile) await expect(fs.stat(fileCid)).to.eventually.include({ fileSize: BigInt(smallFile.length), @@ -72,7 +70,7 @@ describe('stat', function () { }) it('stats a small file', async () => { - const { cid: fileCid } = await importBytes(smallFile, blockstore, { + const fileCid = await fs.addBytes(smallFile, { cidVersion: 0, rawLeaves: false }) @@ -86,7 +84,7 @@ describe('stat', function () { }) it('stats a large file', async () => { - const { cid } = await importBytes(largeFile, blockstore) + const cid = await fs.addBytes(largeFile) await expect(fs.stat(cid)).to.eventually.include({ fileSize: BigInt(largeFile.length), @@ -98,10 +96,10 @@ describe('stat', function () { it('should stat file with mode', async () => { const mode = 0o644 - const { cid } = await importFile({ + const cid = await fs.addFile({ content: smallFile, mode - }, blockstore) + }) await expect(fs.stat(cid)).to.eventually.include({ mode @@ -113,10 +111,10 @@ describe('stat', function () { secs: 5n, nsecs: 0 } - const { cid } = await importFile({ + const cid = await fs.addFile({ content: smallFile, mtime - }, blockstore) + }) await expect(fs.stat(cid)).to.eventually.deep.include({ mtime @@ -186,7 +184,7 @@ describe('stat', function () { it('stats a file inside a sharded directory', async () => { const shardedDirCid = await createShardedDirectory(blockstore) - const { cid: fileCid } = await importBytes(Uint8Array.from([0, 1, 2, 3]), blockstore, { + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3]), { rawLeaves: false }) const fileName = `small-file-${Math.random()}.txt` diff --git a/test/touch.spec.ts b/test/touch.spec.ts index bd124e0d..0ccbead8 100644 --- a/test/touch.spec.ts +++ b/test/touch.spec.ts @@ -6,7 +6,6 @@ import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import type { CID } from 'multiformats/cid' import delay from 'delay' -import { importDirectory, importBytes, importFile } from 'ipfs-unixfs-importer' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' describe('.files.touch', () => { @@ -19,12 +18,11 @@ describe('.files.touch', () => { fs = unixfs({ blockstore }) - const imported = await importDirectory({ path: 'empty' }, blockstore) - emptyDirCid = imported.cid + emptyDirCid = await fs.addDirectory() }) it('should have default mtime', async () => { - const { cid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) await expect(fs.stat(cid)).to.eventually.have.property('mtime') .that.is.undefined() @@ -43,12 +41,12 @@ describe('.files.touch', () => { const mtime = new Date() const seconds = BigInt(Math.floor(mtime.getTime() / 1000)) - const { cid } = await importFile({ + const cid = await fs.addFile({ content: Uint8Array.from([0, 1, 2, 3, 4]), mtime: { secs: seconds } - }, blockstore) + }) await delay(2000) const updatedCid = await fs.touch(cid) @@ -83,7 +81,7 @@ describe('.files.touch', () => { const mtime = new Date() const seconds = Math.floor(mtime.getTime() / 1000) - const { cid } = await importBytes(Uint8Array.from([0, 1, 2, 3, 4]), blockstore) + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) const dirCid = await fs.cp(cid, emptyDirCid, path) await delay(2000) From 4369653892d1434b9519f8f7f93371ae4531bc21 Mon Sep 17 00:00:00 2001 From: Daniel Norman <1992255+2color@users.noreply.github.com> Date: Mon, 27 Feb 2023 17:01:02 +0100 Subject: [PATCH 024/253] docs: fix typos (#4) --- packages/ipns/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 2e13c946..5991e1ea 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -6,7 +6,7 @@ * @example * * ```typescript - * import { gossipsub } from '@chainsafe/libp2p' + * import { gossipsub } from '@chainsafe/libp2p-gossipsub' * import { kadDHT } from '@libp2p/kad-dht' * import { createLibp2p } from 'libp2p' * import { createHelia } from 'helia' @@ -31,7 +31,7 @@ * //.. other options * }) * const name = ipns(helia, [ - * dht(helia) + * dht(helia), * pubsub(helia) * ]) * From bda0c1e7fe0a600d608938567c3e2816dc2a7a06 Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Mon, 27 Feb 2023 17:02:20 +0100 Subject: [PATCH 025/253] ci: update CI config files (#3) --- .github/workflows/automerge.yml | 3 +++ .github/workflows/js-test-and-release.yml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index d57c2a02..3833fc22 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -1,3 +1,6 @@ +# File managed by web3-bot. DO NOT EDIT. +# See https://github.com/protocol/.github/ for details. + name: Automerge on: [ pull_request ] diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 081b272a..0c396fbd 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,3 +1,6 @@ +# File managed by web3-bot. DO NOT EDIT. +# See https://github.com/protocol/.github/ for details. + name: test & maybe release on: push: @@ -139,5 +142,5 @@ jobs: docker-username: ${{ secrets.DOCKER_USERNAME }} - run: npm run --if-present release env: - GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} From 4332c4cc928ea93a1c61ab3a75be4ee0dc5200f1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 27 Feb 2023 16:07:06 +0000 Subject: [PATCH 026/253] chore(release): 1.0.4 [skip ci] ## [1.0.4](https://github.com/ipfs/helia-unixfs/compare/v1.0.3...v1.0.4) (2023-02-27) ### Bug Fixes * simplify shard handling ([#5](https://github.com/ipfs/helia-unixfs/issues/5)) ([52d4786](https://github.com/ipfs/helia-unixfs/commit/52d4786831c3b2b60322de562b752ecfbc8791bb)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65b9aa27..0276802e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.0.4](https://github.com/ipfs/helia-unixfs/compare/v1.0.3...v1.0.4) (2023-02-27) + + +### Bug Fixes + +* simplify shard handling ([#5](https://github.com/ipfs/helia-unixfs/issues/5)) ([52d4786](https://github.com/ipfs/helia-unixfs/commit/52d4786831c3b2b60322de562b752ecfbc8791bb)) + ## [1.0.3](https://github.com/ipfs/helia-unixfs/compare/v1.0.2...v1.0.3) (2023-02-25) diff --git a/package.json b/package.json index 9cbceffb..26c35970 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.0.3", + "version": "1.0.4", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", From a6bd1983bd7baac21af3de6fa269219f52664cde Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 14 Mar 2023 11:35:55 +0100 Subject: [PATCH 027/253] fix: align defaults with filecoin (#6) Filecoin uses a different set of defaults for importing files which will be more performant for file transfers so use them. The increased `maxChildrenPerNode` means fewer levels in a DAG so the parallisation of DAG layer requests fetches more data and the increased chunk size means fewer siblings in a layer. Raw leaves removes some extra bytes from the leaf nodes. --- src/commands/add.ts | 43 ++++++++++++++++++++++++++++++----- src/commands/utils/persist.ts | 2 +- test/fixtures/files.ts | 3 ++- test/stat.spec.ts | 20 ++++++++-------- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/commands/add.ts b/src/commands/add.ts index e37e95ea..15010aed 100644 --- a/src/commands/add.ts +++ b/src/commands/add.ts @@ -1,25 +1,53 @@ import type { CID } from 'multiformats/cid' -import type { Blockstore } from 'interface-blockstore' +import type { Blockstore } from 'ipfs-unixfs-importer' import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer' +import { balanced } from 'ipfs-unixfs-importer/layout' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' + +/** + * Default importer settings match Filecoin + */ +const defaultImporterSettings: ImporterOptions = { + cidVersion: 1, + rawLeaves: true, + layout: balanced({ + maxChildrenPerNode: 1024 + }), + chunker: fixedSize({ + chunkSize: 1048576 + }) +} export async function * addAll (source: ImportCandidateStream, blockstore: Blockstore, options: Partial = {}): AsyncGenerator { - yield * importer(source, blockstore, options) + yield * importer(source, blockstore, { + ...defaultImporterSettings, + ...options + }) } export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, options: Partial = {}): Promise { - const { cid } = await importBytes(bytes, blockstore, options) + const { cid } = await importBytes(bytes, blockstore, { + ...defaultImporterSettings, + ...options + }) return cid } export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, options: Partial = {}): Promise { - const { cid } = await importByteStream(bytes, blockstore, options) + const { cid } = await importByteStream(bytes, blockstore, { + ...defaultImporterSettings, + ...options + }) return cid } export async function addFile (file: FileCandidate, blockstore: Blockstore, options: Partial = {}): Promise { - const { cid } = await importFile(file, blockstore, options) + const { cid } = await importFile(file, blockstore, { + ...defaultImporterSettings, + ...options + }) return cid } @@ -28,7 +56,10 @@ export async function addDirectory (dir: Partial, blockstore const { cid } = await importDirectory({ ...dir, path: dir.path ?? '-' - }, blockstore, options) + }, blockstore, { + ...defaultImporterSettings, + ...options + }) return cid } diff --git a/src/commands/utils/persist.ts b/src/commands/utils/persist.ts index 15f12752..567663a0 100644 --- a/src/commands/utils/persist.ts +++ b/src/commands/utils/persist.ts @@ -1,7 +1,7 @@ import { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' import { sha256 } from 'multiformats/hashes/sha2' -import type { Blockstore } from 'interface-blockstore' +import type { Blockstore } from 'ipfs-unixfs-importer' import type { BlockCodec } from 'multiformats/codecs/interface' import type { Version as CIDVersion } from 'multiformats/cid' diff --git a/test/fixtures/files.ts b/test/fixtures/files.ts index ba4f28b0..cce002d2 100644 --- a/test/fixtures/files.ts +++ b/test/fixtures/files.ts @@ -1,4 +1,5 @@ +const ONE_MEG = 1024 * 1024 -export const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100)) +export const largeFile = Uint8Array.from(new Array(ONE_MEG * 5).fill(0).map(() => Math.random() * 100)) export const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) diff --git a/test/stat.spec.ts b/test/stat.spec.ts index a3d3f10c..32aa7478 100644 --- a/test/stat.spec.ts +++ b/test/stat.spec.ts @@ -38,12 +38,12 @@ describe('stat', function () { const block = await blockstore.get(largeFileCid) const node = dagPb.decode(block) - expect(node.Links).to.have.lengthOf(2) + expect(node.Links).to.have.lengthOf(5) await expect(fs.stat(largeFileCid)).to.eventually.include({ - fileSize: 490668n, - blocks: 3, - localDagSize: 490776n + fileSize: 5242880n, + blocks: 6, + localDagSize: 5243139n }) // remove one of the blocks so we now have an incomplete DAG @@ -51,10 +51,10 @@ describe('stat', function () { // block count and local file/dag sizes should be smaller await expect(fs.stat(largeFileCid)).to.eventually.include({ - fileSize: 490668n, - blocks: 2, - localFileSize: 228524n, - localDagSize: 228632n + fileSize: 5242880n, + blocks: 5, + localFileSize: 4194304n, + localDagSize: 4194563n }) }) @@ -88,8 +88,8 @@ describe('stat', function () { await expect(fs.stat(cid)).to.eventually.include({ fileSize: BigInt(largeFile.length), - dagSize: 490682n, - blocks: 3, + dagSize: 5242907n, + blocks: 6, type: 'file' }) }) From aff56dda5944bd7f2204249468cdd7c838e647f0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 14 Mar 2023 10:40:39 +0000 Subject: [PATCH 028/253] chore(release): 1.0.5 [skip ci] ## [1.0.5](https://github.com/ipfs/helia-unixfs/compare/v1.0.4...v1.0.5) (2023-03-14) ### Bug Fixes * align defaults with filecoin ([#6](https://github.com/ipfs/helia-unixfs/issues/6)) ([a6bd198](https://github.com/ipfs/helia-unixfs/commit/a6bd1983bd7baac21af3de6fa269219f52664cde)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0276802e..5b4d157f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.0.5](https://github.com/ipfs/helia-unixfs/compare/v1.0.4...v1.0.5) (2023-03-14) + + +### Bug Fixes + +* align defaults with filecoin ([#6](https://github.com/ipfs/helia-unixfs/issues/6)) ([a6bd198](https://github.com/ipfs/helia-unixfs/commit/a6bd1983bd7baac21af3de6fa269219f52664cde)) + ## [1.0.4](https://github.com/ipfs/helia-unixfs/compare/v1.0.3...v1.0.4) (2023-02-27) diff --git a/package.json b/package.json index 26c35970..e710f210 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.0.4", + "version": "1.0.5", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", From b81d50a396cef7fabc26ad903a91233cbcafaac0 Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:21:08 +0100 Subject: [PATCH 029/253] update .github/workflows/js-test-and-release.yml (#9) --- .github/workflows/js-test-and-release.yml | 38 ++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 0c396fbd..d4a628b7 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -96,6 +96,42 @@ jobs: with: flags: firefox-webworker + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + test-electron-main: needs: check runs-on: ubuntu-latest @@ -125,7 +161,7 @@ jobs: flags: electron-renderer release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer] + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: From 7f128fdf60003b3425da1fbf965979dd6e012b91 Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:22:12 +0100 Subject: [PATCH 030/253] update .github/workflows/js-test-and-release.yml (#9) --- .github/workflows/js-test-and-release.yml | 38 ++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 0c396fbd..d4a628b7 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -96,6 +96,42 @@ jobs: with: flags: firefox-webworker + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + test-electron-main: needs: check runs-on: ubuntu-latest @@ -125,7 +161,7 @@ jobs: flags: electron-renderer release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer] + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: From 3189737040a9dfe631e1d07f7f884ff19b873f17 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 15 Mar 2023 11:50:48 +0100 Subject: [PATCH 031/253] deps: update blockstore/datastore deps (#10) Updates all deps to the latest versions --- packages/interop/README.md | 5 ---- packages/interop/package.json | 29 +++---------------- .../test/fixtures/create-helia.browser.ts | 3 -- .../interop/test/fixtures/create-helia.ts | 3 -- packages/ipns/package.json | 7 ++--- 5 files changed, 7 insertions(+), 40 deletions(-) diff --git a/packages/interop/README.md b/packages/interop/README.md index e905e050..fbdb8140 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -17,7 +17,6 @@ - [Install](#install) - [Browser ` ``` -## API Docs - -- - ## License Licensed under either of diff --git a/packages/interop/package.json b/packages/interop/package.json index 46b3b9f3..c46c747d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -20,22 +20,6 @@ }, "type": "module", "types": "./dist/src/index.d.ts", - "typesVersions": { - "*": { - "*": [ - "*", - "dist/*", - "dist/src/*", - "dist/src/*/index" - ], - "src/*": [ - "*", - "dist/*", - "dist/src/*", - "dist/src/*/index" - ] - } - }, "files": [ "src", "dist", @@ -46,10 +30,6 @@ ".": { "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js" - }, - "./routing": { - "types": "./dist/src/routing/index.d.ts", - "import": "./dist/src/routing/index.js" } }, "eslintConfig": { @@ -63,7 +43,6 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", - "docs": "aegir docs", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", @@ -79,14 +58,14 @@ "@helia/interface": "next", "@helia/ipns": "^1.0.0", "@libp2p/interface-peer-id": "^2.0.1", - "@libp2p/kad-dht": "^7.0.0", + "@libp2p/kad-dht": "^8.0.0", "@libp2p/peer-id": "^2.0.1", "@libp2p/peer-id-factory": "^2.0.1", "@libp2p/tcp": "^6.1.2", "@libp2p/websockets": "^5.0.3", "aegir": "^38.1.0", - "blockstore-core": "^3.0.0", - "datastore-core": "^8.0.4", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", "go-ipfs": "^0.18.1", "helia": "next", "ipfsd-ctl": "^13.0.0", @@ -95,7 +74,7 @@ "it-last": "^2.0.0", "it-map": "^2.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.42.2", + "libp2p": "next", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "uint8arrays": "^4.0.3", diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index a3684dfc..62eb4707 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -41,9 +41,6 @@ export async function createHeliaNode (): Promise { datastore, nat: { enabled: false - }, - relay: { - enabled: false } }) diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index 881b545d..2515ca35 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -30,9 +30,6 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise Date: Wed, 15 Mar 2023 13:48:18 +0100 Subject: [PATCH 032/253] update .github/workflows/js-test-and-release.yml (#11) --- .github/workflows/js-test-and-release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index d4a628b7..614a1d85 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -99,7 +99,8 @@ jobs: test-webkit: needs: check runs-on: ${{ matrix.os }} - matrix: + strategy: + matrix: os: [ubuntu-latest, macos-latest] node: [lts/*] fail-fast: true @@ -114,10 +115,11 @@ jobs: with: flags: webkit - test-webkit: + test-webkit-webworker: needs: check runs-on: ${{ matrix.os }} - matrix: + strategy: + matrix: os: [ubuntu-latest, macos-latest] node: [lts/*] fail-fast: true From 7558d38327946c7a926a89a968ec7cf2af43d4fb Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Wed, 15 Mar 2023 13:52:23 +0100 Subject: [PATCH 033/253] update .github/workflows/js-test-and-release.yml (#11) --- .github/workflows/js-test-and-release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index d4a628b7..614a1d85 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -99,7 +99,8 @@ jobs: test-webkit: needs: check runs-on: ${{ matrix.os }} - matrix: + strategy: + matrix: os: [ubuntu-latest, macos-latest] node: [lts/*] fail-fast: true @@ -114,10 +115,11 @@ jobs: with: flags: webkit - test-webkit: + test-webkit-webworker: needs: check runs-on: ${{ matrix.os }} - matrix: + strategy: + matrix: os: [ubuntu-latest, macos-latest] node: [lts/*] fail-fast: true From 3ad5f5d8199a5596aa333916d4a240584bc0842a Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 15 Mar 2023 16:06:33 +0100 Subject: [PATCH 034/253] chore: add interop test suite (#12) Splits repo into monorepo and adds a package for doing interop testing between @helia/unixfs and kubo. Also updates all deps. --- README.md | 26 +-- package.json | 173 +++--------------- packages/interop/.aegir.js | 45 +++++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 59 ++++++ packages/interop/package.json | 83 +++++++++ packages/interop/src/index.ts | 1 + packages/interop/test/files.spec.ts | 89 +++++++++ .../test/fixtures/create-helia.browser.ts | 41 +++++ .../interop/test/fixtures/create-helia.ts | 38 ++++ packages/interop/test/fixtures/create-kubo.ts | 28 +++ packages/interop/tsconfig.json | 15 ++ CHANGELOG.md => packages/unixfs/CHANGELOG.md | 0 packages/unixfs/LICENSE | 4 + packages/unixfs/LICENSE-APACHE | 5 + packages/unixfs/LICENSE-MIT | 19 ++ packages/unixfs/README.md | 59 ++++++ packages/unixfs/package.json | 168 +++++++++++++++++ {src => packages/unixfs/src}/commands/add.ts | 12 +- {src => packages/unixfs/src}/commands/cat.ts | 4 +- .../unixfs/src}/commands/chmod.ts | 4 +- {src => packages/unixfs/src}/commands/cp.ts | 4 +- {src => packages/unixfs/src}/commands/ls.ts | 4 +- .../unixfs/src}/commands/mkdir.ts | 4 +- {src => packages/unixfs/src}/commands/rm.ts | 4 +- {src => packages/unixfs/src}/commands/stat.ts | 6 +- .../unixfs/src}/commands/touch.ts | 4 +- .../unixfs/src}/commands/utils/add-link.ts | 10 +- .../src}/commands/utils/cid-to-directory.ts | 4 +- .../src}/commands/utils/cid-to-pblink.ts | 4 +- .../unixfs/src}/commands/utils/constants.ts | 0 .../src}/commands/utils/consumable-hash.ts | 0 .../unixfs/src}/commands/utils/dir-sharded.ts | 8 +- .../unixfs/src}/commands/utils/errors.ts | 0 .../src}/commands/utils/hamt-constants.ts | 0 .../unixfs/src}/commands/utils/hamt-utils.ts | 8 +- .../commands/utils/is-over-shard-threshold.ts | 6 +- .../unixfs/src}/commands/utils/persist.ts | 7 +- .../unixfs/src}/commands/utils/remove-link.ts | 10 +- .../unixfs/src}/commands/utils/resolve.ts | 6 +- {src => packages/unixfs/src}/index.ts | 6 +- {test => packages/unixfs/test}/add.spec.ts | 0 {test => packages/unixfs/test}/cat.spec.ts | 0 {test => packages/unixfs/test}/chmod.spec.ts | 0 {test => packages/unixfs/test}/cp.spec.ts | 0 .../fixtures/create-sharded-directory.ts | 0 .../fixtures/create-subsharded-directory.ts | 0 .../unixfs/test}/fixtures/files.ts | 0 .../unixfs/test}/fixtures/print-tree.ts | 4 +- {test => packages/unixfs/test}/ls.spec.ts | 0 {test => packages/unixfs/test}/mkdir.spec.ts | 0 {test => packages/unixfs/test}/rm.spec.ts | 0 {test => packages/unixfs/test}/stat.spec.ts | 0 {test => packages/unixfs/test}/touch.spec.ts | 0 .../unixfs/tsconfig.json | 0 57 files changed, 779 insertions(+), 221 deletions(-) create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/files.spec.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/tsconfig.json rename CHANGELOG.md => packages/unixfs/CHANGELOG.md (100%) create mode 100644 packages/unixfs/LICENSE create mode 100644 packages/unixfs/LICENSE-APACHE create mode 100644 packages/unixfs/LICENSE-MIT create mode 100644 packages/unixfs/README.md create mode 100644 packages/unixfs/package.json rename {src => packages/unixfs/src}/commands/add.ts (77%) rename {src => packages/unixfs/src}/commands/cat.ts (82%) rename {src => packages/unixfs/src}/commands/chmod.ts (97%) rename {src => packages/unixfs/src}/commands/cp.ts (90%) rename {src => packages/unixfs/src}/commands/ls.ts (83%) rename {src => packages/unixfs/src}/commands/mkdir.ts (94%) rename {src => packages/unixfs/src}/commands/rm.ts (90%) rename {src => packages/unixfs/src}/commands/stat.ts (93%) rename {src => packages/unixfs/src}/commands/touch.ts (96%) rename {src => packages/unixfs/src}/commands/utils/add-link.ts (95%) rename {src => packages/unixfs/src}/commands/utils/cid-to-directory.ts (82%) rename {src => packages/unixfs/src}/commands/utils/cid-to-pblink.ts (87%) rename {src => packages/unixfs/src}/commands/utils/constants.ts (100%) rename {src => packages/unixfs/src}/commands/utils/consumable-hash.ts (100%) rename {src => packages/unixfs/src}/commands/utils/dir-sharded.ts (96%) rename {src => packages/unixfs/src}/commands/utils/errors.ts (100%) rename {src => packages/unixfs/src}/commands/utils/hamt-constants.ts (100%) rename {src => packages/unixfs/src}/commands/utils/hamt-utils.ts (92%) rename {src => packages/unixfs/src}/commands/utils/is-over-shard-threshold.ts (92%) rename {src => packages/unixfs/src}/commands/utils/persist.ts (74%) rename {src => packages/unixfs/src}/commands/utils/remove-link.ts (92%) rename {src => packages/unixfs/src}/commands/utils/resolve.ts (93%) rename {src => packages/unixfs/src}/index.ts (98%) rename {test => packages/unixfs/test}/add.spec.ts (100%) rename {test => packages/unixfs/test}/cat.spec.ts (100%) rename {test => packages/unixfs/test}/chmod.spec.ts (100%) rename {test => packages/unixfs/test}/cp.spec.ts (100%) rename {test => packages/unixfs/test}/fixtures/create-sharded-directory.ts (100%) rename {test => packages/unixfs/test}/fixtures/create-subsharded-directory.ts (100%) rename {test => packages/unixfs/test}/fixtures/files.ts (100%) rename {test => packages/unixfs/test}/fixtures/print-tree.ts (84%) rename {test => packages/unixfs/test}/ls.spec.ts (100%) rename {test => packages/unixfs/test}/mkdir.spec.ts (100%) rename {test => packages/unixfs/test}/rm.spec.ts (100%) rename {test => packages/unixfs/test}/stat.spec.ts (100%) rename {test => packages/unixfs/test}/touch.spec.ts (100%) rename tsconfig.json => packages/unixfs/tsconfig.json (100%) diff --git a/README.md b/README.md index 239e4e49..4b1c0443 100644 --- a/README.md +++ b/README.md @@ -4,36 +4,26 @@

-# @helia/unixfs +# @helia/ipns [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) -> A Helia-compatible wrapper for UnixFS +> An implementation of Unixfs for Helia ## Table of contents -- [Install](#install) - - [Browser ` -``` +- [`/packages/interop`](./packages/interop) Interop tests for @helia/unixfs +- [`/packages/unixfs`](./packages/unixfs) A Helia-compatible wrapper for UnixFS ## API Docs diff --git a/package.json b/package.json index e710f210..d8234687 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@helia/unixfs", - "version": "1.0.5", - "description": "A Helia-compatible wrapper for UnixFS", + "version": "0.0.0", + "description": "An implementation of Unixfs for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs#readme", "repository": { @@ -12,156 +12,37 @@ "url": "https://github.com/ipfs/helia-unixfs/issues" }, "keywords": [ - "IPFS" + "ipfs" ], "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" }, - "type": "module", - "types": "./dist/src/index.d.ts", - "files": [ - "src", - "dist", - "!dist/test", - "!**/*.tsbuildinfo" - ], - "exports": { - ".": { - "types": "./dist/src/index.d.ts", - "import": "./dist/src/index.js" - } - }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "sourceType": "module" - } - }, - "release": { - "branches": [ - "main" - ], - "plugins": [ - [ - "@semantic-release/commit-analyzer", - { - "preset": "conventionalcommits", - "releaseRules": [ - { - "breaking": true, - "release": "major" - }, - { - "revert": true, - "release": "patch" - }, - { - "type": "feat", - "release": "minor" - }, - { - "type": "fix", - "release": "patch" - }, - { - "type": "docs", - "release": "patch" - }, - { - "type": "test", - "release": "patch" - }, - { - "type": "deps", - "release": "patch" - }, - { - "scope": "no-release", - "release": false - } - ] - } - ], - [ - "@semantic-release/release-notes-generator", - { - "preset": "conventionalcommits", - "presetConfig": { - "types": [ - { - "type": "feat", - "section": "Features" - }, - { - "type": "fix", - "section": "Bug Fixes" - }, - { - "type": "chore", - "section": "Trivial Changes" - }, - { - "type": "docs", - "section": "Documentation" - }, - { - "type": "deps", - "section": "Dependencies" - }, - { - "type": "test", - "section": "Tests" - } - ] - } - } - ], - "@semantic-release/changelog", - "@semantic-release/npm", - "@semantic-release/github", - "@semantic-release/git" - ] - }, + "private": true, "scripts": { - "clean": "aegir clean", - "lint": "aegir lint", - "dep-check": "aegir dep-check", - "build": "aegir build", - "docs": "aegir docs", - "test": "aegir test", - "test:chrome": "aegir test -t browser --cov", - "test:chrome-webworker": "aegir test -t webworker", - "test:firefox": "aegir test -t browser -- --browser firefox", - "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", - "test:node": "aegir test -t node --cov", - "test:electron-main": "aegir test -t electron-main", - "release": "aegir release" - }, - "dependencies": { - "@ipld/dag-pb": "^4.0.0", - "@libp2p/interfaces": "^3.3.1", - "@libp2p/logger": "^2.0.5", - "@multiformats/murmur3": "^2.1.2", - "hamt-sharding": "^3.0.2", - "interface-blockstore": "^4.0.1", - "ipfs-unixfs": "^11.0.0", - "ipfs-unixfs-exporter": "^12.0.0", - "ipfs-unixfs-importer": "^14.0.1", - "it-last": "^2.0.0", - "it-pipe": "^2.0.5", - "merge-options": "^3.0.4", - "multiformats": "^11.0.1", - "sparse-array": "^1.3.2" + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^38.1.0", - "blockstore-core": "^3.0.0", - "delay": "^5.0.0", - "it-all": "^2.0.0", - "it-drain": "^2.0.0", - "it-first": "^2.0.0", - "it-to-buffer": "^3.0.0", - "uint8arrays": "^4.0.3" - } + "aegir": "^38.1.0" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] } diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..bb31762f --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/unixfs-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/unixfs + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..7febd31c --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,83 @@ +{ + "name": "@helia/unixfs-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/unixfs", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-unixfs.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-unixfs/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^3.0.5", + "@helia/interface": "next", + "@helia/unixfs": "^1.0.0", + "@libp2p/tcp": "^6.1.2", + "@libp2p/websockets": "^5.0.3", + "aegir": "^38.1.0", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", + "go-ipfs": "^0.18.1", + "helia": "next", + "ipfs-core-types": "^0.14.0", + "ipfs-unixfs-importer": "^15.0.0", + "ipfsd-ctl": "^13.0.0", + "kubo-rpc-client": "^3.0.0", + "libp2p": "next", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/files.spec.ts b/packages/interop/test/files.spec.ts new file mode 100644 index 00000000..dcd5b5c5 --- /dev/null +++ b/packages/interop/test/files.spec.ts @@ -0,0 +1,89 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { UnixFS, unixfs } from '@helia/unixfs' +import { balanced } from 'ipfs-unixfs-importer/layout' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import type { FileCandidate, ImporterOptions } from 'ipfs-unixfs-importer' +import type { CID } from 'multiformats/cid' +import type { AddOptions } from 'ipfs-core-types/src/root.js' + +describe('unixfs interop', () => { + let helia: Helia + let unixFs: UnixFS + let kubo: Controller + + async function importToHelia (data: FileCandidate, opts?: Partial): Promise { + const cid = await unixFs.addFile(data, opts) + + return cid + } + + async function importToKubo (data: FileCandidate, opts?: AddOptions): Promise { + const result = await kubo.api.add(data.content, opts) + + return result.cid + } + + async function expectSameCid (data: () => FileCandidate, heliaOpts: Partial = {}, kuboOpts: AddOptions = {}): Promise { + const heliaCid = await importToHelia(data(), { + // these are the default kubo options + cidVersion: 0, + rawLeaves: false, + layout: balanced({ + maxChildrenPerNode: 174 + }), + chunker: fixedSize({ + chunkSize: 262144 + }), + + ...heliaOpts + }) + const kuboCid = await importToKubo(data(), kuboOpts) + + expect(heliaCid.toString()).to.equal(kuboCid.toString()) + } + + beforeEach(async () => { + helia = await createHeliaNode() + unixFs = unixfs(helia) + kubo = await createKuboNode() + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should create the same CID for a small file', async () => { + const candidate = (): FileCandidate => ({ + content: Uint8Array.from([0, 1, 2, 3, 4]) + }) + + await expectSameCid(candidate) + }) + + it('should create the same CID for a large file', async () => { + const chunkSize = 1024 * 1024 + const size = chunkSize * 10 + + const candidate = (): FileCandidate => ({ + content: (async function * () { + for (let i = 0; i < size; i += chunkSize) { + yield new Uint8Array(chunkSize) + } + }()) + }) + + await expectSameCid(candidate) + }) +}) diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..d28c6efa --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,41 @@ +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..7a7201b0 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,38 @@ +import { createHelia } from 'helia' +import { createLibp2p, Libp2pOptions } from 'libp2p' +import { tcp } from '@libp2p/tcp' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..7fd48729 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { Controller, ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import { isElectronMain, isNode } from 'wherearewe' +import mergeOptions from 'merge-options' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return await createController(opts) +} diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..d54d7166 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../unixfs" + } + ] +} diff --git a/CHANGELOG.md b/packages/unixfs/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to packages/unixfs/CHANGELOG.md diff --git a/packages/unixfs/LICENSE b/packages/unixfs/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/unixfs/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/unixfs/LICENSE-APACHE b/packages/unixfs/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/unixfs/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/unixfs/LICENSE-MIT b/packages/unixfs/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/unixfs/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/unixfs/README.md b/packages/unixfs/README.md new file mode 100644 index 00000000..22527bdb --- /dev/null +++ b/packages/unixfs/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/unixfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> A Helia-compatible wrapper for UnixFS + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json new file mode 100644 index 00000000..890959ca --- /dev/null +++ b/packages/unixfs/package.json @@ -0,0 +1,168 @@ +{ + "name": "@helia/unixfs", + "version": "1.0.5", + "description": "A Helia-compatible wrapper for UnixFS", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-unixfs.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-unixfs/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "next", + "@ipld/dag-pb": "^4.0.0", + "@libp2p/interfaces": "^3.3.1", + "@libp2p/logger": "^2.0.6", + "@multiformats/murmur3": "^2.1.2", + "hamt-sharding": "^3.0.2", + "interface-blockstore": "^5.0.0", + "ipfs-unixfs": "^11.0.0", + "ipfs-unixfs-exporter": "^13.0.0", + "ipfs-unixfs-importer": "^15.0.0", + "it-last": "^2.0.0", + "it-pipe": "^2.0.5", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1", + "sparse-array": "^1.3.2" + }, + "devDependencies": { + "aegir": "^38.1.0", + "blockstore-core": "^4.0.1", + "delay": "^5.0.0", + "it-all": "^2.0.0", + "it-drain": "^2.0.0", + "it-first": "^2.0.0", + "it-to-buffer": "^3.0.0", + "uint8arrays": "^4.0.3" + } +} diff --git a/src/commands/add.ts b/packages/unixfs/src/commands/add.ts similarity index 77% rename from src/commands/add.ts rename to packages/unixfs/src/commands/add.ts index 15010aed..33e334d7 100644 --- a/src/commands/add.ts +++ b/packages/unixfs/src/commands/add.ts @@ -1,5 +1,5 @@ import type { CID } from 'multiformats/cid' -import type { Blockstore } from 'ipfs-unixfs-importer' +import type { Blocks } from '@helia/interface/blocks' import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer' import { balanced } from 'ipfs-unixfs-importer/layout' import { fixedSize } from 'ipfs-unixfs-importer/chunker' @@ -18,14 +18,14 @@ const defaultImporterSettings: ImporterOptions = { }) } -export async function * addAll (source: ImportCandidateStream, blockstore: Blockstore, options: Partial = {}): AsyncGenerator { +export async function * addAll (source: ImportCandidateStream, blockstore: Blocks, options: Partial = {}): AsyncGenerator { yield * importer(source, blockstore, { ...defaultImporterSettings, ...options }) } -export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, options: Partial = {}): Promise { +export async function addBytes (bytes: Uint8Array, blockstore: Blocks, options: Partial = {}): Promise { const { cid } = await importBytes(bytes, blockstore, { ...defaultImporterSettings, ...options @@ -34,7 +34,7 @@ export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, optio return cid } -export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, options: Partial = {}): Promise { +export async function addByteStream (bytes: ByteStream, blockstore: Blocks, options: Partial = {}): Promise { const { cid } = await importByteStream(bytes, blockstore, { ...defaultImporterSettings, ...options @@ -43,7 +43,7 @@ export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, return cid } -export async function addFile (file: FileCandidate, blockstore: Blockstore, options: Partial = {}): Promise { +export async function addFile (file: FileCandidate, blockstore: Blocks, options: Partial = {}): Promise { const { cid } = await importFile(file, blockstore, { ...defaultImporterSettings, ...options @@ -52,7 +52,7 @@ export async function addFile (file: FileCandidate, blockstore: Blockstore, opti return cid } -export async function addDirectory (dir: Partial, blockstore: Blockstore, options: Partial = {}): Promise { +export async function addDirectory (dir: Partial, blockstore: Blocks, options: Partial = {}): Promise { const { cid } = await importDirectory({ ...dir, path: dir.path ?? '-' diff --git a/src/commands/cat.ts b/packages/unixfs/src/commands/cat.ts similarity index 82% rename from src/commands/cat.ts rename to packages/unixfs/src/commands/cat.ts index 2c6d1bd0..eb31c76c 100644 --- a/src/commands/cat.ts +++ b/packages/unixfs/src/commands/cat.ts @@ -3,7 +3,7 @@ import type { CID } from 'multiformats/cid' import type { CatOptions } from '../index.js' import { resolve } from './utils/resolve.js' import mergeOpts from 'merge-options' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import { NoContentError, NotAFileError } from './utils/errors.js' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) @@ -12,7 +12,7 @@ const defaultOptions: CatOptions = { } -export async function * cat (cid: CID, blockstore: Blockstore, options: Partial = {}): AsyncIterable { +export async function * cat (cid: CID, blockstore: Blocks, options: Partial = {}): AsyncIterable { const opts: CatOptions = mergeOptions(defaultOptions, options) const resolved = await resolve(cid, opts.path, blockstore, opts) const result = await exporter(resolved.cid, blockstore, opts) diff --git a/src/commands/chmod.ts b/packages/unixfs/src/commands/chmod.ts similarity index 97% rename from src/commands/chmod.ts rename to packages/unixfs/src/commands/chmod.ts index ea17a0af..0836a296 100644 --- a/src/commands/chmod.ts +++ b/packages/unixfs/src/commands/chmod.ts @@ -10,7 +10,7 @@ import * as dagPB from '@ipld/dag-pb' import type { PBNode, PBLink } from '@ipld/dag-pb' import { importer } from 'ipfs-unixfs-importer' import { persist } from './utils/persist.js' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import last from 'it-last' import { sha256 } from 'multiformats/hashes/sha2' import { resolve, updatePathCids } from './utils/resolve.js' @@ -25,7 +25,7 @@ const defaultOptions: ChmodOptions = { shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES } -export async function chmod (cid: CID, mode: number, blockstore: Blockstore, options: Partial = {}): Promise { +export async function chmod (cid: CID, mode: number, blockstore: Blocks, options: Partial = {}): Promise { const opts: ChmodOptions = mergeOptions(defaultOptions, options) const resolved = await resolve(cid, opts.path, blockstore, options) diff --git a/src/commands/cp.ts b/packages/unixfs/src/commands/cp.ts similarity index 90% rename from src/commands/cp.ts rename to packages/unixfs/src/commands/cp.ts index 4ca837fb..1069b4ab 100644 --- a/src/commands/cp.ts +++ b/packages/unixfs/src/commands/cp.ts @@ -1,4 +1,4 @@ -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import type { CID } from 'multiformats/cid' import type { CpOptions } from '../index.js' import mergeOpts from 'merge-options' @@ -17,7 +17,7 @@ const defaultOptions: CpOptions = { shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES } -export async function cp (source: CID, target: CID, name: string, blockstore: Blockstore, options: Partial = {}): Promise { +export async function cp (source: CID, target: CID, name: string, blockstore: Blocks, options: Partial = {}): Promise { const opts: CpOptions = mergeOptions(defaultOptions, options) if (name.includes('/')) { diff --git a/src/commands/ls.ts b/packages/unixfs/src/commands/ls.ts similarity index 83% rename from src/commands/ls.ts rename to packages/unixfs/src/commands/ls.ts index ca52acfd..f27f2e25 100644 --- a/src/commands/ls.ts +++ b/packages/unixfs/src/commands/ls.ts @@ -3,7 +3,7 @@ import type { CID } from 'multiformats/cid' import type { LsOptions } from '../index.js' import { resolve } from './utils/resolve.js' import mergeOpts from 'merge-options' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import { NoContentError, NotADirectoryError } from './utils/errors.js' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) @@ -12,7 +12,7 @@ const defaultOptions: LsOptions = { } -export async function * ls (cid: CID, blockstore: Blockstore, options: Partial = {}): AsyncIterable { +export async function * ls (cid: CID, blockstore: Blocks, options: Partial = {}): AsyncIterable { const opts: LsOptions = mergeOptions(defaultOptions, options) const resolved = await resolve(cid, opts.path, blockstore, opts) const result = await exporter(resolved.cid, blockstore) diff --git a/src/commands/mkdir.ts b/packages/unixfs/src/commands/mkdir.ts similarity index 94% rename from src/commands/mkdir.ts rename to packages/unixfs/src/commands/mkdir.ts index 6b96905b..b2281023 100644 --- a/src/commands/mkdir.ts +++ b/packages/unixfs/src/commands/mkdir.ts @@ -4,7 +4,7 @@ import { logger } from '@libp2p/logger' import type { MkdirOptions } from '../index.js' import * as dagPB from '@ipld/dag-pb' import { addLink } from './utils/add-link.js' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import { UnixFS } from 'ipfs-unixfs' import { sha256 } from 'multiformats/hashes/sha2' import { exporter } from 'ipfs-unixfs-exporter' @@ -22,7 +22,7 @@ const defaultOptions: MkdirOptions = { shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES } -export async function mkdir (parentCid: CID, dirname: string, blockstore: Blockstore, options: Partial = {}): Promise { +export async function mkdir (parentCid: CID, dirname: string, blockstore: Blocks, options: Partial = {}): Promise { const opts: MkdirOptions = mergeOptions(defaultOptions, options) if (dirname.includes('/')) { diff --git a/src/commands/rm.ts b/packages/unixfs/src/commands/rm.ts similarity index 90% rename from src/commands/rm.ts rename to packages/unixfs/src/commands/rm.ts index 28f75325..8f091b8a 100644 --- a/src/commands/rm.ts +++ b/packages/unixfs/src/commands/rm.ts @@ -1,4 +1,4 @@ -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import type { CID } from 'multiformats/cid' import type { RmOptions } from '../index.js' import mergeOpts from 'merge-options' @@ -15,7 +15,7 @@ const defaultOptions: RmOptions = { shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES } -export async function rm (target: CID, name: string, blockstore: Blockstore, options: Partial = {}): Promise { +export async function rm (target: CID, name: string, blockstore: Blocks, options: Partial = {}): Promise { const opts: RmOptions = mergeOptions(defaultOptions, options) if (name.includes('/')) { diff --git a/src/commands/stat.ts b/packages/unixfs/src/commands/stat.ts similarity index 93% rename from src/commands/stat.ts rename to packages/unixfs/src/commands/stat.ts index 9564ecad..d350319f 100644 --- a/src/commands/stat.ts +++ b/packages/unixfs/src/commands/stat.ts @@ -10,7 +10,7 @@ import type { AbortOptions } from '@libp2p/interfaces' import type { Mtime } from 'ipfs-unixfs' import { resolve } from './utils/resolve.js' import * as raw from 'multiformats/codecs/raw' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:stat') @@ -19,7 +19,7 @@ const defaultOptions: StatOptions = { } -export async function stat (cid: CID, blockstore: Blockstore, options: Partial = {}): Promise { +export async function stat (cid: CID, blockstore: Blocks, options: Partial = {}): Promise { const opts: StatOptions = mergeOptions(defaultOptions, options) const resolved = await resolve(cid, options.path, blockstore, opts) @@ -93,7 +93,7 @@ interface InspectDagResults { blocks: number } -async function inspectDag (cid: CID, blockstore: Blockstore, options: AbortOptions): Promise { +async function inspectDag (cid: CID, blockstore: Blocks, options: AbortOptions): Promise { const results = { localFileSize: 0, localDagSize: 0, diff --git a/src/commands/touch.ts b/packages/unixfs/src/commands/touch.ts similarity index 96% rename from src/commands/touch.ts rename to packages/unixfs/src/commands/touch.ts index cbed2557..2a5b225d 100644 --- a/src/commands/touch.ts +++ b/packages/unixfs/src/commands/touch.ts @@ -10,7 +10,7 @@ import * as dagPB from '@ipld/dag-pb' import type { PBNode, PBLink } from '@ipld/dag-pb' import { importer } from 'ipfs-unixfs-importer' import { persist } from './utils/persist.js' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import last from 'it-last' import { sha256 } from 'multiformats/hashes/sha2' import { resolve, updatePathCids } from './utils/resolve.js' @@ -25,7 +25,7 @@ const defaultOptions: TouchOptions = { shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD_BYTES } -export async function touch (cid: CID, blockstore: Blockstore, options: Partial = {}): Promise { +export async function touch (cid: CID, blockstore: Blocks, options: Partial = {}): Promise { const opts: TouchOptions = mergeOptions(defaultOptions, options) const resolved = await resolve(cid, opts.path, blockstore, opts) const mtime = opts.mtime ?? { diff --git a/src/commands/utils/add-link.ts b/packages/unixfs/src/commands/utils/add-link.ts similarity index 95% rename from src/commands/utils/add-link.ts rename to packages/unixfs/src/commands/utils/add-link.ts index 0eb7566f..8ffc781f 100644 --- a/src/commands/utils/add-link.ts +++ b/packages/unixfs/src/commands/utils/add-link.ts @@ -14,7 +14,7 @@ import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from ' import type { ImportResult } from 'ipfs-unixfs-importer' import type { AbortOptions } from '@libp2p/interfaces' import type { Directory } from './cid-to-directory.js' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import { isOverShardThreshold } from './is-over-shard-threshold.js' import { hamtBucketBits, hamtHashFn } from './hamt-constants.js' // @ts-expect-error no types @@ -35,7 +35,7 @@ export interface AddLinkOptions extends AbortOptions { cidVersion: Version } -export async function addLink (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise { +export async function addLink (parent: Directory, child: Required, blockstore: Blocks, options: AddLinkOptions): Promise { if (parent.node.Data == null) { throw new InvalidParametersError('Invalid parent passed to addLink') } @@ -63,7 +63,7 @@ export async function addLink (parent: Directory, child: Required, block return result } -const convertToShardedDirectory = async (parent: Directory, blockstore: Blockstore): Promise => { +const convertToShardedDirectory = async (parent: Directory, blockstore: Blocks): Promise => { if (parent.node.Data == null) { throw new InvalidParametersError('Invalid parent passed to convertToShardedDirectory') } @@ -85,7 +85,7 @@ const convertToShardedDirectory = async (parent: Directory, blockstore: Blocksto return result } -const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Blockstore, options: AddLinkOptions): Promise => { +const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Blocks, options: AddLinkOptions): Promise => { // Remove existing link if it exists const parentLinks = parent.node.Links.filter((link) => { const matches = link.Name === child.Name @@ -137,7 +137,7 @@ const addToDirectory = async (parent: Directory, child: PBLink, blockstore: Bloc } } -const addToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blockstore, options: AddLinkOptions): Promise => { +const addToShardedDirectory = async (parent: Directory, child: Required, blockstore: Blocks, options: AddLinkOptions): Promise => { const { path, hash } = await recreateShardedDirectory(parent.cid, child.Name, blockstore, options) const finalSegment = path[path.length - 1] diff --git a/src/commands/utils/cid-to-directory.ts b/packages/unixfs/src/commands/utils/cid-to-directory.ts similarity index 82% rename from src/commands/utils/cid-to-directory.ts rename to packages/unixfs/src/commands/utils/cid-to-directory.ts index 8d01526b..04282be0 100644 --- a/src/commands/utils/cid-to-directory.ts +++ b/packages/unixfs/src/commands/utils/cid-to-directory.ts @@ -1,7 +1,7 @@ import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter' import type { CID } from 'multiformats/cid' import type { PBNode } from '@ipld/dag-pb' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import { NotADirectoryError } from './errors.js' export interface Directory { @@ -9,7 +9,7 @@ export interface Directory { node: PBNode } -export async function cidToDirectory (cid: CID, blockstore: Blockstore, options: ExporterOptions = {}): Promise { +export async function cidToDirectory (cid: CID, blockstore: Blocks, options: ExporterOptions = {}): Promise { const entry = await exporter(cid, blockstore, options) if (entry.type !== 'directory') { diff --git a/src/commands/utils/cid-to-pblink.ts b/packages/unixfs/src/commands/utils/cid-to-pblink.ts similarity index 87% rename from src/commands/utils/cid-to-pblink.ts rename to packages/unixfs/src/commands/utils/cid-to-pblink.ts index e819e92c..af351fc1 100644 --- a/src/commands/utils/cid-to-pblink.ts +++ b/packages/unixfs/src/commands/utils/cid-to-pblink.ts @@ -3,9 +3,9 @@ import type { CID } from 'multiformats/cid' import { NotUnixFSError } from './errors.js' import * as dagPb from '@ipld/dag-pb' import type { PBNode, PBLink } from '@ipld/dag-pb' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' -export async function cidToPBLink (cid: CID, name: string, blockstore: Blockstore, options?: ExporterOptions): Promise> { +export async function cidToPBLink (cid: CID, name: string, blockstore: Blocks, options?: ExporterOptions): Promise> { const sourceEntry = await exporter(cid, blockstore, options) if (sourceEntry.type !== 'directory' && sourceEntry.type !== 'file' && sourceEntry.type !== 'raw') { diff --git a/src/commands/utils/constants.ts b/packages/unixfs/src/commands/utils/constants.ts similarity index 100% rename from src/commands/utils/constants.ts rename to packages/unixfs/src/commands/utils/constants.ts diff --git a/src/commands/utils/consumable-hash.ts b/packages/unixfs/src/commands/utils/consumable-hash.ts similarity index 100% rename from src/commands/utils/consumable-hash.ts rename to packages/unixfs/src/commands/utils/consumable-hash.ts diff --git a/src/commands/utils/dir-sharded.ts b/packages/unixfs/src/commands/utils/dir-sharded.ts similarity index 96% rename from src/commands/utils/dir-sharded.ts rename to packages/unixfs/src/commands/utils/dir-sharded.ts index 2f896772..66721742 100644 --- a/src/commands/utils/dir-sharded.ts +++ b/packages/unixfs/src/commands/utils/dir-sharded.ts @@ -8,7 +8,7 @@ import { } from './hamt-constants.js' import { CID } from 'multiformats/cid' import type { Mtime } from 'ipfs-unixfs' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' interface InProgressImportResult extends ImportResult { single?: boolean @@ -69,7 +69,7 @@ abstract class Dir { abstract put (name: string, value: InProgressImportResult | Dir): Promise abstract get (name: string): Promise abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }> - abstract flush (blockstore: Blockstore): AsyncGenerator + abstract flush (blockstore: Blocks): AsyncGenerator abstract estimateNodeSize (): number abstract childCount (): number } @@ -129,7 +129,7 @@ export class DirSharded extends Dir { return this.nodeSize } - async * flush (blockstore: Blockstore): AsyncGenerator { + async * flush (blockstore: Blocks): AsyncGenerator { for await (const entry of flush(this._bucket, blockstore, this, this.options)) { yield { ...entry, @@ -139,7 +139,7 @@ export class DirSharded extends Dir { } } -async function * flush (bucket: Bucket, blockstore: Blockstore, shardRoot: DirSharded | null, options: PersistOptions): AsyncIterable { +async function * flush (bucket: Bucket, blockstore: Blocks, shardRoot: DirSharded | null, options: PersistOptions): AsyncIterable { const children = bucket._children const links: PBLink[] = [] let childrenSize = 0n diff --git a/src/commands/utils/errors.ts b/packages/unixfs/src/commands/utils/errors.ts similarity index 100% rename from src/commands/utils/errors.ts rename to packages/unixfs/src/commands/utils/errors.ts diff --git a/src/commands/utils/hamt-constants.ts b/packages/unixfs/src/commands/utils/hamt-constants.ts similarity index 100% rename from src/commands/utils/hamt-constants.ts rename to packages/unixfs/src/commands/utils/hamt-constants.ts diff --git a/src/commands/utils/hamt-utils.ts b/packages/unixfs/src/commands/utils/hamt-utils.ts similarity index 92% rename from src/commands/utils/hamt-utils.ts rename to packages/unixfs/src/commands/utils/hamt-utils.ts index 7df9a771..cbbf6d32 100644 --- a/src/commands/utils/hamt-utils.ts +++ b/packages/unixfs/src/commands/utils/hamt-utils.ts @@ -9,7 +9,7 @@ import { hamtHashFn, hamtBucketBits } from './hamt-constants.js' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import type { Mtime } from 'ipfs-unixfs' import type { AbortOptions } from '@libp2p/interfaces' import type { ImportResult } from 'ipfs-unixfs-importer' @@ -40,7 +40,7 @@ export interface CreateShardOptions { cidVersion: Version } -export const createShard = async (blockstore: Blockstore, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise => { +export const createShard = async (blockstore: Blocks, contents: Array<{ name: string, size: bigint, cid: CID }>, options: CreateShardOptions): Promise => { const shard = new DirSharded({ root: true, dir: true, @@ -75,7 +75,7 @@ export interface HAMTPath { node: dagPB.PBNode } -export const updateShardedDirectory = async (path: HAMTPath[], blockstore: Blockstore, options: PersistOptions): Promise<{ cid: CID, node: dagPB.PBNode }> => { +export const updateShardedDirectory = async (path: HAMTPath[], blockstore: Blocks, options: PersistOptions): Promise<{ cid: CID, node: dagPB.PBNode }> => { // persist any metadata on the shard root const shardRoot = UnixFS.unmarshal(path[0].node.Data ?? new Uint8Array(0)) @@ -142,7 +142,7 @@ export const updateShardedDirectory = async (path: HAMTPath[], blockstore: Block return { cid, node } } -export const recreateShardedDirectory = async (cid: CID, fileName: string, blockstore: Blockstore, options: AbortOptions): Promise<{ path: HAMTPath[], hash: InfiniteHash }> => { +export const recreateShardedDirectory = async (cid: CID, fileName: string, blockstore: Blocks, options: AbortOptions): Promise<{ path: HAMTPath[], hash: InfiniteHash }> => { const wrapped = wrapHash(hamtHashFn) const hash = wrapped(uint8ArrayFromString(fileName)) const path: HAMTPath[] = [] diff --git a/src/commands/utils/is-over-shard-threshold.ts b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts similarity index 92% rename from src/commands/utils/is-over-shard-threshold.ts rename to packages/unixfs/src/commands/utils/is-over-shard-threshold.ts index 9902eb90..093dee6d 100644 --- a/src/commands/utils/is-over-shard-threshold.ts +++ b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts @@ -1,5 +1,5 @@ import type { PBNode } from '@ipld/dag-pb' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import { UnixFS } from 'ipfs-unixfs' import * as dagPb from '@ipld/dag-pb' import { CID_V0, CID_V1 } from './dir-sharded.js' @@ -10,7 +10,7 @@ import { CID_V0, CID_V1 } from './dir-sharded.js' * * If the node is a hamt sharded directory the calculation is based on if it was a regular directory. */ -export async function isOverShardThreshold (node: PBNode, blockstore: Blockstore, threshold: number): Promise { +export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number): Promise { if (node.Data == null) { throw new Error('DagPB node had no data') } @@ -42,7 +42,7 @@ function estimateNodeSize (node: PBNode): number { return size } -async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blockstore): Promise { +async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks): Promise { if (current > max) { return max } diff --git a/src/commands/utils/persist.ts b/packages/unixfs/src/commands/utils/persist.ts similarity index 74% rename from src/commands/utils/persist.ts rename to packages/unixfs/src/commands/utils/persist.ts index 567663a0..ba14330d 100644 --- a/src/commands/utils/persist.ts +++ b/packages/unixfs/src/commands/utils/persist.ts @@ -1,7 +1,7 @@ import { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' import { sha256 } from 'multiformats/hashes/sha2' -import type { Blockstore } from 'ipfs-unixfs-importer' +import type { Blocks } from '@helia/interface/blocks' import type { BlockCodec } from 'multiformats/codecs/interface' import type { Version as CIDVersion } from 'multiformats/cid' @@ -11,7 +11,9 @@ export interface PersistOptions { signal?: AbortSignal } -export const persist = async (buffer: Uint8Array, blockstore: Blockstore, options: PersistOptions): Promise => { +type PutStore = Pick + +export const persist = async (buffer: Uint8Array, blockstore: PutStore, options: PersistOptions): Promise => { if (options.codec == null) { options.codec = dagPb } @@ -20,6 +22,7 @@ export const persist = async (buffer: Uint8Array, blockstore: Blockstore, option const cid = CID.create(options.cidVersion, options.codec.code, multihash) await blockstore.put(cid, buffer, { + ...options, signal: options.signal }) diff --git a/src/commands/utils/remove-link.ts b/packages/unixfs/src/commands/utils/remove-link.ts similarity index 92% rename from src/commands/utils/remove-link.ts rename to packages/unixfs/src/commands/utils/remove-link.ts index 8b2a9a57..f74e2c04 100644 --- a/src/commands/utils/remove-link.ts +++ b/packages/unixfs/src/commands/utils/remove-link.ts @@ -9,7 +9,7 @@ import { updateShardedDirectory } from './hamt-utils.js' import type { PBNode } from '@ipld/dag-pb' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import type { Directory } from './cid-to-directory.js' import type { AbortOptions } from '@libp2p/interfaces' import { InvalidParametersError, InvalidPBNodeError } from './errors.js' @@ -29,7 +29,7 @@ export interface RemoveLinkResult { cid: CID } -export async function removeLink (parent: Directory, name: string, blockstore: Blockstore, options: RmLinkOptions): Promise { +export async function removeLink (parent: Directory, name: string, blockstore: Blocks, options: RmLinkOptions): Promise { if (parent.node.Data == null) { throw new InvalidPBNodeError('Parent node had no data') } @@ -55,7 +55,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B return await removeFromDirectory(parent, name, blockstore, options) } -const removeFromDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: AbortOptions): Promise => { +const removeFromDirectory = async (parent: Directory, name: string, blockstore: Blocks, options: AbortOptions): Promise => { // Remove existing link if it exists parent.node.Links = parent.node.Links.filter((link) => { return link.Name !== name @@ -75,7 +75,7 @@ const removeFromDirectory = async (parent: Directory, name: string, blockstore: } } -const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blockstore, options: UpdateHamtDirectoryOptions): Promise<{ cid: CID, node: PBNode }> => { +const removeFromShardedDirectory = async (parent: Directory, name: string, blockstore: Blocks, options: UpdateHamtDirectoryOptions): Promise<{ cid: CID, node: PBNode }> => { const { path } = await recreateShardedDirectory(parent.cid, name, blockstore, options) const finalSegment = path[path.length - 1] @@ -132,7 +132,7 @@ const removeFromShardedDirectory = async (parent: Directory, name: string, block return await updateShardedDirectory(path, blockstore, options) } -const convertToFlatDirectory = async (parent: Directory, blockstore: Blockstore, options: RmLinkOptions): Promise => { +const convertToFlatDirectory = async (parent: Directory, blockstore: Blocks, options: RmLinkOptions): Promise => { if (parent.node.Data == null) { throw new InvalidParametersError('Invalid parent passed to convertToFlatDirectory') } diff --git a/src/commands/utils/resolve.ts b/packages/unixfs/src/commands/utils/resolve.ts similarity index 93% rename from src/commands/utils/resolve.ts rename to packages/unixfs/src/commands/utils/resolve.ts index 55f9a79d..08094fb6 100644 --- a/src/commands/utils/resolve.ts +++ b/packages/unixfs/src/commands/utils/resolve.ts @@ -6,7 +6,7 @@ import { DoesNotExistError, InvalidParametersError } from './errors.js' import { addLink } from './add-link.js' import { cidToDirectory } from './cid-to-directory.js' import { cidToPBLink } from './cid-to-pblink.js' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' const log = logger('helia:unixfs:components:utils:resolve') @@ -32,7 +32,7 @@ export interface ResolveResult { segments?: Segment[] } -export async function resolve (cid: CID, path: string | undefined, blockstore: Blockstore, options: AbortOptions): Promise { +export async function resolve (cid: CID, path: string | undefined, blockstore: Blocks, options: AbortOptions): Promise { if (path == null || path === '') { return { cid } } @@ -101,7 +101,7 @@ export interface UpdatePathCidsOptions extends AbortOptions { * Where we have descended into a DAG to update a child node, ascend up the DAG creating * new hashes and blocks for the changed content */ -export async function updatePathCids (cid: CID, result: ResolveResult, blockstore: Blockstore, options: UpdatePathCidsOptions): Promise { +export async function updatePathCids (cid: CID, result: ResolveResult, blockstore: Blocks, options: UpdatePathCidsOptions): Promise { if (result.segments == null || result.segments.length === 0) { return cid } diff --git a/src/index.ts b/packages/unixfs/src/index.ts similarity index 98% rename from src/index.ts rename to packages/unixfs/src/index.ts index 23b05d36..e7468932 100644 --- a/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -32,7 +32,7 @@ */ import type { CID, Version } from 'multiformats/cid' -import type { Blockstore } from 'interface-blockstore' +import type { Blocks } from '@helia/interface/blocks' import type { AbortOptions } from '@libp2p/interfaces' import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js' import { cat } from './commands/cat.js' @@ -48,7 +48,7 @@ import { ls } from './commands/ls.js' import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportResult } from 'ipfs-unixfs-importer' export interface UnixFSComponents { - blockstore: Blockstore + blockstore: Blocks } /** @@ -545,6 +545,6 @@ class DefaultUnixFS implements UnixFS { /** * Create a {@link UnixFS} instance for use with {@link https://github.com/ipfs/helia Helia} */ -export function unixfs (helia: { blockstore: Blockstore }): UnixFS { +export function unixfs (helia: { blockstore: Blocks }): UnixFS { return new DefaultUnixFS(helia) } diff --git a/test/add.spec.ts b/packages/unixfs/test/add.spec.ts similarity index 100% rename from test/add.spec.ts rename to packages/unixfs/test/add.spec.ts diff --git a/test/cat.spec.ts b/packages/unixfs/test/cat.spec.ts similarity index 100% rename from test/cat.spec.ts rename to packages/unixfs/test/cat.spec.ts diff --git a/test/chmod.spec.ts b/packages/unixfs/test/chmod.spec.ts similarity index 100% rename from test/chmod.spec.ts rename to packages/unixfs/test/chmod.spec.ts diff --git a/test/cp.spec.ts b/packages/unixfs/test/cp.spec.ts similarity index 100% rename from test/cp.spec.ts rename to packages/unixfs/test/cp.spec.ts diff --git a/test/fixtures/create-sharded-directory.ts b/packages/unixfs/test/fixtures/create-sharded-directory.ts similarity index 100% rename from test/fixtures/create-sharded-directory.ts rename to packages/unixfs/test/fixtures/create-sharded-directory.ts diff --git a/test/fixtures/create-subsharded-directory.ts b/packages/unixfs/test/fixtures/create-subsharded-directory.ts similarity index 100% rename from test/fixtures/create-subsharded-directory.ts rename to packages/unixfs/test/fixtures/create-subsharded-directory.ts diff --git a/test/fixtures/files.ts b/packages/unixfs/test/fixtures/files.ts similarity index 100% rename from test/fixtures/files.ts rename to packages/unixfs/test/fixtures/files.ts diff --git a/test/fixtures/print-tree.ts b/packages/unixfs/test/fixtures/print-tree.ts similarity index 84% rename from test/fixtures/print-tree.ts rename to packages/unixfs/test/fixtures/print-tree.ts index e2cb75a4..72c0aa33 100644 --- a/test/fixtures/print-tree.ts +++ b/packages/unixfs/test/fixtures/print-tree.ts @@ -4,7 +4,9 @@ import type { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' import * as raw from 'multiformats/codecs/raw' -export async function printTree (cid: CID, blockstore: Blockstore, name: string = '', indent: string = ''): Promise { +type GetStore = Pick + +export async function printTree (cid: CID, blockstore: GetStore, name: string = '', indent: string = ''): Promise { const block = await blockstore.get(cid) if (cid.code === dagPb.code) { diff --git a/test/ls.spec.ts b/packages/unixfs/test/ls.spec.ts similarity index 100% rename from test/ls.spec.ts rename to packages/unixfs/test/ls.spec.ts diff --git a/test/mkdir.spec.ts b/packages/unixfs/test/mkdir.spec.ts similarity index 100% rename from test/mkdir.spec.ts rename to packages/unixfs/test/mkdir.spec.ts diff --git a/test/rm.spec.ts b/packages/unixfs/test/rm.spec.ts similarity index 100% rename from test/rm.spec.ts rename to packages/unixfs/test/rm.spec.ts diff --git a/test/stat.spec.ts b/packages/unixfs/test/stat.spec.ts similarity index 100% rename from test/stat.spec.ts rename to packages/unixfs/test/stat.spec.ts diff --git a/test/touch.spec.ts b/packages/unixfs/test/touch.spec.ts similarity index 100% rename from test/touch.spec.ts rename to packages/unixfs/test/touch.spec.ts diff --git a/tsconfig.json b/packages/unixfs/tsconfig.json similarity index 100% rename from tsconfig.json rename to packages/unixfs/tsconfig.json From 4c8d1240eb5eec1d7e2e018e3316cc2c8411762d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Mar 2023 16:08:04 +0100 Subject: [PATCH 035/253] docs: update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b1c0443..871658ff 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@

-# @helia/ipns +# @helia/unixfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) > An implementation of Unixfs for Helia From de78f4d03ebafe9ed9a2dfcbfb7a516fa215585c Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 15 Mar 2023 18:24:52 +0100 Subject: [PATCH 036/253] feat: expose progress events from importer, blockstore and bitswap (#13) Expose progress event types that are passed to `onProgress` callback passed to all operations. --- packages/interop/package.json | 3 +- packages/interop/test/bitswap.spec.ts | 82 +++++++++++++++++++++++++++ packages/interop/test/files.spec.ts | 12 ++-- packages/unixfs/package.json | 8 ++- packages/unixfs/src/commands/chmod.ts | 3 +- packages/unixfs/src/index.ts | 50 +++++++++------- 6 files changed, 127 insertions(+), 31 deletions(-) create mode 100644 packages/interop/test/bitswap.spec.ts diff --git a/packages/interop/package.json b/packages/interop/package.json index 7febd31c..d8e99cb4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -64,8 +64,9 @@ "go-ipfs": "^0.18.1", "helia": "next", "ipfs-core-types": "^0.14.0", - "ipfs-unixfs-importer": "^15.0.0", + "ipfs-unixfs-importer": "^15.0.1", "ipfsd-ctl": "^13.0.0", + "it-to-buffer": "^3.0.1", "kubo-rpc-client": "^3.0.0", "libp2p": "next", "merge-options": "^3.0.4", diff --git a/packages/interop/test/bitswap.spec.ts b/packages/interop/test/bitswap.spec.ts new file mode 100644 index 00000000..c67fc611 --- /dev/null +++ b/packages/interop/test/bitswap.spec.ts @@ -0,0 +1,82 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { UnixFS, unixfs } from '@helia/unixfs' +import type { FileCandidate } from 'ipfs-unixfs-importer' +import toBuffer from 'it-to-buffer' + +describe('unixfs bitswap interop', () => { + let helia: Helia + let unixFs: UnixFS + let kubo: Controller + + beforeEach(async () => { + helia = await createHeliaNode() + unixFs = unixfs(helia) + kubo = await createKuboNode() + + // connect helia to kubo + await helia.libp2p.peerStore.addressBook.add(kubo.peer.id, kubo.peer.addresses) + await helia.libp2p.dial(kubo.peer.id) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should add a large file to helia and fetch it from kubo', async () => { + const chunkSize = 1024 * 1024 + const size = chunkSize * 10 + const input: Uint8Array[] = [] + + const candidate: FileCandidate = { + content: (async function * () { + for (let i = 0; i < size; i += chunkSize) { + const buf = new Uint8Array(chunkSize) + input.push(buf) + + yield buf + } + }()) + } + + const cid = await unixFs.addFile(candidate) + + const bytes = await toBuffer(kubo.api.cat(cid)) + + expect(bytes).to.equalBytes(await toBuffer(input)) + }) + + it('should add a large file to kubo and fetch it from helia', async () => { + const chunkSize = 1024 * 1024 + const size = chunkSize * 10 + const input: Uint8Array[] = [] + + const candidate: FileCandidate = { + content: (async function * () { + for (let i = 0; i < size; i += chunkSize) { + const buf = new Uint8Array(chunkSize) + input.push(buf) + + yield buf + } + }()) + } + + const { cid } = await kubo.api.add(candidate.content) + + const bytes = await toBuffer(unixFs.cat(cid)) + + expect(bytes).to.equalBytes(await toBuffer(input)) + }) +}) diff --git a/packages/interop/test/files.spec.ts b/packages/interop/test/files.spec.ts index dcd5b5c5..94f75604 100644 --- a/packages/interop/test/files.spec.ts +++ b/packages/interop/test/files.spec.ts @@ -5,31 +5,31 @@ import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' import type { Controller } from 'ipfsd-ctl' -import { UnixFS, unixfs } from '@helia/unixfs' +import { AddOptions, UnixFS, unixfs } from '@helia/unixfs' import { balanced } from 'ipfs-unixfs-importer/layout' import { fixedSize } from 'ipfs-unixfs-importer/chunker' -import type { FileCandidate, ImporterOptions } from 'ipfs-unixfs-importer' +import type { FileCandidate } from 'ipfs-unixfs-importer' import type { CID } from 'multiformats/cid' -import type { AddOptions } from 'ipfs-core-types/src/root.js' +import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js' describe('unixfs interop', () => { let helia: Helia let unixFs: UnixFS let kubo: Controller - async function importToHelia (data: FileCandidate, opts?: Partial): Promise { + async function importToHelia (data: FileCandidate, opts?: Partial): Promise { const cid = await unixFs.addFile(data, opts) return cid } - async function importToKubo (data: FileCandidate, opts?: AddOptions): Promise { + async function importToKubo (data: FileCandidate, opts?: KuboAddOptions): Promise { const result = await kubo.api.add(data.content, opts) return result.cid } - async function expectSameCid (data: () => FileCandidate, heliaOpts: Partial = {}, kuboOpts: AddOptions = {}): Promise { + async function expectSameCid (data: () => FileCandidate, heliaOpts: Partial = {}, kuboOpts: KuboAddOptions = {}): Promise { const heliaCid = await importToHelia(data(), { // these are the default kubo options cidVersion: 0, diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 890959ca..8a315279 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -147,12 +147,13 @@ "hamt-sharding": "^3.0.2", "interface-blockstore": "^5.0.0", "ipfs-unixfs": "^11.0.0", - "ipfs-unixfs-exporter": "^13.0.0", - "ipfs-unixfs-importer": "^15.0.0", + "ipfs-unixfs-exporter": "^13.0.1", + "ipfs-unixfs-importer": "^15.0.1", "it-last": "^2.0.0", "it-pipe": "^2.0.5", "merge-options": "^3.0.4", "multiformats": "^11.0.1", + "progress-events": "^1.0.0", "sparse-array": "^1.3.2" }, "devDependencies": { @@ -164,5 +165,8 @@ "it-first": "^2.0.0", "it-to-buffer": "^3.0.0", "uint8arrays": "^4.0.3" + }, + "typedoc": { + "entryPoint": "./src/index.ts" } } diff --git a/packages/unixfs/src/commands/chmod.ts b/packages/unixfs/src/commands/chmod.ts index 0836a296..3785f058 100644 --- a/packages/unixfs/src/commands/chmod.ts +++ b/packages/unixfs/src/commands/chmod.ts @@ -36,7 +36,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options // but do not reimport files, only manipulate dag-pb nodes const root = await pipe( async function * () { - for await (const entry of recursive(resolved.cid, blockstore)) { + for await (const entry of recursive(resolved.cid, blockstore, options)) { let metadata: UnixFS let links: PBLink[] = [] @@ -63,6 +63,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options } } }, + // @ts-expect-error cannot combine progress types (source) => importer(source, blockstore, { ...opts, dagBuilder: async function * (source, block) { diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index e7468932..d20f303a 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -32,7 +32,7 @@ */ import type { CID, Version } from 'multiformats/cid' -import type { Blocks } from '@helia/interface/blocks' +import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' import type { AbortOptions } from '@libp2p/interfaces' import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js' import { cat } from './commands/cat.js' @@ -45,16 +45,24 @@ import { touch } from './commands/touch.js' import { chmod } from './commands/chmod.js' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' import { ls } from './commands/ls.js' -import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportResult } from 'ipfs-unixfs-importer' +import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportProgressEvents, ImportResult } from 'ipfs-unixfs-importer' +import type { ProgressOptions } from 'progress-events' export interface UnixFSComponents { blockstore: Blocks } +export type AddEvents = PutBlockProgressEvents +| ImportProgressEvents + +export interface AddOptions extends AbortOptions, Omit, ProgressOptions { + +} + /** * Options to pass to the cat command */ -export interface CatOptions extends AbortOptions { +export interface CatOptions extends AbortOptions, ProgressOptions { /** * Start reading the file at this offset */ @@ -74,7 +82,7 @@ export interface CatOptions extends AbortOptions { /** * Options to pass to the chmod command */ -export interface ChmodOptions extends AbortOptions { +export interface ChmodOptions extends AbortOptions, ProgressOptions { /** * If the target of the operation is a directory and this is true, * apply the new mode to all directory contents @@ -96,7 +104,7 @@ export interface ChmodOptions extends AbortOptions { /** * Options to pass to the cp command */ -export interface CpOptions extends AbortOptions { +export interface CpOptions extends AbortOptions, ProgressOptions { /** * If true, allow overwriting existing directory entries (default: false) */ @@ -112,7 +120,7 @@ export interface CpOptions extends AbortOptions { /** * Options to pass to the ls command */ -export interface LsOptions extends AbortOptions { +export interface LsOptions extends AbortOptions, ProgressOptions { /** * Optional path to list subdirectory contents if the target CID resolves to * a directory @@ -133,7 +141,7 @@ export interface LsOptions extends AbortOptions { /** * Options to pass to the mkdir command */ -export interface MkdirOptions extends AbortOptions { +export interface MkdirOptions extends AbortOptions, ProgressOptions { /** * The CID version to create the new directory with - defaults to the same * version as the containing directory @@ -165,7 +173,7 @@ export interface MkdirOptions extends AbortOptions { /** * Options to pass to the rm command */ -export interface RmOptions extends AbortOptions { +export interface RmOptions extends AbortOptions, ProgressOptions { /** * DAGs with a root block larger than this value will be sharded. Blocks * smaller than this value will be regular UnixFS directories. @@ -176,7 +184,7 @@ export interface RmOptions extends AbortOptions { /** * Options to pass to the stat command */ -export interface StatOptions extends AbortOptions { +export interface StatOptions extends AbortOptions, ProgressOptions { /** * An optional path to allow statting paths inside directories */ @@ -292,7 +300,7 @@ export interface UnixFS { * } * ``` */ - addAll: (source: ImportCandidateStream, options?: Partial) => AsyncIterable + addAll: (source: ImportCandidateStream, options?: Partial) => AsyncIterable /** * Add a single `Uint8Array` to your Helia node as a file. @@ -305,7 +313,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addBytes: (bytes: Uint8Array, options?: Partial) => Promise + addBytes: (bytes: Uint8Array, options?: Partial) => Promise /** * Add a stream of `Uint8Array` to your Helia node as a file. @@ -321,7 +329,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addByteStream: (bytes: ByteStream, options?: Partial) => Promise + addByteStream: (bytes: ByteStream, options?: Partial) => Promise /** * Add a file to your Helia node with optional metadata. @@ -342,7 +350,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addFile: (file: FileCandidate, options?: Partial) => Promise + addFile: (file: FileCandidate, options?: Partial) => Promise /** * Add a directory to your Helia node. @@ -355,7 +363,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addDirectory: (dir?: Partial, options?: Partial) => Promise + addDirectory: (dir?: Partial, options?: Partial) => Promise /** * Retrieve the contents of a file from your Helia node. @@ -368,7 +376,7 @@ export interface UnixFS { * } * ``` */ - cat: (cid: CID, options?: Partial) => AsyncIterable + cat: (cid: CID, options?: Partial & ProgressOptions) => AsyncIterable /** * Change the permissions on a file or directory in a DAG @@ -415,7 +423,7 @@ export interface UnixFS { * } * ``` */ - ls: (cid: CID, options?: Partial) => AsyncIterable + ls: (cid: CID, options?: Partial & ProgressOptions) => AsyncIterable /** * Make a new directory under an existing directory. @@ -489,23 +497,23 @@ class DefaultUnixFS implements UnixFS { this.components = components } - async * addAll (source: ImportCandidateStream, options: Partial = {}): AsyncIterable { + async * addAll (source: ImportCandidateStream, options: Partial = {}): AsyncIterable { yield * addAll(source, this.components.blockstore, options) } - async addBytes (bytes: Uint8Array, options: Partial = {}): Promise { + async addBytes (bytes: Uint8Array, options: Partial = {}): Promise { return await addBytes(bytes, this.components.blockstore, options) } - async addByteStream (bytes: ByteStream, options: Partial = {}): Promise { + async addByteStream (bytes: ByteStream, options: Partial = {}): Promise { return await addByteStream(bytes, this.components.blockstore, options) } - async addFile (file: FileCandidate, options: Partial = {}): Promise { + async addFile (file: FileCandidate, options: Partial = {}): Promise { return await addFile(file, this.components.blockstore, options) } - async addDirectory (dir: Partial = {}, options: Partial = {}): Promise { + async addDirectory (dir: Partial = {}, options: Partial = {}): Promise { return await addDirectory(dir, this.components.blockstore, options) } From 972f9715b2ec8f6685897c4b008a668c10e8e072 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 15 Mar 2023 17:33:28 +0000 Subject: [PATCH 037/253] chore(release): 1.1.0 [skip ci] ## [@helia/unixfs-v1.1.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.0.5...@helia/unixfs-v1.1.0) (2023-03-15) ### Features * expose progress events from importer, blockstore and bitswap ([#13](https://github.com/ipfs/helia-unixfs/issues/13)) ([de78f4d](https://github.com/ipfs/helia-unixfs/commit/de78f4d03ebafe9ed9a2dfcbfb7a516fa215585c)) ### Trivial Changes * add interop test suite ([#12](https://github.com/ipfs/helia-unixfs/issues/12)) ([3ad5f5d](https://github.com/ipfs/helia-unixfs/commit/3ad5f5d8199a5596aa333916d4a240584bc0842a)) --- packages/unixfs/CHANGELOG.md | 12 ++++++++++++ packages/unixfs/package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 5b4d157f..be30b9c1 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,15 @@ +## [@helia/unixfs-v1.1.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.0.5...@helia/unixfs-v1.1.0) (2023-03-15) + + +### Features + +* expose progress events from importer, blockstore and bitswap ([#13](https://github.com/ipfs/helia-unixfs/issues/13)) ([de78f4d](https://github.com/ipfs/helia-unixfs/commit/de78f4d03ebafe9ed9a2dfcbfb7a516fa215585c)) + + +### Trivial Changes + +* add interop test suite ([#12](https://github.com/ipfs/helia-unixfs/issues/12)) ([3ad5f5d](https://github.com/ipfs/helia-unixfs/commit/3ad5f5d8199a5596aa333916d4a240584bc0842a)) + ## [1.0.5](https://github.com/ipfs/helia-unixfs/compare/v1.0.4...v1.0.5) (2023-03-14) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 8a315279..40be6aef 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.0.5", + "version": "1.1.0", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 36cf3b2143276a59b685ceb58299c4f881545fee Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 17 Mar 2023 12:27:21 +0100 Subject: [PATCH 038/253] feat: expose unixfs progress events in types (#14) Expands types to include unixfs progress events as well as bitswap and blockstore events. --- packages/interop/package.json | 2 +- packages/unixfs/package.json | 4 ++-- packages/unixfs/src/commands/cat.ts | 5 +---- packages/unixfs/src/commands/touch.ts | 1 + packages/unixfs/src/index.ts | 29 +++++++++++++++------------ 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index d8e99cb4..0f2dee73 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -64,7 +64,7 @@ "go-ipfs": "^0.18.1", "helia": "next", "ipfs-core-types": "^0.14.0", - "ipfs-unixfs-importer": "^15.0.1", + "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^3.0.1", "kubo-rpc-client": "^3.0.0", diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 40be6aef..81015ff2 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -147,8 +147,8 @@ "hamt-sharding": "^3.0.2", "interface-blockstore": "^5.0.0", "ipfs-unixfs": "^11.0.0", - "ipfs-unixfs-exporter": "^13.0.1", - "ipfs-unixfs-importer": "^15.0.1", + "ipfs-unixfs-exporter": "^13.1.0", + "ipfs-unixfs-importer": "^15.1.0", "it-last": "^2.0.0", "it-pipe": "^2.0.5", "merge-options": "^3.0.4", diff --git a/packages/unixfs/src/commands/cat.ts b/packages/unixfs/src/commands/cat.ts index eb31c76c..328cdae9 100644 --- a/packages/unixfs/src/commands/cat.ts +++ b/packages/unixfs/src/commands/cat.ts @@ -25,8 +25,5 @@ export async function * cat (cid: CID, blockstore: Blocks, options: Partial importer(source, blockstore, { ...opts, dagBuilder: async function * (source, block) { diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index d20f303a..2f71c2a8 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -43,9 +43,9 @@ import { rm } from './commands/rm.js' import { stat } from './commands/stat.js' import { touch } from './commands/touch.js' import { chmod } from './commands/chmod.js' -import type { UnixFSEntry } from 'ipfs-unixfs-exporter' +import type { ExporterProgressEvents, UnixFSEntry } from 'ipfs-unixfs-exporter' import { ls } from './commands/ls.js' -import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportProgressEvents, ImportResult } from 'ipfs-unixfs-importer' +import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImporterProgressEvents, ImportResult } from 'ipfs-unixfs-importer' import type { ProgressOptions } from 'progress-events' export interface UnixFSComponents { @@ -53,16 +53,19 @@ export interface UnixFSComponents { } export type AddEvents = PutBlockProgressEvents -| ImportProgressEvents +| ImporterProgressEvents export interface AddOptions extends AbortOptions, Omit, ProgressOptions { } +export type GetEvents = GetBlockProgressEvents +| ExporterProgressEvents + /** * Options to pass to the cat command */ -export interface CatOptions extends AbortOptions, ProgressOptions { +export interface CatOptions extends AbortOptions, ProgressOptions { /** * Start reading the file at this offset */ @@ -82,7 +85,7 @@ export interface CatOptions extends AbortOptions, ProgressOptions { +export interface ChmodOptions extends AbortOptions, ProgressOptions { /** * If the target of the operation is a directory and this is true, * apply the new mode to all directory contents @@ -104,7 +107,7 @@ export interface ChmodOptions extends AbortOptions, ProgressOptions { +export interface CpOptions extends AbortOptions, ProgressOptions { /** * If true, allow overwriting existing directory entries (default: false) */ @@ -120,7 +123,7 @@ export interface CpOptions extends AbortOptions, ProgressOptions { +export interface LsOptions extends AbortOptions, ProgressOptions { /** * Optional path to list subdirectory contents if the target CID resolves to * a directory @@ -141,7 +144,7 @@ export interface LsOptions extends AbortOptions, ProgressOptions { +export interface MkdirOptions extends AbortOptions, ProgressOptions { /** * The CID version to create the new directory with - defaults to the same * version as the containing directory @@ -173,7 +176,7 @@ export interface MkdirOptions extends AbortOptions, ProgressOptions { +export interface RmOptions extends AbortOptions, ProgressOptions { /** * DAGs with a root block larger than this value will be sharded. Blocks * smaller than this value will be regular UnixFS directories. @@ -184,7 +187,7 @@ export interface RmOptions extends AbortOptions, ProgressOptions { +export interface StatOptions extends AbortOptions, ProgressOptions { /** * An optional path to allow statting paths inside directories */ @@ -251,7 +254,7 @@ export interface UnixFSStats { /** * Options to pass to the touch command */ -export interface TouchOptions extends AbortOptions { +export interface TouchOptions extends AbortOptions, ProgressOptions { /** * Optional mtime to set on the DAG root, defaults to the current time */ @@ -376,7 +379,7 @@ export interface UnixFS { * } * ``` */ - cat: (cid: CID, options?: Partial & ProgressOptions) => AsyncIterable + cat: (cid: CID, options?: Partial) => AsyncIterable /** * Change the permissions on a file or directory in a DAG @@ -423,7 +426,7 @@ export interface UnixFS { * } * ``` */ - ls: (cid: CID, options?: Partial & ProgressOptions) => AsyncIterable + ls: (cid: CID, options?: Partial) => AsyncIterable /** * Make a new directory under an existing directory. From 38637a4629abe0540780ad9f9a7dbcb3b7f2782e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Mar 2023 11:37:01 +0000 Subject: [PATCH 039/253] chore(release): 1.2.0 [skip ci] ## [@helia/unixfs-v1.2.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.1.0...@helia/unixfs-v1.2.0) (2023-03-17) ### Features * expose unixfs progress events in types ([#14](https://github.com/ipfs/helia-unixfs/issues/14)) ([36cf3b2](https://github.com/ipfs/helia-unixfs/commit/36cf3b2143276a59b685ceb58299c4f881545fee)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index be30b9c1..483bb8be 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.2.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.1.0...@helia/unixfs-v1.2.0) (2023-03-17) + + +### Features + +* expose unixfs progress events in types ([#14](https://github.com/ipfs/helia-unixfs/issues/14)) ([36cf3b2](https://github.com/ipfs/helia-unixfs/commit/36cf3b2143276a59b685ceb58299c4f881545fee)) + ## [@helia/unixfs-v1.1.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.0.5...@helia/unixfs-v1.1.0) (2023-03-15) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 81015ff2..9777f360 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.1.0", + "version": "1.2.0", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 6866638830f32442f9cfeadbde795e74b0865e00 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 21 Mar 2023 10:46:49 +0000 Subject: [PATCH 040/253] deps: update ipns to 6.x.x (#12) Updates to ipns@6.x.x with interface-datastore@8.x.x --- packages/interop/package.json | 2 +- packages/ipns/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index c46c747d..562de9af 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -69,7 +69,7 @@ "go-ipfs": "^0.18.1", "helia": "next", "ipfsd-ctl": "^13.0.0", - "ipns": "^5.0.1", + "ipns": "^6.0.0", "it-all": "^2.0.0", "it-last": "^2.0.0", "it-map": "^2.0.0", diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 71dd7da0..7c1271c0 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -168,7 +168,7 @@ "@libp2p/record": "^3.0.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", - "ipns": "^5.0.1", + "ipns": "^6.0.0", "is-ipfs": "^8.0.1", "multiformats": "^11.0.1", "p-queue": "^7.3.0", From ad76689266610925f37acc090922acf20adee626 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 21 Mar 2023 10:54:57 +0000 Subject: [PATCH 041/253] chore(release): 1.0.1 [skip ci] ## [@helia/ipns-v1.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.0.0...@helia/ipns-v1.0.1) (2023-03-21) ### Documentation * fix typos ([#4](https://github.com/ipfs/helia-ipns/issues/4)) ([4369653](https://github.com/ipfs/helia-ipns/commit/4369653892d1434b9519f8f7f93371ae4531bc21)) ### Dependencies * update blockstore/datastore deps ([#10](https://github.com/ipfs/helia-ipns/issues/10)) ([3189737](https://github.com/ipfs/helia-ipns/commit/3189737040a9dfe631e1d07f7f884ff19b873f17)) * update ipns to 6.x.x ([#12](https://github.com/ipfs/helia-ipns/issues/12)) ([6866638](https://github.com/ipfs/helia-ipns/commit/6866638830f32442f9cfeadbde795e74b0865e00)) --- packages/ipns/CHANGELOG.md | 13 +++++++++++++ packages/ipns/package.json | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index e1284b39..2f85d2d5 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,16 @@ +## [@helia/ipns-v1.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.0.0...@helia/ipns-v1.0.1) (2023-03-21) + + +### Documentation + +* fix typos ([#4](https://github.com/ipfs/helia-ipns/issues/4)) ([4369653](https://github.com/ipfs/helia-ipns/commit/4369653892d1434b9519f8f7f93371ae4531bc21)) + + +### Dependencies + +* update blockstore/datastore deps ([#10](https://github.com/ipfs/helia-ipns/issues/10)) ([3189737](https://github.com/ipfs/helia-ipns/commit/3189737040a9dfe631e1d07f7f884ff19b873f17)) +* update ipns to 6.x.x ([#12](https://github.com/ipfs/helia-ipns/issues/12)) ([6866638](https://github.com/ipfs/helia-ipns/commit/6866638830f32442f9cfeadbde795e74b0865e00)) + ## @helia/ipns-v1.0.0 (2023-02-15) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 7c1271c0..c97ef617 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.0.0", + "version": "1.0.1", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 0f43c84be417926ef433602ce95b48a1559d459c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:51:40 +0000 Subject: [PATCH 042/253] deps(dev): bump go-ipfs from 0.18.1 to 0.19.0 (#15) Bumps [go-ipfs](https://github.com/ipfs/npm-go-ipfs) from 0.18.1 to 0.19.0. - [Release notes](https://github.com/ipfs/npm-go-ipfs/releases) - [Commits](https://github.com/ipfs/npm-go-ipfs/compare/v0.18.1...v0.19.0) --- updated-dependencies: - dependency-name: go-ipfs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 0f2dee73..eb844a98 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -61,7 +61,7 @@ "aegir": "^38.1.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.18.1", + "go-ipfs": "^0.19.0", "helia": "next", "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", From 74975903ec619a4662e5bfa9546997641e9f8e8c Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 23 Mar 2023 17:38:06 +0000 Subject: [PATCH 043/253] deps: update helia deps to v1 (#16) Helia v1 has shipped so update dep versions --- packages/interop/package.json | 6 +++--- packages/unixfs/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index eb844a98..17f176b3 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -54,7 +54,7 @@ "devDependencies": { "@chainsafe/libp2p-noise": "^11.0.0", "@chainsafe/libp2p-yamux": "^3.0.5", - "@helia/interface": "next", + "@helia/interface": "^1.0.0", "@helia/unixfs": "^1.0.0", "@libp2p/tcp": "^6.1.2", "@libp2p/websockets": "^5.0.3", @@ -62,13 +62,13 @@ "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", "go-ipfs": "^0.19.0", - "helia": "next", + "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^3.0.1", "kubo-rpc-client": "^3.0.0", - "libp2p": "next", + "libp2p": "^0.43.2", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "wherearewe": "^2.0.1" diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 9777f360..dfc8e01a 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -139,7 +139,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "next", + "@helia/interface": "^1.0.0", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", "@libp2p/logger": "^2.0.6", From a51fbe31e8debfd2ee209f3ba38fd9dbc0372dc3 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 23 Mar 2023 17:42:42 +0000 Subject: [PATCH 044/253] deps: update helia deps to v1 (#14) Now that Helia v1 has shipped, update all the deps --- packages/interop/.aegir.js | 4 ++-- packages/interop/package.json | 8 ++++---- packages/interop/test/fixtures/connect.ts | 3 ++- packages/interop/test/fixtures/create-helia.ts | 2 +- packages/interop/test/fixtures/create-kubo.ts | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index 498799ac..cc52af59 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -18,8 +18,8 @@ export default { config: { Addresses: { Swarm: [ - "/ip4/0.0.0.0/tcp/4001", - "/ip4/0.0.0.0/tcp/4002/ws" + "/ip4/127.0.0.1/tcp/4001", + "/ip4/127.0.0.1/tcp/4002/ws" ] } } diff --git a/packages/interop/package.json b/packages/interop/package.json index 562de9af..9b17f715 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -55,7 +55,7 @@ "@chainsafe/libp2p-gossipsub": "^6.1.0", "@chainsafe/libp2p-noise": "^11.0.0", "@chainsafe/libp2p-yamux": "^3.0.5", - "@helia/interface": "next", + "@helia/interface": "^1.0.0", "@helia/ipns": "^1.0.0", "@libp2p/interface-peer-id": "^2.0.1", "@libp2p/kad-dht": "^8.0.0", @@ -66,15 +66,15 @@ "aegir": "^38.1.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.18.1", - "helia": "next", + "go-ipfs": "^0.19.0", + "helia": "^1.0.0", "ipfsd-ctl": "^13.0.0", "ipns": "^6.0.0", "it-all": "^2.0.0", "it-last": "^2.0.0", "it-map": "^2.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "next", + "libp2p": "^0.43.2", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "uint8arrays": "^4.0.3", diff --git a/packages/interop/test/fixtures/connect.ts b/packages/interop/test/fixtures/connect.ts index e7685a3b..38b5f0fe 100644 --- a/packages/interop/test/fixtures/connect.ts +++ b/packages/interop/test/fixtures/connect.ts @@ -14,5 +14,6 @@ export async function connect (helia: Helia, kubo: Controller, protocol: string) break } catch { } } - expect(connected).to.be.true('could not connect hHlia to Kubo') + + expect(connected).to.be.true('could not connect Helia to Kubo') } diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index 2515ca35..e2076c58 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -14,7 +14,7 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise = {}): Pr config: { Addresses: { Swarm: [ - '/ip4/0.0.0.0/tcp/4001', - '/ip4/0.0.0.0/tcp/4002/ws' + '/ip4/127.0.0.1/tcp/4001', + '/ip4/127.0.0.1/tcp/4002/ws' ] } } From 5b0c023e27207fad01ab222c1c4e42071815375c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 23 Mar 2023 17:47:59 +0000 Subject: [PATCH 045/253] chore(release): 1.2.1 [skip ci] ## [@helia/unixfs-v1.2.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.0...@helia/unixfs-v1.2.1) (2023-03-23) ### Dependencies * update helia deps to v1 ([#16](https://github.com/ipfs/helia-unixfs/issues/16)) ([7497590](https://github.com/ipfs/helia-unixfs/commit/74975903ec619a4662e5bfa9546997641e9f8e8c)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 483bb8be..e00510b6 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.2.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.0...@helia/unixfs-v1.2.1) (2023-03-23) + + +### Dependencies + +* update helia deps to v1 ([#16](https://github.com/ipfs/helia-unixfs/issues/16)) ([7497590](https://github.com/ipfs/helia-unixfs/commit/74975903ec619a4662e5bfa9546997641e9f8e8c)) + ## [@helia/unixfs-v1.2.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.1.0...@helia/unixfs-v1.2.0) (2023-03-17) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index dfc8e01a..6eaac804 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.2.0", + "version": "1.2.1", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 43e32a20f44fffd533531a57e6d60883cebc55ca Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 29 Mar 2023 16:01:29 +0100 Subject: [PATCH 046/253] feat: allow publish/resolve using only local datastore (#15) Adds an `offline` option to publish and resolve that causes this module to only use the local datastore instead of any configured routers. --- packages/ipns/package.json | 3 +- packages/ipns/src/index.ts | 46 +++++++++++++++++++++--------- packages/ipns/test/publish.spec.ts | 19 ++++++++++-- packages/ipns/test/resolve.spec.ts | 29 +++++++++++++++++-- 4 files changed, 77 insertions(+), 20 deletions(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index c97ef617..427857c5 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -179,7 +179,8 @@ "@libp2p/peer-id-factory": "^2.0.1", "aegir": "^38.1.0", "datastore-core": "^9.0.3", - "sinon": "^15.0.1" + "sinon": "^15.0.1", + "sinon-ts": "^1.0.0" }, "browser": { "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 5991e1ea..7e2345d1 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -104,21 +104,33 @@ export type RepublishProgressEvents = export interface PublishOptions extends AbortOptions, ProgressOptions { /** - * Time duration of the record in ms + * Time duration of the record in ms (default: 24hrs) */ lifetime?: number + + /** + * Only publish to a local datastore (default: false) + */ + offline?: boolean } export interface ResolveOptions extends AbortOptions, ProgressOptions { /** - * do not use cached entries + * Do not query the network for the IPNS record (default: false) + */ + offline?: boolean +} + +export interface ResolveDNSOptions extends ResolveOptions { + /** + * Do not use cached DNS entries (default: false) */ nocache?: boolean } export interface RepublishOptions extends AbortOptions, ProgressOptions { /** - * The republish interval in ms (default: 24hrs) + * The republish interval in ms (default: 23hrs) */ interval?: number } @@ -140,7 +152,7 @@ export interface IPNS { /** * Resolve a CID from a dns-link style IPNS record */ - resolveDns: (domain: string, options?: ResolveOptions) => Promise + resolveDns: (domain: string, options?: ResolveDNSOptions) => Promise /** * Periodically republish all IPNS records found in the datastore @@ -192,8 +204,10 @@ class DefaultIPNS implements IPNS { await this.localStore.put(routingKey, marshaledRecord, options) - // publish record to routing - await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) + if (options.offline !== true) { + // publish record to routing + await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) + } return record } catch (err: any) { @@ -207,13 +221,13 @@ class DefaultIPNS implements IPNS { const record = await this.#findIpnsRecord(routingKey, options) const str = uint8ArrayToString(record.value) - return await this.#resolve(str) + return await this.#resolve(str, options) } - async resolveDns (domain: string, options: ResolveOptions = {}): Promise { + async resolveDns (domain: string, options: ResolveDNSOptions = {}): Promise { const dnslink = await resolveDnslink(domain, options) - return await this.#resolve(dnslink) + return await this.#resolve(dnslink, options) } republish (options: RepublishOptions = {}): void { @@ -252,14 +266,14 @@ class DefaultIPNS implements IPNS { }, options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS) } - async #resolve (ipfsPath: string): Promise { + async #resolve (ipfsPath: string, options: ResolveOptions = {}): Promise { const parts = ipfsPath.split('/') if (parts.length === 3) { const scheme = parts[1] if (scheme === 'ipns') { - return await this.resolve(peerIdFromString(parts[2])) + return await this.resolve(peerIdFromString(parts[2]), options) } else if (scheme === 'ipfs') { return CID.parse(parts[2]) } @@ -269,12 +283,18 @@ class DefaultIPNS implements IPNS { throw new Error('Invalid value') } - async #findIpnsRecord (routingKey: Uint8Array, options: AbortOptions): Promise { - const routers = [ + async #findIpnsRecord (routingKey: Uint8Array, options: ResolveOptions = {}): Promise { + let routers = [ this.localStore, ...this.routers ] + if (options.offline === true) { + routers = [ + this.localStore + ] + } + const unmarshaledRecord = await Promise.any( routers.map(async (router) => { const unmarshaledRecord = await router.get(routingKey, options) diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index 2ff3ea49..3e8b7aab 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -2,21 +2,25 @@ import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' -import type { IPNS } from '../src/index.js' +import type { IPNS, IPNSRouting } from '../src/index.js' import { ipns } from '../src/index.js' import { CID } from 'multiformats/cid' import { createEd25519PeerId } from '@libp2p/peer-id-factory' import Sinon from 'sinon' +import { StubbedInstance, stubInterface } from 'sinon-ts' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') describe('publish', () => { let name: IPNS + let routing: StubbedInstance - before(async () => { + beforeEach(async () => { const datastore = new MemoryDatastore() + routing = stubInterface() + routing.get.throws(new Error('Not found')) - name = ipns({ datastore }) + name = ipns({ datastore }, [routing]) }) it('should publish an IPNS record with the default params', async function () { @@ -38,6 +42,15 @@ describe('publish', () => { expect(ipnsEntry).to.have.property('ttl', BigInt(lifetime) * 100000n) }) + it('should publish a record offline', async () => { + const key = await createEd25519PeerId() + await name.publish(key, cid, { + offline: true + }) + + expect(routing.put.called).to.be.false() + }) + it('should emit progress events', async function () { const key = await createEd25519PeerId() const onProgress = Sinon.stub() diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index e4891278..70bec018 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -2,21 +2,25 @@ import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' -import type { IPNS } from '../src/index.js' +import type { IPNS, IPNSRouting } from '../src/index.js' import { ipns } from '../src/index.js' import { CID } from 'multiformats/cid' import { createEd25519PeerId } from '@libp2p/peer-id-factory' import Sinon from 'sinon' +import { StubbedInstance, stubInterface } from 'sinon-ts' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') describe('resolve', () => { let name: IPNS + let routing: StubbedInstance - before(async () => { + beforeEach(async () => { const datastore = new MemoryDatastore() + routing = stubInterface() + routing.get.throws(new Error('Not found')) - name = ipns({ datastore }) + name = ipns({ datastore }, [routing]) }) it('should resolve a record', async () => { @@ -32,6 +36,25 @@ describe('resolve', () => { expect(resolvedValue.toString()).to.equal(cid.toString()) }) + it('should resolve a record offline', async () => { + const key = await createEd25519PeerId() + await name.publish(key, cid) + + expect(routing.put.called).to.be.true() + + const resolvedValue = await name.resolve(key, { + offline: true + }) + + expect(routing.get.called).to.be.false() + + if (resolvedValue == null) { + throw new Error('Did not resolve entry') + } + + expect(resolvedValue.toString()).to.equal(cid.toString()) + }) + it('should resolve a recursive record', async () => { const key1 = await createEd25519PeerId() const key2 = await createEd25519PeerId() From e04eca9c4bd9730bfcd974f3fe87d28c9034168a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 29 Mar 2023 15:09:29 +0000 Subject: [PATCH 047/253] chore(release): 1.1.0 [skip ci] ## [@helia/ipns-v1.1.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.0.1...@helia/ipns-v1.1.0) (2023-03-29) ### Features * allow publish/resolve using only local datastore ([#15](https://github.com/ipfs/helia-ipns/issues/15)) ([43e32a2](https://github.com/ipfs/helia-ipns/commit/43e32a20f44fffd533531a57e6d60883cebc55ca)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 2f85d2d5..5a9e78de 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.0.1...@helia/ipns-v1.1.0) (2023-03-29) + + +### Features + +* allow publish/resolve using only local datastore ([#15](https://github.com/ipfs/helia-ipns/issues/15)) ([43e32a2](https://github.com/ipfs/helia-ipns/commit/43e32a20f44fffd533531a57e6d60883cebc55ca)) + ## [@helia/ipns-v1.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.0.0...@helia/ipns-v1.0.1) (2023-03-21) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 427857c5..4cb8035e 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.0.1", + "version": "1.1.0", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 9388c402462a1d45fcb7ded285262881718b7dd0 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 12 Apr 2023 08:19:15 +0100 Subject: [PATCH 048/253] deps: update all it-* deps to latest versions (#25) --- packages/interop/package.json | 2 +- packages/interop/test/bitswap.spec.ts | 4 ++-- packages/unixfs/package.json | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 17f176b3..565bc3eb 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -66,7 +66,7 @@ "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", - "it-to-buffer": "^3.0.1", + "it-to-buffer": "^4.0.1", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.43.2", "merge-options": "^3.0.4", diff --git a/packages/interop/test/bitswap.spec.ts b/packages/interop/test/bitswap.spec.ts index c67fc611..13a5fe96 100644 --- a/packages/interop/test/bitswap.spec.ts +++ b/packages/interop/test/bitswap.spec.ts @@ -54,7 +54,7 @@ describe('unixfs bitswap interop', () => { const bytes = await toBuffer(kubo.api.cat(cid)) - expect(bytes).to.equalBytes(await toBuffer(input)) + expect(bytes).to.equalBytes(toBuffer(input)) }) it('should add a large file to kubo and fetch it from helia', async () => { @@ -77,6 +77,6 @@ describe('unixfs bitswap interop', () => { const bytes = await toBuffer(unixFs.cat(cid)) - expect(bytes).to.equalBytes(await toBuffer(input)) + expect(bytes).to.equalBytes(toBuffer(input)) }) }) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 6eaac804..b2f827a4 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -149,8 +149,8 @@ "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^13.1.0", "ipfs-unixfs-importer": "^15.1.0", - "it-last": "^2.0.0", - "it-pipe": "^2.0.5", + "it-last": "^3.0.1", + "it-pipe": "^3.0.1", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "progress-events": "^1.0.0", @@ -160,10 +160,10 @@ "aegir": "^38.1.0", "blockstore-core": "^4.0.1", "delay": "^5.0.0", - "it-all": "^2.0.0", - "it-drain": "^2.0.0", - "it-first": "^2.0.0", - "it-to-buffer": "^3.0.0", + "it-all": "^3.0.1", + "it-drain": "^3.0.1", + "it-first": "^3.0.1", + "it-to-buffer": "^4.0.1", "uint8arrays": "^4.0.3" }, "typedoc": { From 933179c028805868dcf5bd0a936210eb6d653db9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Apr 2023 07:29:30 +0000 Subject: [PATCH 049/253] chore(release): 1.2.2 [skip ci] ## [@helia/unixfs-v1.2.2](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.1...@helia/unixfs-v1.2.2) (2023-04-12) ### Dependencies * update all it-* deps to latest versions ([#25](https://github.com/ipfs/helia-unixfs/issues/25)) ([9388c40](https://github.com/ipfs/helia-unixfs/commit/9388c402462a1d45fcb7ded285262881718b7dd0)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index e00510b6..7220d239 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.2.2](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.1...@helia/unixfs-v1.2.2) (2023-04-12) + + +### Dependencies + +* update all it-* deps to latest versions ([#25](https://github.com/ipfs/helia-unixfs/issues/25)) ([9388c40](https://github.com/ipfs/helia-unixfs/commit/9388c402462a1d45fcb7ded285262881718b7dd0)) + ## [@helia/unixfs-v1.2.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.0...@helia/unixfs-v1.2.1) (2023-03-23) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index b2f827a4..6c8bb09d 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.2.1", + "version": "1.2.2", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 23a13d844c3b9adfdea13214d974f1c7e1d7539d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 13:46:26 +0100 Subject: [PATCH 050/253] feat: initial import --- .github/dependabot.yml | 11 ++ .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 181 ++++++++++++++++++ .github/workflows/stale.yml | 26 +++ .gitignore | 9 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ README.md | 57 ++++++ package.json | 48 +++++ packages/interop/.aegir.js | 45 +++++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 67 +++++++ packages/interop/package.json | 84 ++++++++ packages/interop/src/index.ts | 1 + .../test/fixtures/create-helia.browser.ts | 41 ++++ .../interop/test/fixtures/create-helia.ts | 38 ++++ packages/interop/test/fixtures/create-kubo.ts | 28 +++ packages/interop/test/strings.spec.ts | 65 +++++++ packages/interop/tsconfig.json | 15 ++ packages/strings/CHANGELOG.md | 83 ++++++++ packages/strings/LICENSE | 4 + packages/strings/LICENSE-APACHE | 5 + packages/strings/LICENSE-MIT | 19 ++ packages/strings/README.md | 67 +++++++ packages/strings/package.json | 154 +++++++++++++++ packages/strings/src/index.ts | 122 ++++++++++++ packages/strings/test/add.spec.ts | 23 +++ packages/strings/test/get.spec.ts | 26 +++ packages/strings/tsconfig.json | 10 + 32 files changed, 1293 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 package.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/test/strings.spec.ts create mode 100644 packages/interop/tsconfig.json create mode 100644 packages/strings/CHANGELOG.md create mode 100644 packages/strings/LICENSE create mode 100644 packages/strings/LICENSE-APACHE create mode 100644 packages/strings/LICENSE-MIT create mode 100644 packages/strings/README.md create mode 100644 packages/strings/package.json create mode 100644 packages/strings/src/index.ts create mode 100644 packages/strings/test/add.spec.ts create mode 100644 packages/strings/test/get.spec.ts create mode 100644 packages/strings/tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..51686414 --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,181 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox-webworker + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit-webworker: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7ad9e674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..85e49462 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# @helia/strings + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-strings.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-strings) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-strings/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-strings/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Add/get IPLD blocks containing strings with your Helia node + +## Table of contents + +- - [Structure](#structure) +- [@helia/unixfs ](#heliaunixfs----omit-in-toc---) + - [API Docs](#api-docs) + - [License](#license) + - [Contribute](#contribute) + +## Structure + +- [`/packages/interop`](./packages/interop) Interop tests for @helia/unixfs +- [`/packages/strings`](./packages/strings) Add/get IPLD blocks containing strings with your Helia node + +

+ + Helia logo + +

+ +# @helia/unixfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/package.json b/package.json new file mode 100644 index 00000000..d0eb9683 --- /dev/null +++ b/package.json @@ -0,0 +1,48 @@ +{ + "name": "@helia/strings", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing strings with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-strings#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-strings.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-strings/issues" + }, + "keywords": [ + "ipfs" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && aegir run release && npm run docs" + }, + "devDependencies": { + "aegir": "^38.1.0" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..7469aa2a --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,67 @@ +# @helia/unixfs-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-strings.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-strings) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-strings/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-strings/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/unixfs + +## Table of contents + +- - [Install](#install) + - [Browser ` +``` + +

+ + Helia logo + +

+ +# @helia/unixfs-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..8fbc53f7 --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,84 @@ +{ + "name": "@helia/unixfs-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/unixfs", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-strings.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-strings/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^3.0.5", + "@helia/interface": "^1.0.0", + "@helia/strings": "~0.0.0", + "@libp2p/tcp": "^6.1.2", + "@libp2p/websockets": "^5.0.3", + "aegir": "^38.1.0", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", + "go-ipfs": "^0.19.0", + "helia": "^1.0.0", + "ipfs-core-types": "^0.14.0", + "ipfs-unixfs-importer": "^15.1.0", + "ipfsd-ctl": "^13.0.0", + "it-to-buffer": "^4.0.1", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.43.2", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..d28c6efa --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,41 @@ +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..7a7201b0 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,38 @@ +import { createHelia } from 'helia' +import { createLibp2p, Libp2pOptions } from 'libp2p' +import { tcp } from '@libp2p/tcp' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..7fd48729 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { Controller, ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import { isElectronMain, isNode } from 'wherearewe' +import mergeOptions from 'merge-options' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return await createController(opts) +} diff --git a/packages/interop/test/strings.spec.ts b/packages/interop/test/strings.spec.ts new file mode 100644 index 00000000..a243a58a --- /dev/null +++ b/packages/interop/test/strings.spec.ts @@ -0,0 +1,65 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { strings, Strings, AddOptions } from '@helia/strings' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' + +describe('strings interop', () => { + let helia: Helia + let str: Strings + let kubo: Controller + + async function expectSameCid (data: () => string, heliaOpts: Partial = {}, kuboOpts: KuboAddOptions = {}): Promise { + const heliaCid = await str.add(data(), heliaOpts) + const kuboCid = await kubo.api.block.put(uint8ArrayFromString(data()), kuboOpts) + + expect(heliaCid.toString()).to.equal(kuboCid.toString()) + } + + beforeEach(async () => { + helia = await createHeliaNode() + str = strings(helia) + kubo = await createKuboNode() + + await helia.libp2p.dial((await (kubo.api.id())).addresses) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should create the same CID for a string', async () => { + const candidate = (): string => 'hello world' + + await expectSameCid(candidate) + }) + + it('should add to helia and fetch from kubo', async () => { + const input = 'hello world' + const cid = await str.add(input) + const block = await kubo.api.block.get(cid) + const output = uint8ArrayToString(block) + + expect(output).to.equal(input) + }) + + it('should add to kubo and fetch from helia', async () => { + const input = 'hello world' + const cid = await kubo.api.block.put(uint8ArrayFromString(input)) + const output = await str.get(cid) + + expect(output).to.equal(input) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..a5e5a4c4 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../strings" + } + ] +} diff --git a/packages/strings/CHANGELOG.md b/packages/strings/CHANGELOG.md new file mode 100644 index 00000000..e00510b6 --- /dev/null +++ b/packages/strings/CHANGELOG.md @@ -0,0 +1,83 @@ +## [@helia/unixfs-v1.2.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.0...@helia/unixfs-v1.2.1) (2023-03-23) + + +### Dependencies + +* update helia deps to v1 ([#16](https://github.com/ipfs/helia-unixfs/issues/16)) ([7497590](https://github.com/ipfs/helia-unixfs/commit/74975903ec619a4662e5bfa9546997641e9f8e8c)) + +## [@helia/unixfs-v1.2.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.1.0...@helia/unixfs-v1.2.0) (2023-03-17) + + +### Features + +* expose unixfs progress events in types ([#14](https://github.com/ipfs/helia-unixfs/issues/14)) ([36cf3b2](https://github.com/ipfs/helia-unixfs/commit/36cf3b2143276a59b685ceb58299c4f881545fee)) + +## [@helia/unixfs-v1.1.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.0.5...@helia/unixfs-v1.1.0) (2023-03-15) + + +### Features + +* expose progress events from importer, blockstore and bitswap ([#13](https://github.com/ipfs/helia-unixfs/issues/13)) ([de78f4d](https://github.com/ipfs/helia-unixfs/commit/de78f4d03ebafe9ed9a2dfcbfb7a516fa215585c)) + + +### Trivial Changes + +* add interop test suite ([#12](https://github.com/ipfs/helia-unixfs/issues/12)) ([3ad5f5d](https://github.com/ipfs/helia-unixfs/commit/3ad5f5d8199a5596aa333916d4a240584bc0842a)) + +## [1.0.5](https://github.com/ipfs/helia-unixfs/compare/v1.0.4...v1.0.5) (2023-03-14) + + +### Bug Fixes + +* align defaults with filecoin ([#6](https://github.com/ipfs/helia-unixfs/issues/6)) ([a6bd198](https://github.com/ipfs/helia-unixfs/commit/a6bd1983bd7baac21af3de6fa269219f52664cde)) + +## [1.0.4](https://github.com/ipfs/helia-unixfs/compare/v1.0.3...v1.0.4) (2023-02-27) + + +### Bug Fixes + +* simplify shard handling ([#5](https://github.com/ipfs/helia-unixfs/issues/5)) ([52d4786](https://github.com/ipfs/helia-unixfs/commit/52d4786831c3b2b60322de562b752ecfbc8791bb)) + +## [1.0.3](https://github.com/ipfs/helia-unixfs/compare/v1.0.2...v1.0.3) (2023-02-25) + + +### Bug Fixes + +* enable last shard tests ([#4](https://github.com/ipfs/helia-unixfs/issues/4)) ([9774460](https://github.com/ipfs/helia-unixfs/commit/97744606d6da2e61a1aefa6af8f0f3b68f8827ab)) + +## [1.0.2](https://github.com/ipfs/helia-unixfs/compare/v1.0.1...v1.0.2) (2023-02-24) + + +### Bug Fixes + +* add methods to import data ([#3](https://github.com/ipfs/helia-unixfs/issues/3)) ([917a564](https://github.com/ipfs/helia-unixfs/commit/917a564c0d990dfd35d4615436fc8e3609c72a76)) + + +### Tests + +* move test file data to fixtures ([1b76084](https://github.com/ipfs/helia-unixfs/commit/1b760847a18b7b7c1e3fa8c871fd75acb298480b)) + +## [1.0.1](https://github.com/ipfs/helia-unixfs/compare/v1.0.0...v1.0.1) (2023-02-17) + + +### Bug Fixes + +* update unixfs importer ([f6edeca](https://github.com/ipfs/helia-unixfs/commit/f6edeca471da4aaa2171b0b3f2d2ea91d527a00e)) + +## 1.0.0 (2023-02-16) + + +### Features + +* initial implementation ([#1](https://github.com/ipfs/helia-unixfs/issues/1)) ([adb6b0e](https://github.com/ipfs/helia-unixfs/commit/adb6b0e2626a3bdd08cdc4445e3367f104bc5bb8)) + + +### Trivial Changes + +* initial commit ([35e8a8f](https://github.com/ipfs/helia-unixfs/commit/35e8a8fd7c1ca68b21320b95211304bf01b30086)) +* Update .github/workflows/stale.yml [skip ci] ([bcb060d](https://github.com/ipfs/helia-unixfs/commit/bcb060d880175ab885479388049a1ca2e5873629)) + + +### Documentation + +* update readme ([83e5e0e](https://github.com/ipfs/helia-unixfs/commit/83e5e0e0ccfd27f9371c9a8940c237e398e9b68f)) diff --git a/packages/strings/LICENSE b/packages/strings/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/strings/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/strings/LICENSE-APACHE b/packages/strings/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/strings/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/strings/LICENSE-MIT b/packages/strings/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/strings/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/strings/README.md b/packages/strings/README.md new file mode 100644 index 00000000..afe76044 --- /dev/null +++ b/packages/strings/README.md @@ -0,0 +1,67 @@ +# @helia/strings + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-strings.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-strings) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-strings/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-strings/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Add/get IPLD blocks containing strings with your Helia node + +## Table of contents + +- - [Install](#install) + - [Browser ` +``` + +

+ + Helia logo + +

+ +# @helia/unixfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/strings/package.json b/packages/strings/package.json new file mode 100644 index 00000000..e0d35ae3 --- /dev/null +++ b/packages/strings/package.json @@ -0,0 +1,154 @@ +{ + "name": "@helia/strings", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing strings with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/strings#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-strings.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-strings/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "interface-blockstore": "^5.0.0", + "multiformats": "^11.0.1", + "progress-events": "^1.0.0", + "uint8arrays": "^4.0.3" + }, + "devDependencies": { + "aegir": "^38.1.0", + "blockstore-core": "^4.0.1" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/strings/src/index.ts b/packages/strings/src/index.ts new file mode 100644 index 00000000..4ee415e0 --- /dev/null +++ b/packages/strings/src/index.ts @@ -0,0 +1,122 @@ +/** + * @packageDocumentation + * + * `@helia/strings` makes working with strings {@link https://github.com/ipfs/helia Helia} simple & straightforward. + * + * See the {@link Strings Strings interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { strings } from '@helia/strings' + * import { CID } from 'multiformats/cid' + * + * const str = strings(helia) + * const cid = await str.put('hello world') + * const string = await str.get(cid) + * + * console.info(string) + * // hello world + * ``` + */ + +import { CID } from 'multiformats/cid' +import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { ProgressOptions } from 'progress-events' +import { sha256 } from 'multiformats/hashes/sha2' +import * as raw from 'multiformats/codecs/raw' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import type { BlockCodec } from 'multiformats/codecs/interface' +import type { MultihashHasher } from 'multiformats/hashes/interface' + +export interface StringsComponents { + blockstore: Blocks +} + +export interface AddOptions extends AbortOptions, ProgressOptions { + hasher: MultihashHasher + codec: BlockCodec +} + +export interface GetOptions extends AbortOptions, ProgressOptions { + codec: BlockCodec +} + +/** + * The UnixFS interface provides familiar filesystem operations to make working with + * UnixFS DAGs simple and intuitive. + */ +export interface Strings { + /** + * Add a string to your Helia node and get a CID that refers to the block the + * string has been stored as. + * + * @example + * + * ```typescript + * import { strings } from '@helia/strings' + * + * const str = strings(helia) + * const cid = await str.add('hello world') + * + * console.info(cid) + * // CID(QmFoo) + * ``` + */ + add: (str: string, options?: Partial) => Promise + + /** + * Get a string from your Helia node, either previously added to it or to + * another node on the network. + * + * @example + * + * ```typescript + * import { strings } from '@helia/strings' + * import { CID } from 'multiformats/cid' + * + * const str = strings(helia) + * const cid = CID.parse('') + * const string = await str.get(cid) + * + * console.info(string) + * // hello world + * ``` + */ + get: (cid: CID, options?: Partial) => Promise +} + +class DefaultStrings implements Strings { + private readonly components: StringsComponents + + constructor (components: StringsComponents) { + this.components = components + } + + async add (string: string, options: Partial = {}): Promise { + const buf = uint8ArrayFromString(string) + const hash = await (options.hasher ?? sha256).digest(buf) + const codec = options.codec ?? raw + const cid = CID.createV1(codec.code, hash) + + await this.components.blockstore.put(cid, buf, options) + + return cid + } + + async get (cid: CID, options: Partial = {}): Promise { + const buf = await this.components.blockstore.get(cid, options) + + return uint8ArrayToString(buf) + } +} + +/** + * Create a {@link Strings} instance for use with {@link https://github.com/ipfs/helia Helia} + */ +export function strings (helia: { blockstore: Blocks }): Strings { + return new DefaultStrings(helia) +} diff --git a/packages/strings/test/add.spec.ts b/packages/strings/test/add.spec.ts new file mode 100644 index 00000000..9864b26c --- /dev/null +++ b/packages/strings/test/add.spec.ts @@ -0,0 +1,23 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { strings, Strings } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' + +describe('put', () => { + let blockstore: Blockstore + let str: Strings + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + str = strings({ blockstore }) + }) + + it('adds a string', async () => { + const cid = await str.add('hello world') + + expect(`${cid}`).to.equal('bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e') + }) +}) diff --git a/packages/strings/test/get.spec.ts b/packages/strings/test/get.spec.ts new file mode 100644 index 00000000..7fbfca2f --- /dev/null +++ b/packages/strings/test/get.spec.ts @@ -0,0 +1,26 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { strings, Strings } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import type { CID } from 'multiformats/cid' + +describe('get', () => { + let blockstore: Blockstore + let str: Strings + let cid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + str = strings({ blockstore }) + cid = await str.add('hello world') + }) + + it('adds a string', async () => { + const string = await str.get(cid) + + expect(`${string}`).to.equal('hello world') + }) +}) diff --git a/packages/strings/tsconfig.json b/packages/strings/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/strings/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} From 2a700dc30945857e5ec596a8551adf488dc18009 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 13:47:52 +0100 Subject: [PATCH 051/253] docs: update readmes --- README.md | 19 ++++++------------- packages/interop/README.md | 21 +++++++-------------- packages/interop/package.json | 4 ++-- packages/strings/README.md | 19 ++++++------------- 4 files changed, 21 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 85e49462..a7808cac 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +

+ + Helia logo + +

+ # @helia/strings [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) @@ -20,19 +26,6 @@ - [`/packages/interop`](./packages/interop) Interop tests for @helia/unixfs - [`/packages/strings`](./packages/strings) Add/get IPLD blocks containing strings with your Helia node -

- - Helia logo - -

- -# @helia/unixfs - -[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) -[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) - ## API Docs - diff --git a/packages/interop/README.md b/packages/interop/README.md index 7469aa2a..c3c81308 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -1,4 +1,10 @@ -# @helia/unixfs-interop +

+ + Helia logo + +

+ +# @helia/strings-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -30,19 +36,6 @@ Loading this module through a script tag will make it's exports available as `He ``` -

- - Helia logo - -

- -# @helia/unixfs-interop - -[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) -[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) - ## API Docs - diff --git a/packages/interop/package.json b/packages/interop/package.json index 8fbc53f7..67dbc219 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -1,7 +1,7 @@ { - "name": "@helia/unixfs-interop", + "name": "@helia/strings-interop", "version": "0.0.0", - "description": "Interop tests for @helia/unixfs", + "description": "Interop tests for @helia/strings", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/interop#readme", "repository": { diff --git a/packages/strings/README.md b/packages/strings/README.md index afe76044..911b57e5 100644 --- a/packages/strings/README.md +++ b/packages/strings/README.md @@ -1,3 +1,9 @@ +

+ + Helia logo + +

+ # @helia/strings [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) @@ -30,19 +36,6 @@ Loading this module through a script tag will make it's exports available as `He ``` -

- - Helia logo - -

- -# @helia/unixfs - -[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) -[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) - ## API Docs - From 3d4573d9bc22bdd79043b6fec570e8410c8d1228 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 13:48:42 +0100 Subject: [PATCH 052/253] docs: update tocs --- README.md | 9 ++++----- packages/interop/README.md | 11 +++++------ packages/strings/README.md | 11 +++++------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a7808cac..a734c103 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,10 @@ ## Table of contents -- - [Structure](#structure) -- [@helia/unixfs ](#heliaunixfs----omit-in-toc---) - - [API Docs](#api-docs) - - [License](#license) - - [Contribute](#contribute) +- [Structure](#structure) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) ## Structure diff --git a/packages/interop/README.md b/packages/interop/README.md index c3c81308..cf65d0f8 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -15,12 +15,11 @@ ## Table of contents -- - [Install](#install) - - [Browser ` + ``` ## API Docs -- +- ## License diff --git a/packages/strings/CHANGELOG.md b/packages/strings/CHANGELOG.md index e00510b6..e69de29b 100644 --- a/packages/strings/CHANGELOG.md +++ b/packages/strings/CHANGELOG.md @@ -1,83 +0,0 @@ -## [@helia/unixfs-v1.2.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.0...@helia/unixfs-v1.2.1) (2023-03-23) - - -### Dependencies - -* update helia deps to v1 ([#16](https://github.com/ipfs/helia-unixfs/issues/16)) ([7497590](https://github.com/ipfs/helia-unixfs/commit/74975903ec619a4662e5bfa9546997641e9f8e8c)) - -## [@helia/unixfs-v1.2.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.1.0...@helia/unixfs-v1.2.0) (2023-03-17) - - -### Features - -* expose unixfs progress events in types ([#14](https://github.com/ipfs/helia-unixfs/issues/14)) ([36cf3b2](https://github.com/ipfs/helia-unixfs/commit/36cf3b2143276a59b685ceb58299c4f881545fee)) - -## [@helia/unixfs-v1.1.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.0.5...@helia/unixfs-v1.1.0) (2023-03-15) - - -### Features - -* expose progress events from importer, blockstore and bitswap ([#13](https://github.com/ipfs/helia-unixfs/issues/13)) ([de78f4d](https://github.com/ipfs/helia-unixfs/commit/de78f4d03ebafe9ed9a2dfcbfb7a516fa215585c)) - - -### Trivial Changes - -* add interop test suite ([#12](https://github.com/ipfs/helia-unixfs/issues/12)) ([3ad5f5d](https://github.com/ipfs/helia-unixfs/commit/3ad5f5d8199a5596aa333916d4a240584bc0842a)) - -## [1.0.5](https://github.com/ipfs/helia-unixfs/compare/v1.0.4...v1.0.5) (2023-03-14) - - -### Bug Fixes - -* align defaults with filecoin ([#6](https://github.com/ipfs/helia-unixfs/issues/6)) ([a6bd198](https://github.com/ipfs/helia-unixfs/commit/a6bd1983bd7baac21af3de6fa269219f52664cde)) - -## [1.0.4](https://github.com/ipfs/helia-unixfs/compare/v1.0.3...v1.0.4) (2023-02-27) - - -### Bug Fixes - -* simplify shard handling ([#5](https://github.com/ipfs/helia-unixfs/issues/5)) ([52d4786](https://github.com/ipfs/helia-unixfs/commit/52d4786831c3b2b60322de562b752ecfbc8791bb)) - -## [1.0.3](https://github.com/ipfs/helia-unixfs/compare/v1.0.2...v1.0.3) (2023-02-25) - - -### Bug Fixes - -* enable last shard tests ([#4](https://github.com/ipfs/helia-unixfs/issues/4)) ([9774460](https://github.com/ipfs/helia-unixfs/commit/97744606d6da2e61a1aefa6af8f0f3b68f8827ab)) - -## [1.0.2](https://github.com/ipfs/helia-unixfs/compare/v1.0.1...v1.0.2) (2023-02-24) - - -### Bug Fixes - -* add methods to import data ([#3](https://github.com/ipfs/helia-unixfs/issues/3)) ([917a564](https://github.com/ipfs/helia-unixfs/commit/917a564c0d990dfd35d4615436fc8e3609c72a76)) - - -### Tests - -* move test file data to fixtures ([1b76084](https://github.com/ipfs/helia-unixfs/commit/1b760847a18b7b7c1e3fa8c871fd75acb298480b)) - -## [1.0.1](https://github.com/ipfs/helia-unixfs/compare/v1.0.0...v1.0.1) (2023-02-17) - - -### Bug Fixes - -* update unixfs importer ([f6edeca](https://github.com/ipfs/helia-unixfs/commit/f6edeca471da4aaa2171b0b3f2d2ea91d527a00e)) - -## 1.0.0 (2023-02-16) - - -### Features - -* initial implementation ([#1](https://github.com/ipfs/helia-unixfs/issues/1)) ([adb6b0e](https://github.com/ipfs/helia-unixfs/commit/adb6b0e2626a3bdd08cdc4445e3367f104bc5bb8)) - - -### Trivial Changes - -* initial commit ([35e8a8f](https://github.com/ipfs/helia-unixfs/commit/35e8a8fd7c1ca68b21320b95211304bf01b30086)) -* Update .github/workflows/stale.yml [skip ci] ([bcb060d](https://github.com/ipfs/helia-unixfs/commit/bcb060d880175ab885479388049a1ca2e5873629)) - - -### Documentation - -* update readme ([83e5e0e](https://github.com/ipfs/helia-unixfs/commit/83e5e0e0ccfd27f9371c9a8940c237e398e9b68f)) diff --git a/packages/strings/src/index.ts b/packages/strings/src/index.ts index 9006dd6d..df7ae5ef 100644 --- a/packages/strings/src/index.ts +++ b/packages/strings/src/index.ts @@ -63,7 +63,7 @@ export interface Strings { * const cid = await str.add('hello world') * * console.info(cid) - * // CID(QmFoo) + * // CID(bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e) * ``` */ add: (str: string, options?: Partial) => Promise @@ -79,7 +79,7 @@ export interface Strings { * import { CID } from 'multiformats/cid' * * const str = strings(helia) - * const cid = CID.parse('') + * const cid = CID.parse('bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e') * const string = await str.get(cid) * * console.info(string) From 78ad71b02ac136b704aa3d7a56e4d6d1c9c93f8e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 17:51:38 +0100 Subject: [PATCH 055/253] feat: inital import --- .github/dependabot.yml | 11 ++ .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 181 ++++++++++++++++++ .github/workflows/stale.yml | 26 +++ .gitignore | 9 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ README.md | 49 +++++ package.json | 48 +++++ packages/interop/.aegir.js | 45 +++++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 59 ++++++ packages/interop/package.json | 82 ++++++++ packages/interop/src/index.ts | 1 + .../test/fixtures/create-helia.browser.ts | 41 ++++ .../interop/test/fixtures/create-helia.ts | 38 ++++ packages/interop/test/fixtures/create-kubo.ts | 28 +++ packages/interop/test/json.spec.ts | 64 +++++++ packages/interop/tsconfig.json | 15 ++ packages/json/CHANGELOG.md | 0 packages/json/LICENSE | 4 + packages/json/LICENSE-APACHE | 5 + packages/json/LICENSE-MIT | 19 ++ packages/json/README.md | 59 ++++++ packages/json/package.json | 155 +++++++++++++++ packages/json/src/index.ts | 120 ++++++++++++ packages/json/test/add.spec.ts | 36 ++++ packages/json/test/get.spec.ts | 42 ++++ packages/json/tsconfig.json | 10 + 32 files changed, 1211 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 package.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/test/json.spec.ts create mode 100644 packages/interop/tsconfig.json create mode 100644 packages/json/CHANGELOG.md create mode 100644 packages/json/LICENSE create mode 100644 packages/json/LICENSE-APACHE create mode 100644 packages/json/LICENSE-MIT create mode 100644 packages/json/README.md create mode 100644 packages/json/package.json create mode 100644 packages/json/src/index.ts create mode 100644 packages/json/test/add.spec.ts create mode 100644 packages/json/test/get.spec.ts create mode 100644 packages/json/tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..51686414 --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,181 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox-webworker + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit-webworker: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7ad9e674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..dc2a35b2 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +

+ + Helia logo + +

+ +# @helia/json + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-json) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Add/get IPLD blocks containing json with your Helia node + +## Table of contents + +- [Structure](#structure) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) + +## Structure + +- [`/packages/interop`](./packages/interop) Interop tests for @helia/json +- [`/packages/json`](./packages/json) Add/get IPLD blocks containing json with your Helia node + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-json/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/package.json b/package.json new file mode 100644 index 00000000..135b1be4 --- /dev/null +++ b/package.json @@ -0,0 +1,48 @@ +{ + "name": "@helia/json", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing json with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-json#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-json.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-json/issues" + }, + "keywords": [ + "ipfs" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && aegir run release && npm run docs" + }, + "devDependencies": { + "aegir": "^39.0.3" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..29719cd9 --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/json-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-json) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/json + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-json/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..c8ffa30f --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,82 @@ +{ + "name": "@helia/json-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/json", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-json.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-json/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^4.0.1", + "@helia/interface": "^1.0.0", + "@helia/json": "~0.0.0", + "@libp2p/tcp": "^7.0.1", + "@libp2p/websockets": "^6.0.1", + "aegir": "^39.0.3", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", + "go-ipfs": "^0.19.0", + "helia": "^1.0.0", + "ipfs-core-types": "^0.14.0", + "ipfsd-ctl": "^13.0.0", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.44.0", + "merge-options": "^3.0.4", + "multiformats": "^11.0.2", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..6baa5299 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,41 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..c92d309b --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,38 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..52ed76e1 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import mergeOptions from 'merge-options' +import { isElectronMain, isNode } from 'wherearewe' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return createController(opts) +} diff --git a/packages/interop/test/json.spec.ts b/packages/interop/test/json.spec.ts new file mode 100644 index 00000000..6dbd7983 --- /dev/null +++ b/packages/interop/test/json.spec.ts @@ -0,0 +1,64 @@ +/* eslint-env mocha */ + +import { json, type JSON, type AddOptions } from '@helia/json' +import { expect } from 'aegir/chai' +import * as jsonCodec from 'multiformats/codecs/json' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' +import type { Controller } from 'ipfsd-ctl' + +describe('json interop', () => { + let helia: Helia + let j: JSON + let kubo: Controller + + async function expectSameCid (data: () => any, heliaOpts: Partial = {}, kuboOpts: KuboAddOptions = { format: 'json' }): Promise { + const heliaCid = await j.add(data(), heliaOpts) + const kuboCid = await kubo.api.block.put(jsonCodec.encode(data()), kuboOpts) + + expect(heliaCid.toString()).to.equal(kuboCid.toString()) + } + + beforeEach(async () => { + helia = await createHeliaNode() + j = json(helia) + kubo = await createKuboNode() + + await helia.libp2p.dial((await (kubo.api.id())).addresses) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should create the same CID for a string', async () => { + const candidate = (): any => ({ hello: 'world' }) + + await expectSameCid(candidate) + }) + + it('should add to helia and fetch from kubo', async () => { + const input = { hello: 'world' } + const cid = await j.add(input) + const block = await kubo.api.block.get(cid) + const output = jsonCodec.decode(block) + + expect(output).to.deep.equal(input) + }) + + it('should add to kubo and fetch from helia', async () => { + const input = { hello: 'world' } + const cid = await kubo.api.block.put(jsonCodec.encode(input)) + const output = await j.get(cid) + + expect(output).to.deep.equal(input) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..fdbd0de6 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../json" + } + ] +} diff --git a/packages/json/CHANGELOG.md b/packages/json/CHANGELOG.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/json/LICENSE b/packages/json/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/json/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/json/LICENSE-APACHE b/packages/json/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/json/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/json/LICENSE-MIT b/packages/json/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/json/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/json/README.md b/packages/json/README.md new file mode 100644 index 00000000..98fa3fb8 --- /dev/null +++ b/packages/json/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/json + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-json) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Add/get IPLD blocks containing json with your Helia node + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-json/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/json/package.json b/packages/json/package.json new file mode 100644 index 00000000..176ae3f4 --- /dev/null +++ b/packages/json/package.json @@ -0,0 +1,155 @@ +{ + "name": "@helia/json", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing json with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/json#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-json.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-json/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "^1.0.0", + "@libp2p/interfaces": "^3.3.1", + "interface-blockstore": "^5.0.0", + "multiformats": "^11.0.1", + "progress-events": "^1.0.0" + }, + "devDependencies": { + "aegir": "^39.0.3", + "blockstore-core": "^4.0.1" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/json/src/index.ts b/packages/json/src/index.ts new file mode 100644 index 00000000..c808f898 --- /dev/null +++ b/packages/json/src/index.ts @@ -0,0 +1,120 @@ +/** + * @packageDocumentation + * + * `@helia/json` makes working with strings {@link https://github.com/ipfs/helia Helia} simple & straightforward. + * + * See the {@link Strings Strings interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { json } from '@helia/json' + * import { CID } from 'multiformats/cid' + * + * const j = json(helia) + * const cid = await j.put({ + * hello: 'world' + * }) + * const obj = await j.get(cid) + * + * console.info(obj) + * // { hello: 'world' } + * ``` + */ + +import { CID } from 'multiformats/cid' +import * as jsonCodec from 'multiformats/codecs/json' +import { sha256 } from 'multiformats/hashes/sha2' +import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { BlockCodec } from 'multiformats/codecs/interface' +import type { MultihashHasher } from 'multiformats/hashes/interface' +import type { ProgressOptions } from 'progress-events' + +export interface JSONComponents { + blockstore: Blocks +} + +export interface AddOptions extends AbortOptions, ProgressOptions { + hasher: MultihashHasher +} + +export interface GetOptions extends AbortOptions, ProgressOptions { + codec: BlockCodec +} + +/** + * The JSON interface provides a simple and intuitive way to add/get objects + * with your Helia node and is a great place to start learning about IPFS. + */ +export interface JSON { + /** + * Add an object to your Helia node and get a CID that refers to the block the + * object has been stored as. + * + * @example + * + * ```typescript + * import { json } from '@helia/json' + * + * const j = json(helia) + * const cid = await str.add({ hello: 'world' }) + * + * console.info(cid) + * // CID(bagaaierasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea) + * ``` + */ + add: (str: unknown, options?: Partial) => Promise + + /** + * Get an object from your Helia node, either previously added to it or to + * another node on the network. + * + * @example + * + * ```typescript + * import { json } from '@helia/json' + * import { CID } from 'multiformats/cid' + * + * const j = json(helia) + * const cid = CID.parse('bagaaierasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea') + * const obj = await j.get(cid) + * + * console.info(obj) + * // { hello: 'world' } + * ``` + */ + get: (cid: CID, options?: Partial) => Promise +} + +class DefaultJSON implements JSON { + private readonly components: JSONComponents + + constructor (components: JSONComponents) { + this.components = components + } + + async add (obj: any, options: Partial = {}): Promise { + const buf = jsonCodec.encode(obj) + const hash = await (options.hasher ?? sha256).digest(buf) + const cid = CID.createV1(jsonCodec.code, hash) + + await this.components.blockstore.put(cid, buf, options) + + return cid + } + + async get (cid: CID, options: Partial = {}): Promise { + const buf = await this.components.blockstore.get(cid, options) + + return jsonCodec.decode(buf) + } +} + +/** + * Create a {@link JSON} instance for use with {@link https://github.com/ipfs/helia Helia} + */ +export function json (helia: { blockstore: Blocks }): JSON { + return new DefaultJSON(helia) +} diff --git a/packages/json/test/add.spec.ts b/packages/json/test/add.spec.ts new file mode 100644 index 00000000..871acb9e --- /dev/null +++ b/packages/json/test/add.spec.ts @@ -0,0 +1,36 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { identity } from 'multiformats/hashes/identity' +import { json, type JSON } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' + +describe('put', () => { + let blockstore: Blockstore + let j: JSON + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + j = json({ blockstore }) + }) + + it('adds an object', async () => { + const cid = await j.add({ + hello: 'world' + }) + + expect(`${cid}`).to.equal('bagaaierasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea') + }) + + it('adds an object with a non-default hashing algorithm', async () => { + const cid = await j.add({ + hello: 'world' + }, { + hasher: identity + }) + + expect(`${cid}`).to.equal('bagaaiaarpmrgqzlmnrxseorco5xxe3deej6q') + }) +}) diff --git a/packages/json/test/get.spec.ts b/packages/json/test/get.spec.ts new file mode 100644 index 00000000..42d47247 --- /dev/null +++ b/packages/json/test/get.spec.ts @@ -0,0 +1,42 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { identity } from 'multiformats/hashes/identity' +import { json, type JSON } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +describe('get', () => { + let blockstore: Blockstore + let j: JSON + let cid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + j = json({ blockstore }) + cid = await j.add({ + hello: 'world' + }) + }) + + it('gets an object', async () => { + const result = await j.get(cid) + + expect(result).to.deep.equal({ + hello: 'world' + }) + }) + + it('gets an object with a non-default hashing algorithm', async () => { + const input = { + hello: 'world' + } + const cid = await j.add(input, { + hasher: identity + }) + + await expect(j.get(cid)).to.eventually.deep.equal(input) + }) +}) diff --git a/packages/json/tsconfig.json b/packages/json/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/json/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} From 1e0d49a4ecb94b1ef07b8a814a095cea533222a3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 17:57:26 +0100 Subject: [PATCH 056/253] docs: update comment --- packages/json/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/json/src/index.ts b/packages/json/src/index.ts index c808f898..1d39213f 100644 --- a/packages/json/src/index.ts +++ b/packages/json/src/index.ts @@ -1,9 +1,9 @@ /** * @packageDocumentation * - * `@helia/json` makes working with strings {@link https://github.com/ipfs/helia Helia} simple & straightforward. + * `@helia/json` makes working with JSON in {@link https://github.com/ipfs/helia Helia} simple & straightforward. * - * See the {@link Strings Strings interface} for all available operations. + * See the {@link JSON JSON interface} for all available operations. * * @example * From bac0ac5f2778f16a3d8219c73a3e6f0665adf3dd Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 18:20:58 +0100 Subject: [PATCH 057/253] feat: initial import --- .github/dependabot.yml | 11 ++ .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 181 ++++++++++++++++++ .github/workflows/stale.yml | 26 +++ .gitignore | 9 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ README.md | 50 +++++ package.json | 48 +++++ packages/dag-json/CHANGELOG.md | 0 packages/dag-json/LICENSE | 4 + packages/dag-json/LICENSE-APACHE | 5 + packages/dag-json/LICENSE-MIT | 19 ++ packages/dag-json/README.md | 60 ++++++ packages/dag-json/package.json | 156 +++++++++++++++ packages/dag-json/src/index.ts | 120 ++++++++++++ packages/dag-json/test/add.spec.ts | 36 ++++ packages/dag-json/test/get.spec.ts | 42 ++++ packages/dag-json/tsconfig.json | 10 + packages/interop/.aegir.js | 45 +++++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 60 ++++++ packages/interop/package.json | 82 ++++++++ packages/interop/src/index.ts | 1 + packages/interop/test/dag-json.spec.ts | 64 +++++++ .../test/fixtures/create-helia.browser.ts | 41 ++++ .../interop/test/fixtures/create-helia.ts | 38 ++++ packages/interop/test/fixtures/create-kubo.ts | 28 +++ packages/interop/tsconfig.json | 15 ++ 32 files changed, 1215 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 package.json create mode 100644 packages/dag-json/CHANGELOG.md create mode 100644 packages/dag-json/LICENSE create mode 100644 packages/dag-json/LICENSE-APACHE create mode 100644 packages/dag-json/LICENSE-MIT create mode 100644 packages/dag-json/README.md create mode 100644 packages/dag-json/package.json create mode 100644 packages/dag-json/src/index.ts create mode 100644 packages/dag-json/test/add.spec.ts create mode 100644 packages/dag-json/test/get.spec.ts create mode 100644 packages/dag-json/tsconfig.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/dag-json.spec.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..51686414 --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,181 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox-webworker + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit-webworker: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7ad9e674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..5c92b70a --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +

+ + Helia logo + +

+ +# @helia/dag-json + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-json) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Add/get IPLD blocks containing json with your Helia node + +## Table of contents + +- - [Structure](#structure) +- [@helia/dag-json ](#heliadag-json----omit-in-toc---) + - [API Docs](#api-docs) + - [License](#license) + - [Contribute](#contribute) + +## Structure + +- [`/packages/dag-json`](./packages/dag-json) Add/get IPLD blocks containing json with your Helia node +- [`/packages/interop`](./packages/interop) Interop tests for @helia/dag-json + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-json/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/package.json b/package.json new file mode 100644 index 00000000..38dfa048 --- /dev/null +++ b/package.json @@ -0,0 +1,48 @@ +{ + "name": "@helia/dag-json", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing json with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-dag-json#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-dag-json.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-dag-json/issues" + }, + "keywords": [ + "ipfs" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && aegir run release && npm run docs" + }, + "devDependencies": { + "aegir": "^39.0.3" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/dag-json/CHANGELOG.md b/packages/dag-json/CHANGELOG.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/dag-json/LICENSE b/packages/dag-json/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/dag-json/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/dag-json/LICENSE-APACHE b/packages/dag-json/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/dag-json/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/dag-json/LICENSE-MIT b/packages/dag-json/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/dag-json/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/dag-json/README.md b/packages/dag-json/README.md new file mode 100644 index 00000000..c42117e0 --- /dev/null +++ b/packages/dag-json/README.md @@ -0,0 +1,60 @@ +

+ + Helia logo + +

+ +# @helia/dag-json + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-json) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Add/get IPLD blocks containing json with your Helia node + +## Table of contents + +- - [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-json/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json new file mode 100644 index 00000000..651943fa --- /dev/null +++ b/packages/dag-json/package.json @@ -0,0 +1,156 @@ +{ + "name": "@helia/dag-json", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing json with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-dag-json.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-dag-json/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "^1.0.0", + "@ipld/dag-json": "^10.0.1", + "@libp2p/interfaces": "^3.3.1", + "interface-blockstore": "^5.0.0", + "multiformats": "^11.0.1", + "progress-events": "^1.0.0" + }, + "devDependencies": { + "aegir": "^39.0.3", + "blockstore-core": "^4.0.1" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/dag-json/src/index.ts b/packages/dag-json/src/index.ts new file mode 100644 index 00000000..9c2c4075 --- /dev/null +++ b/packages/dag-json/src/index.ts @@ -0,0 +1,120 @@ +/** + * @packageDocumentation + * + * `@helia/dag-json` makes working with DAG-JSON {@link https://github.com/ipfs/helia Helia} simple & straightforward. + * + * See the {@link Strings Strings interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { json } from '@helia/dag-json' + * import { CID } from 'multiformats/cid' + * + * const j = json(helia) + * const cid = await j.put({ + * hello: 'world' + * }) + * const obj = await j.get(cid) + * + * console.info(obj) + * // { hello: 'world' } + * ``` + */ + +import { CID } from 'multiformats/cid' +import * as codec from '@ipld/dag-json' +import { sha256 } from 'multiformats/hashes/sha2' +import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { BlockCodec } from 'multiformats/codecs/interface' +import type { MultihashHasher } from 'multiformats/hashes/interface' +import type { ProgressOptions } from 'progress-events' + +export interface DAGJSONComponents { + blockstore: Blocks +} + +export interface AddOptions extends AbortOptions, ProgressOptions { + hasher: MultihashHasher +} + +export interface GetOptions extends AbortOptions, ProgressOptions { + codec: BlockCodec +} + +/** + * The JSON interface provides a simple and intuitive way to add/get objects + * with your Helia node and is a great place to start learning about IPFS. + */ +export interface DAGJSON { + /** + * Add an object to your Helia node and get a CID that refers to the block the + * object has been stored as. + * + * @example + * + * ```typescript + * import { json } from '@helia/dag-json' + * + * const j = json(helia) + * const cid = await str.add({ hello: 'world' }) + * + * console.info(cid) + * // CID(baguqeerasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea) + * ``` + */ + add: (str: unknown, options?: Partial) => Promise + + /** + * Get an object from your Helia node, either previously added to it or to + * another node on the network. + * + * @example + * + * ```typescript + * import { json } from '@helia/dag-json' + * import { CID } from 'multiformats/cid' + * + * const j = json(helia) + * const cid = CID.parse('baguqeerasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea') + * const obj = await j.get(cid) + * + * console.info(obj) + * // { hello: 'world' } + * ``` + */ + get: (cid: CID, options?: Partial) => Promise +} + +class DefaultDAGJSON implements DAGJSON { + private readonly components: DAGJSONComponents + + constructor (components: DAGJSONComponents) { + this.components = components + } + + async add (obj: any, options: Partial = {}): Promise { + const buf = codec.encode(obj) + const hash = await (options.hasher ?? sha256).digest(buf) + const cid = CID.createV1(codec.code, hash) + + await this.components.blockstore.put(cid, buf, options) + + return cid + } + + async get (cid: CID, options: Partial = {}): Promise { + const buf = await this.components.blockstore.get(cid, options) + + return codec.decode(buf) + } +} + +/** + * Create a {@link DAGJSON} instance for use with {@link https://github.com/ipfs/helia Helia} + */ +export function dagJson (helia: { blockstore: Blocks }): DAGJSON { + return new DefaultDAGJSON(helia) +} diff --git a/packages/dag-json/test/add.spec.ts b/packages/dag-json/test/add.spec.ts new file mode 100644 index 00000000..7a146b9e --- /dev/null +++ b/packages/dag-json/test/add.spec.ts @@ -0,0 +1,36 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { identity } from 'multiformats/hashes/identity' +import { dagJson, type DAGJSON } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' + +describe('put', () => { + let blockstore: Blockstore + let d: DAGJSON + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + d = dagJson({ blockstore }) + }) + + it('adds an object', async () => { + const cid = await d.add({ + hello: 'world' + }) + + expect(`${cid}`).to.equal('baguqeerasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea') + }) + + it('adds an object with a non-default hashing algorithm', async () => { + const cid = await d.add({ + hello: 'world' + }, { + hasher: identity + }) + + expect(`${cid}`).to.equal('baguqeaarpmrgqzlmnrxseorco5xxe3deej6q') + }) +}) diff --git a/packages/dag-json/test/get.spec.ts b/packages/dag-json/test/get.spec.ts new file mode 100644 index 00000000..5ca202c3 --- /dev/null +++ b/packages/dag-json/test/get.spec.ts @@ -0,0 +1,42 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { identity } from 'multiformats/hashes/identity' +import { dagJson, type DAGJSON } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +describe('get', () => { + let blockstore: Blockstore + let d: DAGJSON + let cid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + d = dagJson({ blockstore }) + cid = await d.add({ + hello: 'world' + }) + }) + + it('gets an object', async () => { + const result = await d.get(cid) + + expect(result).to.deep.equal({ + hello: 'world' + }) + }) + + it('gets an object with a non-default hashing algorithm', async () => { + const input = { + hello: 'world' + } + const cid = await d.add(input, { + hasher: identity + }) + + await expect(d.get(cid)).to.eventually.deep.equal(input) + }) +}) diff --git a/packages/dag-json/tsconfig.json b/packages/dag-json/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/dag-json/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..b03f855c --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,60 @@ +

+ + Helia logo + +

+ +# @helia/dag-json-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-json) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/dag-json + +## Table of contents + +- - [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-json/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..e7d9d225 --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,82 @@ +{ + "name": "@helia/dag-json-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/dag-json", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-dag-json.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-dag-json/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^4.0.1", + "@helia/dag-json": "~0.0.0", + "@helia/interface": "^1.0.0", + "@libp2p/tcp": "^7.0.1", + "@libp2p/websockets": "^6.0.1", + "aegir": "^39.0.3", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", + "go-ipfs": "^0.19.0", + "helia": "^1.0.0", + "ipfs-core-types": "^0.14.0", + "ipfsd-ctl": "^13.0.0", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.44.0", + "merge-options": "^3.0.4", + "multiformats": "^11.0.2", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/dag-json.spec.ts b/packages/interop/test/dag-json.spec.ts new file mode 100644 index 00000000..820ef6fb --- /dev/null +++ b/packages/interop/test/dag-json.spec.ts @@ -0,0 +1,64 @@ +/* eslint-env mocha */ + +import { dagJson, type DAGJSON, type AddOptions } from '@helia/dag-json' +import { expect } from 'aegir/chai' +import * as jsonCodec from 'multiformats/codecs/json' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' +import type { Controller } from 'ipfsd-ctl' + +describe('dag-json interop', () => { + let helia: Helia + let d: DAGJSON + let kubo: Controller + + async function expectSameCid (data: () => any, heliaOpts: Partial = {}, kuboOpts: KuboAddOptions = { format: 'dag-json' }): Promise { + const heliaCid = await d.add(data(), heliaOpts) + const kuboCid = await kubo.api.block.put(jsonCodec.encode(data()), kuboOpts) + + expect(heliaCid.toString()).to.equal(kuboCid.toString()) + } + + beforeEach(async () => { + helia = await createHeliaNode() + d = dagJson(helia) + kubo = await createKuboNode() + + await helia.libp2p.dial((await (kubo.api.id())).addresses) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should create the same CID for a string', async () => { + const candidate = (): any => ({ hello: 'world' }) + + await expectSameCid(candidate) + }) + + it('should add to helia and fetch from kubo', async () => { + const input = { hello: 'world' } + const cid = await d.add(input) + const block = await kubo.api.block.get(cid) + const output = jsonCodec.decode(block) + + expect(output).to.deep.equal(input) + }) + + it('should add to kubo and fetch from helia', async () => { + const input = { hello: 'world' } + const cid = await kubo.api.block.put(jsonCodec.encode(input)) + const output = await d.get(cid) + + expect(output).to.deep.equal(input) + }) +}) diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..6baa5299 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,41 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..c92d309b --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,38 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..52ed76e1 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import mergeOptions from 'merge-options' +import { isElectronMain, isNode } from 'wherearewe' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return createController(opts) +} diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..12b6fd1d --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../dag-json" + } + ] +} From 9d9d341583d34c4516e5cfaa8bccfd5b6ac860a1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 18:24:56 +0100 Subject: [PATCH 058/253] chore: fix linting --- packages/dag-json/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dag-json/src/index.ts b/packages/dag-json/src/index.ts index 9c2c4075..860a3094 100644 --- a/packages/dag-json/src/index.ts +++ b/packages/dag-json/src/index.ts @@ -23,8 +23,8 @@ * ``` */ -import { CID } from 'multiformats/cid' import * as codec from '@ipld/dag-json' +import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' import type { AbortOptions } from '@libp2p/interfaces' From f1944b04271a599eee987d56d4d8506eaeb8a69d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 18:26:08 +0100 Subject: [PATCH 059/253] docs: replace references to json with dag-json --- README.md | 4 ++-- package.json | 2 +- packages/dag-json/README.md | 2 +- packages/dag-json/package.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5c92b70a..4bf9c97a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-json) [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) -> Add/get IPLD blocks containing json with your Helia node +> Add/get IPLD blocks containing dag-json with your Helia node ## Table of contents @@ -23,7 +23,7 @@ ## Structure -- [`/packages/dag-json`](./packages/dag-json) Add/get IPLD blocks containing json with your Helia node +- [`/packages/dag-json`](./packages/dag-json) Add/get IPLD blocks containing dag-json with your Helia node - [`/packages/interop`](./packages/interop) Interop tests for @helia/dag-json ## API Docs diff --git a/package.json b/package.json index 38dfa048..180614e2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@helia/dag-json", "version": "0.0.0", - "description": "Add/get IPLD blocks containing json with your Helia node", + "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json#readme", "repository": { diff --git a/packages/dag-json/README.md b/packages/dag-json/README.md index c42117e0..4e8cafa4 100644 --- a/packages/dag-json/README.md +++ b/packages/dag-json/README.md @@ -11,7 +11,7 @@ [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-json.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-json) [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-json/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-json/actions/workflows/js-test-and-release.yml?query=branch%3Amain) -> Add/get IPLD blocks containing json with your Helia node +> Add/get IPLD blocks containing dag-json with your Helia node ## Table of contents diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 651943fa..148b25d0 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -1,7 +1,7 @@ { "name": "@helia/dag-json", "version": "0.0.0", - "description": "Add/get IPLD blocks containing json with your Helia node", + "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", "repository": { From 0b4bac4583f790686ceaf89f2f2ab6642677c4fd Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 18:26:46 +0100 Subject: [PATCH 060/253] docs: update tocs --- README.md | 9 ++++----- packages/dag-json/README.md | 11 +++++------ packages/interop/README.md | 11 +++++------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 4bf9c97a..3d1bb81e 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,10 @@ ## Table of contents -- - [Structure](#structure) -- [@helia/dag-json ](#heliadag-json----omit-in-toc---) - - [API Docs](#api-docs) - - [License](#license) - - [Contribute](#contribute) +- [Structure](#structure) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) ## Structure diff --git a/packages/dag-json/README.md b/packages/dag-json/README.md index 4e8cafa4..eb50d3ee 100644 --- a/packages/dag-json/README.md +++ b/packages/dag-json/README.md @@ -15,12 +15,11 @@ ## Table of contents -- - [Install](#install) - - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-cbor/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json new file mode 100644 index 00000000..6d417ebd --- /dev/null +++ b/packages/dag-cbor/package.json @@ -0,0 +1,156 @@ +{ + "name": "@helia/dag-cbor", + "version": "0.0.0", + "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/dag-cbor#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-dag-cbor.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-dag-cbor/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "^1.0.0", + "@ipld/dag-cbor": "^9.0.0", + "@libp2p/interfaces": "^3.3.1", + "interface-blockstore": "^5.0.0", + "multiformats": "^11.0.1", + "progress-events": "^1.0.0" + }, + "devDependencies": { + "aegir": "^39.0.3", + "blockstore-core": "^4.0.1" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/dag-cbor/src/index.ts b/packages/dag-cbor/src/index.ts new file mode 100644 index 00000000..18f148f9 --- /dev/null +++ b/packages/dag-cbor/src/index.ts @@ -0,0 +1,120 @@ +/** + * @packageDocumentation + * + * `@helia/dag-cbor` makes working with DAG-JSON {@link https://github.com/ipfs/helia Helia} simple & straightforward. + * + * See the {@link Strings Strings interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { dagCbor } from '@helia/dag-cbor' + * import { CID } from 'multiformats/cid' + * + * const d = dagCbor(helia) + * const cid = await d.put({ + * hello: 'world' + * }) + * const obj = await d.get(cid) + * + * console.info(obj) + * // { hello: 'world' } + * ``` + */ + +import * as codec from '@ipld/dag-cbor' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { BlockCodec } from 'multiformats/codecs/interface' +import type { MultihashHasher } from 'multiformats/hashes/interface' +import type { ProgressOptions } from 'progress-events' + +export interface DAGCBORComponents { + blockstore: Blocks +} + +export interface AddOptions extends AbortOptions, ProgressOptions { + hasher: MultihashHasher +} + +export interface GetOptions extends AbortOptions, ProgressOptions { + codec: BlockCodec +} + +/** + * The JSON interface provides a simple and intuitive way to add/get objects + * with your Helia node and is a great place to start learning about IPFS. + */ +export interface DAGCBOR { + /** + * Add an object to your Helia node and get a CID that refers to the block the + * object has been stored as. + * + * @example + * + * ```typescript + * import { json } from '@helia/dag-cbor' + * + * const j = json(helia) + * const cid = await str.add({ hello: 'world' }) + * + * console.info(cid) + * // CID(bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae) + * ``` + */ + add: (str: unknown, options?: Partial) => Promise + + /** + * Get an object from your Helia node, either previously added to it or to + * another node on the network. + * + * @example + * + * ```typescript + * import { json } from '@helia/dag-cbor' + * import { CID } from 'multiformats/cid' + * + * const j = json(helia) + * const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + * const obj = await j.get(cid) + * + * console.info(obj) + * // { hello: 'world' } + * ``` + */ + get: (cid: CID, options?: Partial) => Promise +} + +class DefaultDAGCBOR implements DAGCBOR { + private readonly components: DAGCBORComponents + + constructor (components: DAGCBORComponents) { + this.components = components + } + + async add (obj: any, options: Partial = {}): Promise { + const buf = codec.encode(obj) + const hash = await (options.hasher ?? sha256).digest(buf) + const cid = CID.createV1(codec.code, hash) + + await this.components.blockstore.put(cid, buf, options) + + return cid + } + + async get (cid: CID, options: Partial = {}): Promise { + const buf = await this.components.blockstore.get(cid, options) + + return codec.decode(buf) + } +} + +/** + * Create a {@link DAGCBOR} instance for use with {@link https://github.com/ipfs/helia Helia} + */ +export function dagCbor (helia: { blockstore: Blocks }): DAGCBOR { + return new DefaultDAGCBOR(helia) +} diff --git a/packages/dag-cbor/test/add.spec.ts b/packages/dag-cbor/test/add.spec.ts new file mode 100644 index 00000000..eb1e52c7 --- /dev/null +++ b/packages/dag-cbor/test/add.spec.ts @@ -0,0 +1,36 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { identity } from 'multiformats/hashes/identity' +import { dagCbor, type DAGCBOR } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' + +describe('put', () => { + let blockstore: Blockstore + let d: DAGCBOR + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + d = dagCbor({ blockstore }) + }) + + it('adds an object', async () => { + const cid = await d.add({ + hello: 'world' + }) + + expect(`${cid}`).to.equal('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + }) + + it('adds an object with a non-default hashing algorithm', async () => { + const cid = await d.add({ + hello: 'world' + }, { + hasher: identity + }) + + expect(`${cid}`).to.equal('bafyqadnbmvugk3dmn5sxo33snrsa') + }) +}) diff --git a/packages/dag-cbor/test/get.spec.ts b/packages/dag-cbor/test/get.spec.ts new file mode 100644 index 00000000..e893125d --- /dev/null +++ b/packages/dag-cbor/test/get.spec.ts @@ -0,0 +1,42 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { identity } from 'multiformats/hashes/identity' +import { dagCbor, type DAGCBOR } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +describe('get', () => { + let blockstore: Blockstore + let d: DAGCBOR + let cid: CID + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + d = dagCbor({ blockstore }) + cid = await d.add({ + hello: 'world' + }) + }) + + it('gets an object', async () => { + const result = await d.get(cid) + + expect(result).to.deep.equal({ + hello: 'world' + }) + }) + + it('gets an object with a non-default hashing algorithm', async () => { + const input = { + hello: 'world' + } + const cid = await d.add(input, { + hasher: identity + }) + + await expect(d.get(cid)).to.eventually.deep.equal(input) + }) +}) diff --git a/packages/dag-cbor/tsconfig.json b/packages/dag-cbor/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/dag-cbor/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..b7fddf73 --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/dag-cbor-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-cbor.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-cbor) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-cbor/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-cbor/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/dag-cbor + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-cbor/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..f9f31332 --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,82 @@ +{ + "name": "@helia/dag-cbor-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/dag-cbor", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-dag-cbor.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-dag-cbor/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^4.0.1", + "@helia/dag-cbor": "~0.0.0", + "@helia/interface": "^1.0.0", + "@ipld/dag-cbor": "^9.0.0", + "@libp2p/tcp": "^7.0.1", + "@libp2p/websockets": "^6.0.1", + "aegir": "^39.0.3", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", + "go-ipfs": "^0.19.0", + "helia": "^1.0.0", + "ipfs-core-types": "^0.14.0", + "ipfsd-ctl": "^13.0.0", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.44.0", + "merge-options": "^3.0.4", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/dag-json.spec.ts b/packages/interop/test/dag-json.spec.ts new file mode 100644 index 00000000..dd4ef3e1 --- /dev/null +++ b/packages/interop/test/dag-json.spec.ts @@ -0,0 +1,64 @@ +/* eslint-env mocha */ + +import { dagCbor, type DAGCBOR, type AddOptions } from '@helia/dag-cbor' +import * as codec from '@ipld/dag-cbor' +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' +import type { Controller } from 'ipfsd-ctl' + +describe('dag-json interop', () => { + let helia: Helia + let d: DAGCBOR + let kubo: Controller + + async function expectSameCid (data: () => any, heliaOpts: Partial = {}, kuboOpts: KuboAddOptions = {}): Promise { + const heliaCid = await d.add(data(), heliaOpts) + const kuboCid = await kubo.api.dag.put(data(), kuboOpts) + + expect(heliaCid.toString()).to.equal(kuboCid.toString()) + } + + beforeEach(async () => { + helia = await createHeliaNode() + d = dagCbor(helia) + kubo = await createKuboNode() + + await helia.libp2p.dial((await (kubo.api.id())).addresses) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should create the same CID for a string', async () => { + const candidate = (): any => ({ hello: 'world' }) + + await expectSameCid(candidate) + }) + + it('should add to helia and fetch from kubo', async () => { + const input = { hello: 'world' } + const cid = await d.add(input) + const block = await kubo.api.block.get(cid) + const output = codec.decode(block) + + expect(output).to.deep.equal(input) + }) + + it('should add to kubo and fetch from helia', async () => { + const input = { hello: 'world' } + const cid = await kubo.api.block.put(codec.encode(input)) + const output = await d.get(cid) + + expect(output).to.deep.equal(input) + }) +}) diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..6baa5299 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,41 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..c92d309b --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,38 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..52ed76e1 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import mergeOptions from 'merge-options' +import { isElectronMain, isNode } from 'wherearewe' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return createController(opts) +} diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..a1ee8237 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../dag-cbor" + } + ] +} From 65cc7bb1a3817350bf93cc793c069538bc946480 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 May 2023 18:46:17 +0100 Subject: [PATCH 066/253] docs: update json reference --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 06a10056..7f59adb7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@helia/dag-cbor", "version": "0.0.0", - "description": "Add/get IPLD blocks containing json with your Helia node", + "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-cbor#readme", "repository": { From 871c5704749aa0a852ab345c14bacce104589b84 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 3 May 2023 11:44:35 +0000 Subject: [PATCH 067/253] chore(release): 1.0.0 [skip ci] ## @helia/strings-v1.0.0 (2023-05-03) ### Bug Fixes * linting and deps ([22d3900](https://github.com/ipfs/helia-strings/commit/22d3900c15b0876419460c4db57b41f91e78d52f)) ### Documentation * update readme ([#6](https://github.com/ipfs/helia-strings/issues/6)) ([c62f784](https://github.com/ipfs/helia-strings/commit/c62f78499d75ba96da60a4de2f6a0ae3f007abfb)) * update readmes ([2a700dc](https://github.com/ipfs/helia-strings/commit/2a700dc30945857e5ec596a8551adf488dc18009)) * update tocs ([3d4573d](https://github.com/ipfs/helia-strings/commit/3d4573d9bc22bdd79043b6fec570e8410c8d1228)) --- packages/strings/CHANGELOG.md | 13 +++++++++++++ packages/strings/package.json | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/strings/CHANGELOG.md b/packages/strings/CHANGELOG.md index e69de29b..ab5d5632 100644 --- a/packages/strings/CHANGELOG.md +++ b/packages/strings/CHANGELOG.md @@ -0,0 +1,13 @@ +## @helia/strings-v1.0.0 (2023-05-03) + + +### Bug Fixes + +* linting and deps ([22d3900](https://github.com/ipfs/helia-strings/commit/22d3900c15b0876419460c4db57b41f91e78d52f)) + + +### Documentation + +* update readme ([#6](https://github.com/ipfs/helia-strings/issues/6)) ([c62f784](https://github.com/ipfs/helia-strings/commit/c62f78499d75ba96da60a4de2f6a0ae3f007abfb)) +* update readmes ([2a700dc](https://github.com/ipfs/helia-strings/commit/2a700dc30945857e5ec596a8551adf488dc18009)) +* update tocs ([3d4573d](https://github.com/ipfs/helia-strings/commit/3d4573d9bc22bdd79043b6fec570e8410c8d1228)) diff --git a/packages/strings/package.json b/packages/strings/package.json index 8b973115..64494b61 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -1,6 +1,6 @@ { "name": "@helia/strings", - "version": "0.0.0", + "version": "1.0.0", "description": "Add/get IPLD blocks containing strings with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/strings#readme", From 9f1cff2c869697f510c169f264e3b3294d4d3692 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Wed, 3 May 2023 11:44:44 +0000 Subject: [PATCH 068/253] deps: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 1f2bd506..87a9446e 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -55,7 +55,7 @@ "@chainsafe/libp2p-noise": "^11.0.0", "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/interface": "^1.0.0", - "@helia/strings": "~0.0.0", + "@helia/strings": "^1.0.0", "@libp2p/tcp": "^6.1.2", "@libp2p/websockets": "^6.0.1", "aegir": "^39.0.3", From 9402e6c83aa796a3431bd4cff857023e32971864 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 3 May 2023 11:51:00 +0000 Subject: [PATCH 069/253] chore(release): 1.0.0 [skip ci] ## @helia/json-v1.0.0 (2023-05-03) ### Documentation * update comment ([1e0d49a](https://github.com/ipfs/helia-json/commit/1e0d49a4ecb94b1ef07b8a814a095cea533222a3)) --- packages/json/CHANGELOG.md | 6 ++++++ packages/json/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/json/CHANGELOG.md b/packages/json/CHANGELOG.md index e69de29b..b4debf14 100644 --- a/packages/json/CHANGELOG.md +++ b/packages/json/CHANGELOG.md @@ -0,0 +1,6 @@ +## @helia/json-v1.0.0 (2023-05-03) + + +### Documentation + +* update comment ([1e0d49a](https://github.com/ipfs/helia-json/commit/1e0d49a4ecb94b1ef07b8a814a095cea533222a3)) diff --git a/packages/json/package.json b/packages/json/package.json index 176ae3f4..c6919255 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/json", - "version": "0.0.0", + "version": "1.0.0", "description": "Add/get IPLD blocks containing json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/json#readme", From 68c3a6c4764f147530023ba3b7d37857849cebf0 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Wed, 3 May 2023 11:51:11 +0000 Subject: [PATCH 070/253] deps: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index c8ffa30f..8f48336f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -55,7 +55,7 @@ "@chainsafe/libp2p-noise": "^11.0.0", "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/interface": "^1.0.0", - "@helia/json": "~0.0.0", + "@helia/json": "^1.0.0", "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", "aegir": "^39.0.3", From 2cfa7c669e47a0c9dc5a1760c3d3b8412d1cf73f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 3 May 2023 11:51:59 +0000 Subject: [PATCH 071/253] chore(release): 1.0.0 [skip ci] ## @helia/dag-json-v1.0.0 (2023-05-03) ### Trivial Changes * fix linting ([9d9d341](https://github.com/ipfs/helia-dag-json/commit/9d9d341583d34c4516e5cfaa8bccfd5b6ac860a1)) ### Documentation * replace references to json with dag-json ([f1944b0](https://github.com/ipfs/helia-dag-json/commit/f1944b04271a599eee987d56d4d8506eaeb8a69d)) * update tocs ([0b4bac4](https://github.com/ipfs/helia-dag-json/commit/0b4bac4583f790686ceaf89f2f2ab6642677c4fd)) --- packages/dag-json/CHANGELOG.md | 12 ++++++++++++ packages/dag-json/package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/dag-json/CHANGELOG.md b/packages/dag-json/CHANGELOG.md index e69de29b..1a13d6eb 100644 --- a/packages/dag-json/CHANGELOG.md +++ b/packages/dag-json/CHANGELOG.md @@ -0,0 +1,12 @@ +## @helia/dag-json-v1.0.0 (2023-05-03) + + +### Trivial Changes + +* fix linting ([9d9d341](https://github.com/ipfs/helia-dag-json/commit/9d9d341583d34c4516e5cfaa8bccfd5b6ac860a1)) + + +### Documentation + +* replace references to json with dag-json ([f1944b0](https://github.com/ipfs/helia-dag-json/commit/f1944b04271a599eee987d56d4d8506eaeb8a69d)) +* update tocs ([0b4bac4](https://github.com/ipfs/helia-dag-json/commit/0b4bac4583f790686ceaf89f2f2ab6642677c4fd)) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 148b25d0..0a9cab52 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-json", - "version": "0.0.0", + "version": "1.0.0", "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", From 91263ad75211f6c982fae7e3a88c6b244aa8aeb2 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Wed, 3 May 2023 11:52:10 +0000 Subject: [PATCH 072/253] deps: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index e7d9d225..ca450fbe 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -54,7 +54,7 @@ "devDependencies": { "@chainsafe/libp2p-noise": "^11.0.0", "@chainsafe/libp2p-yamux": "^4.0.1", - "@helia/dag-json": "~0.0.0", + "@helia/dag-json": "^1.0.0", "@helia/interface": "^1.0.0", "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", From 55208ccfdc4f3a799736f29e614910cbd8375a9d Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 5 May 2023 17:50:47 +0100 Subject: [PATCH 073/253] fix: use the content routing api for get/put operations (#34) The content routing API is more stable than the DHT API and if the node has a DHT implementation it will be set up to work via the content routing methods. --- packages/ipns/package.json | 1 + packages/ipns/src/routing/dht.ts | 54 ++++---------------------------- 2 files changed, 7 insertions(+), 48 deletions(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 4cb8035e..6e4c2136 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -159,6 +159,7 @@ "release": "aegir release" }, "dependencies": { + "@libp2p/interface-content-routing": "^2.1.0", "@libp2p/interface-dht": "^2.0.1", "@libp2p/interface-peer-id": "^2.0.1", "@libp2p/interface-pubsub": "^3.0.6", diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/dht.ts index a160cecc..93215f9d 100644 --- a/packages/ipns/src/routing/dht.ts +++ b/packages/ipns/src/routing/dht.ts @@ -1,61 +1,35 @@ -import { logger } from '@libp2p/logger' import type { IPNSRouting } from '../index.js' -import type { DHT, QueryEvent } from '@libp2p/interface-dht' import type { GetOptions, PutOptions } from './index.js' import { CustomProgressEvent, ProgressEvent } from 'progress-events' - -const log = logger('helia:ipns:routing:dht') +import type { ContentRouting } from '@libp2p/interface-content-routing' export interface DHTRoutingComponents { libp2p: { - dht: DHT + contentRouting: ContentRouting } } export type DHTProgressEvents = - ProgressEvent<'ipns:routing:dht:query', QueryEvent> | ProgressEvent<'ipns:routing:dht:error', Error> export class DHTRouting implements IPNSRouting { - private readonly dht: DHT + private readonly contentRouting: ContentRouting constructor (components: DHTRoutingComponents) { - this.dht = components.libp2p.dht + this.contentRouting = components.libp2p.contentRouting } async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise { - let putValue = false - try { - for await (const event of this.dht.put(routingKey, marshaledRecord, options)) { - logEvent('DHT put event', event) - - options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:query', event)) - - if (event.name === 'PEER_RESPONSE' && event.messageName === 'PUT_VALUE') { - putValue = true - } - } + await this.contentRouting.put(routingKey, marshaledRecord, options) } catch (err: any) { options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) } - - if (!putValue) { - throw new Error('Could not put value to DHT') - } } async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { try { - for await (const event of this.dht.get(routingKey, options)) { - logEvent('DHT get event', event) - - options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:query', event)) - - if (event.name === 'VALUE') { - return event.value - } - } + return await this.contentRouting.get(routingKey, options) } catch (err: any) { options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) } @@ -64,22 +38,6 @@ export class DHTRouting implements IPNSRouting { } } -function logEvent (prefix: string, event: QueryEvent): void { - if (event.name === 'SENDING_QUERY') { - log(prefix, event.name, event.messageName, '->', event.to.toString()) - } else if (event.name === 'PEER_RESPONSE') { - log(prefix, event.name, event.messageName, '<-', event.from.toString()) - } else if (event.name === 'FINAL_PEER') { - log(prefix, event.name, event.peer.id.toString()) - } else if (event.name === 'QUERY_ERROR') { - log(prefix, event.name, event.error.message) - } else if (event.name === 'PROVIDER') { - log(prefix, event.name, event.providers.map(p => p.id.toString()).join(', ')) - } else { - log(prefix, event.name) - } -} - export function dht (components: DHTRoutingComponents): IPNSRouting { return new DHTRouting(components) } From 4e24094e488dadc4cdd8fdbbbeb4d9d2e4911ae5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 May 2023 17:51:02 +0100 Subject: [PATCH 074/253] deps(dev): bump it-last from 2.0.1 to 3.0.1 (#17) Bumps [it-last](https://github.com/achingbrain/it) from 2.0.1 to 3.0.1. - [Release notes](https://github.com/achingbrain/it/releases) - [Commits](https://github.com/achingbrain/it/compare/it-last-v2.0.1...it-last-v3.0.1) --- updated-dependencies: - dependency-name: it-last dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 9b17f715..6915127e 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -71,7 +71,7 @@ "ipfsd-ctl": "^13.0.0", "ipns": "^6.0.0", "it-all": "^2.0.0", - "it-last": "^2.0.0", + "it-last": "^3.0.1", "it-map": "^2.0.0", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.43.2", From a4edce7154cd3a25ec1e827a5ba61bb874a87166 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 5 May 2023 16:57:59 +0000 Subject: [PATCH 075/253] chore(release): 1.1.1 [skip ci] ## [@helia/ipns-v1.1.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.0...@helia/ipns-v1.1.1) (2023-05-05) ### Bug Fixes * use the content routing api for get/put operations ([#34](https://github.com/ipfs/helia-ipns/issues/34)) ([55208cc](https://github.com/ipfs/helia-ipns/commit/55208ccfdc4f3a799736f29e614910cbd8375a9d)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 5a9e78de..56da0455 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.0...@helia/ipns-v1.1.1) (2023-05-05) + + +### Bug Fixes + +* use the content routing api for get/put operations ([#34](https://github.com/ipfs/helia-ipns/issues/34)) ([55208cc](https://github.com/ipfs/helia-ipns/commit/55208ccfdc4f3a799736f29e614910cbd8375a9d)) + ## [@helia/ipns-v1.1.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.0.1...@helia/ipns-v1.1.0) (2023-03-29) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 6e4c2136..092f7c50 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.0", + "version": "1.1.1", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 587505dd4336253315855d9be64a935599a0b5a6 Mon Sep 17 00:00:00 2001 From: "ipfs-mgmt-read-write[bot]" <104492829+ipfs-mgmt-read-write[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 07:39:31 +0000 Subject: [PATCH 076/253] chore: Update .github/workflows/stale.yml [skip ci] From 86b2ac2c6427d04c67323572092cd2f9058eadd2 Mon Sep 17 00:00:00 2001 From: "ipfs-mgmt-read-write[bot]" <104492829+ipfs-mgmt-read-write[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 07:39:32 +0000 Subject: [PATCH 077/253] chore: Update .github/workflows/stale.yml [skip ci] From 567f16f23e92475877a7b679ab7448cc202adc6a Mon Sep 17 00:00:00 2001 From: "ipfs-mgmt-read-write[bot]" <104492829+ipfs-mgmt-read-write[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 07:39:33 +0000 Subject: [PATCH 078/253] chore: Update .github/workflows/stale.yml [skip ci] From c38d13be29dbdc77e1f78670daac01b8d716f9b3 Mon Sep 17 00:00:00 2001 From: "ipfs-mgmt-read-write[bot]" <104492829+ipfs-mgmt-read-write[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 07:39:34 +0000 Subject: [PATCH 079/253] chore: Update .github/workflows/stale.yml [skip ci] From 704b41355768b3e8723560c5f7ed3d7c12b58c3b Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 9 May 2023 09:04:54 +0100 Subject: [PATCH 080/253] fix: cache IPNS entries after resolving (#35) After resolving one or more IPNS records, use the selector to choose one and then cache the result. Fixes #20 --- packages/ipns/src/index.ts | 28 ++++++++++--- packages/ipns/src/routing/local-store.ts | 4 +- packages/ipns/test/resolve.spec.ts | 52 +++++++++++++++++++++++- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 7e2345d1..932361b4 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -68,6 +68,7 @@ import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' import type { IPNSEntry } from 'ipns' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' import { CID } from 'multiformats/cid' import { resolveDnslink } from './utils/resolve-dns-link.js' import { logger } from '@libp2p/logger' @@ -78,6 +79,7 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import type { Datastore } from 'interface-datastore' import { localStore, LocalStore } from './routing/local-store.js' +import { CodeError } from '@libp2p/interfaces/errors' const log = logger('helia:ipns') @@ -295,16 +297,30 @@ class DefaultIPNS implements IPNS { ] } - const unmarshaledRecord = await Promise.any( - routers.map(async (router) => { - const unmarshaledRecord = await router.get(routingKey, options) - await ipnsValidator(routingKey, unmarshaledRecord) + const records: Uint8Array[] = [] - return unmarshaledRecord + await Promise.all( + routers.map(async (router) => { + try { + const record = await router.get(routingKey, options) + await ipnsValidator(routingKey, record) + + records.push(record) + } catch (err) { + log.error('error finding IPNS record', err) + } }) ) - return unmarshal(unmarshaledRecord) + if (records.length === 0) { + throw new CodeError('Could not find record for routing key', 'ERR_NOT_FOUND') + } + + const record = records[ipnsSelector(routingKey, records)] + + await this.localStore.put(routingKey, record, options) + + return unmarshal(record) } } diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index 4ea0a375..f9c1dbc8 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -25,12 +25,12 @@ export interface LocalStore extends IPNSRouting { */ export function localStore (datastore: Datastore): LocalStore { return { - async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}) { + async put (routingKey: Uint8Array, marshalledRecord: Uint8Array, options: PutOptions = {}) { try { const key = dhtRoutingKey(routingKey) // Marshal to libp2p record as the DHT does - const record = new Libp2pRecord(routingKey, marshaledRecord, new Date()) + const record = new Libp2pRecord(routingKey, marshalledRecord, new Date()) options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put')) await datastore.put(key, record.serialize(), options) diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index 70bec018..051e68bc 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -8,15 +8,21 @@ import { CID } from 'multiformats/cid' import { createEd25519PeerId } from '@libp2p/peer-id-factory' import Sinon from 'sinon' import { StubbedInstance, stubInterface } from 'sinon-ts' +import { create, marshal, peerIdToRoutingKey } from 'ipns' +import { Datastore, Key } from 'interface-datastore' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { Libp2pRecord } from '@libp2p/record' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') describe('resolve', () => { let name: IPNS let routing: StubbedInstance + let datastore: Datastore beforeEach(async () => { - const datastore = new MemoryDatastore() + datastore = new MemoryDatastore() routing = stubInterface() routing.get.throws(new Error('Not found')) @@ -98,4 +104,48 @@ describe('resolve', () => { expect(onProgress).to.have.property('called', true) }) + + it('should cache a record', async function () { + const peerId = await createEd25519PeerId() + const routingKey = peerIdToRoutingKey(peerId) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(routingKey, 'base32'), false) + + expect(datastore.has(dhtKey)).to.be.false('already had record') + + const bytes = uint8ArrayFromString(`/ipfs/${cid.toString()}`) + const record = await create(peerId, bytes, 0n, 60000) + const marshalledRecord = marshal(record) + + routing.get.withArgs(routingKey).resolves(marshalledRecord) + + const result = await name.resolve(peerId) + expect(result.toString()).to.equal(cid.toString(), 'incorrect record resolved') + + expect(datastore.has(dhtKey)).to.be.true('did not cache record locally') + }) + + it('should cache the most recent record', async function () { + const peerId = await createEd25519PeerId() + const routingKey = peerIdToRoutingKey(peerId) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(routingKey, 'base32'), false) + + const marshalledRecordA = marshal(await create(peerId, uint8ArrayFromString(`/ipfs/${cid.toString()}`), 0n, 60000)) + const marshalledRecordB = marshal(await create(peerId, uint8ArrayFromString(`/ipfs/${cid.toString()}`), 10n, 60000)) + + // records should not match + expect(marshalledRecordA).to.not.equalBytes(marshalledRecordB) + + // cache has older record + await datastore.put(dhtKey, marshalledRecordA) + routing.get.withArgs(routingKey).resolves(marshalledRecordB) + + const result = await name.resolve(peerId) + expect(result.toString()).to.equal(cid.toString(), 'incorrect record resolved') + + const cached = await datastore.get(dhtKey) + const record = Libp2pRecord.deserialize(cached) + + // should have cached the updated record + expect(record.value).to.equalBytes(marshalledRecordB) + }) }) From 90efbd370a597cae5dcbeb7b608982680895ace2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 9 May 2023 08:13:25 +0000 Subject: [PATCH 081/253] chore(release): 1.1.2 [skip ci] ## [@helia/ipns-v1.1.2](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.1...@helia/ipns-v1.1.2) (2023-05-09) ### Bug Fixes * cache IPNS entries after resolving ([#35](https://github.com/ipfs/helia-ipns/issues/35)) ([704b413](https://github.com/ipfs/helia-ipns/commit/704b41355768b3e8723560c5f7ed3d7c12b58c3b)), closes [#20](https://github.com/ipfs/helia-ipns/issues/20) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 56da0455..33f543e1 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.2](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.1...@helia/ipns-v1.1.2) (2023-05-09) + + +### Bug Fixes + +* cache IPNS entries after resolving ([#35](https://github.com/ipfs/helia-ipns/issues/35)) ([704b413](https://github.com/ipfs/helia-ipns/commit/704b41355768b3e8723560c5f7ed3d7c12b58c3b)), closes [#20](https://github.com/ipfs/helia-ipns/issues/20) + ## [@helia/ipns-v1.1.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.0...@helia/ipns-v1.1.1) (2023-05-05) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 092f7c50..07327db9 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.1", + "version": "1.1.2", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 4cdba4fda743e7805725f4155242b93bc74ba4ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 12:30:38 +0100 Subject: [PATCH 082/253] deps: update all deps and fix linting * deps(dev): bump aegir from 38.1.8 to 39.0.8 Bumps [aegir](https://github.com/ipfs/aegir) from 38.1.8 to 39.0.8. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v38.1.8...v39.0.8) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * chore: update all deps and fix linting --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 2 +- packages/interop/package.json | 24 ++++---- packages/interop/test/dht.spec.ts | 61 +++++++++++-------- .../test/fixtures/create-helia.browser.ts | 31 +++------- .../interop/test/fixtures/create-helia.ts | 11 ++-- packages/interop/test/fixtures/create-kubo.ts | 6 +- .../interop/test/fixtures/create-peer-ids.ts | 8 +-- packages/interop/test/pubsub.spec.ts | 42 +++++++------ packages/ipns/package.json | 5 +- packages/ipns/src/index.ts | 30 ++++----- packages/ipns/src/routing/dht.ts | 4 +- packages/ipns/src/routing/index.ts | 4 +- packages/ipns/src/routing/local-store.ts | 8 +-- packages/ipns/src/routing/pubsub.ts | 22 ++++--- .../src/utils/resolve-dns-link.browser.ts | 4 +- packages/ipns/src/utils/resolve-dns-link.ts | 6 +- packages/ipns/test/publish.spec.ts | 8 +-- packages/ipns/test/resolve.spec.ts | 16 ++--- 18 files changed, 147 insertions(+), 145 deletions(-) diff --git a/package.json b/package.json index 46be1855..059fda1a 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^38.1.0" + "aegir": "^39.0.8" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 6915127e..edc05fda 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,29 +52,31 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-gossipsub": "^6.1.0", - "@chainsafe/libp2p-noise": "^11.0.0", - "@chainsafe/libp2p-yamux": "^3.0.5", + "@chainsafe/libp2p-gossipsub": "^8.0.0", + "@chainsafe/libp2p-noise": "^12.0.1", + "@chainsafe/libp2p-yamux": "^4.0.2", "@helia/interface": "^1.0.0", "@helia/ipns": "^1.0.0", + "@libp2p/interface-libp2p": "^3.2.0", "@libp2p/interface-peer-id": "^2.0.1", - "@libp2p/kad-dht": "^8.0.0", + "@libp2p/interface-pubsub": "^4.0.1", + "@libp2p/kad-dht": "^9.3.4", "@libp2p/peer-id": "^2.0.1", "@libp2p/peer-id-factory": "^2.0.1", - "@libp2p/tcp": "^6.1.2", - "@libp2p/websockets": "^5.0.3", - "aegir": "^38.1.0", + "@libp2p/tcp": "^7.0.1", + "@libp2p/websockets": "^6.0.1", + "aegir": "^39.0.8", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.19.0", + "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfsd-ctl": "^13.0.0", "ipns": "^6.0.0", - "it-all": "^2.0.0", + "it-all": "^3.0.2", "it-last": "^3.0.1", - "it-map": "^2.0.0", + "it-map": "^3.0.3", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.43.2", + "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "uint8arrays": "^4.0.3", diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index 053424dc..638d15c0 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -1,31 +1,33 @@ /* eslint-env mocha */ -import { expect } from 'aegir/chai' -import { createHeliaNode } from './fixtures/create-helia.js' -import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' -import type { Controller } from 'ipfsd-ctl' -import { sha256 } from 'multiformats/hashes/sha2' -import { CID } from 'multiformats/cid' -import * as raw from 'multiformats/codecs/raw' -import type { IPNS } from '@helia/ipns' import { ipns } from '@helia/ipns' import { dht } from '@helia/ipns/routing' -import last from 'it-last' -import { kadDHT } from '@libp2p/kad-dht' -import { ipnsValidator } from 'ipns/validator' -import { ipnsSelector } from 'ipns/selector' +import { type KadDHT, kadDHT } from '@libp2p/kad-dht' import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { sortClosestPeers } from './fixtures/create-peer-ids.js' -import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' +import { expect } from 'aegir/chai' +import { ipnsSelector } from 'ipns/selector' +import { ipnsValidator } from 'ipns/validator' +import last from 'it-last' +import { identifyService } from 'libp2p/identify' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { waitFor } from './fixtures/wait-for.js' -import { connect } from './fixtures/connect.js' import { isElectronMain } from 'wherearewe' +import { connect } from './fixtures/connect.js' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import { sortClosestPeers } from './fixtures/create-peer-ids.js' +import { waitFor } from './fixtures/wait-for.js' +import type { Helia } from '@helia/interface' +import type { IPNS } from '@helia/ipns' +import type { Libp2p } from '@libp2p/interface-libp2p' +import type { Controller } from 'ipfsd-ctl' +import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' describe('dht routing', () => { - let helia: Helia + let helia: Helia> let kubo: Controller let name: IPNS @@ -46,14 +48,19 @@ describe('dht routing', () => { value = CID.createV1(raw.code, digest) helia = await createHeliaNode({ - dht: kadDHT({ - validators: { - ipns: ipnsValidator - }, - selectors: { - ipns: ipnsSelector - } - }) + services: { + identify: identifyService(), + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + }, + // skips waiting for the initial self-query to find peers + allowQueryWithZeroPeers: true + }) + } }) kubo = await createKuboNode() @@ -86,7 +93,7 @@ describe('dht routing', () => { await waitFor(async () => { let found = false - for await (const event of helia.libp2p.dht.findPeer(kubo.peer.id)) { + for await (const event of helia.libp2p.services.dht.findPeer(kubo.peer.id)) { if (event.name === 'FINAL_PEER') { found = true } diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 62eb4707..f24c3518 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,18 +1,15 @@ -import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' -import { webSockets } from '@libp2p/websockets' -import { all } from '@libp2p/websockets/filters' import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' import type { Helia } from '@helia/interface' -import { kadDHT } from '@libp2p/kad-dht' -import { gossipsub } from '@chainsafe/libp2p-gossipsub' -import { ipnsValidator } from 'ipns/validator' -import { ipnsSelector } from 'ipns/selector' +import type { Libp2p } from '@libp2p/interface-libp2p' -export async function createHeliaNode (): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -29,19 +26,11 @@ export async function createHeliaNode (): Promise { streamMuxers: [ yamux() ], - dht: kadDHT({ - validators: { - ipns: ipnsValidator - }, - selectors: { - ipns: ipnsSelector - } - }), - pubsub: gossipsub(), datastore, - nat: { - enabled: false - } + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config }) const helia = await createHelia({ diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index e2076c58..9ffe4c7a 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,13 +1,13 @@ -import { createHelia } from 'helia' -import { createLibp2p, Libp2pOptions } from 'libp2p' -import { tcp } from '@libp2p/tcp' import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2p, type Libp2pOptions } from 'libp2p' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -27,9 +27,6 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise = {}): Promise { const opts = mergeOptions({ @@ -24,5 +24,5 @@ export async function createKuboNode (options: ControllerOptions<'go'> = {}): Pr } }, options) - return await createController(opts) + return createController(opts) } diff --git a/packages/interop/test/fixtures/create-peer-ids.ts b/packages/interop/test/fixtures/create-peer-ids.ts index c38834ef..36403d85 100644 --- a/packages/interop/test/fixtures/create-peer-ids.ts +++ b/packages/interop/test/fixtures/create-peer-ids.ts @@ -1,9 +1,9 @@ -import { xor as uint8ArrayXor } from 'uint8arrays/xor' -import { compare as uint8ArrayCompare } from 'uint8arrays/compare' import all from 'it-all' import map from 'it-map' -import type { PeerId } from '@libp2p/interface-peer-id' import { sha256 } from 'multiformats/hashes/sha2' +import { compare as uint8ArrayCompare } from 'uint8arrays/compare' +import { xor as uint8ArrayXor } from 'uint8arrays/xor' +import type { PeerId } from '@libp2p/interface-peer-id' /** * Sort peers by distance to the KadID of the passed buffer @@ -33,7 +33,7 @@ export async function sortClosestPeers (buf: Uint8Array, peers: PeerId[]): Promi * Creates a DHT ID by hashing a Peer ID */ export async function convertPeerId (peerId: PeerId): Promise { - return await convertBuffer(peerId.toBytes()) + return convertBuffer(peerId.toBytes()) } /** diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index 02453832..5e315736 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -1,37 +1,43 @@ /* eslint-env mocha */ +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { ipns } from '@helia/ipns' +import { pubsub } from '@helia/ipns/routing' +import { peerIdFromKeys } from '@libp2p/peer-id' import { expect } from 'aegir/chai' -import { createHeliaNode } from './fixtures/create-helia.js' -import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' -import type { Controller } from 'ipfsd-ctl' -import { sha256 } from 'multiformats/hashes/sha2' +import last from 'it-last' +import { identifyService } from 'libp2p/identify' +import { base36 } from 'multiformats/bases/base36' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' -import { base36 } from 'multiformats/bases/base36' -import type { IPNS } from '@helia/ipns' -import { ipns } from '@helia/ipns' -import { pubsub } from '@helia/ipns/routing' -import last from 'it-last' -import { peerIdFromKeys } from '@libp2p/peer-id' -import { gossipsub } from '@chainsafe/libp2p-gossipsub' -import { waitFor } from './fixtures/wait-for.js' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { sha256 } from 'multiformats/hashes/sha2' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { connect } from './fixtures/connect.js' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import { waitFor } from './fixtures/wait-for.js' +import type { Helia } from '@helia/interface' +import type { IPNS } from '@helia/ipns' +import type { PubSub } from '@libp2p/interface-pubsub' +import type { Controller } from 'ipfsd-ctl' +import type { Libp2p } from 'libp2p' const LIBP2P_KEY_CODEC = 0x72 describe('pubsub routing', () => { - let helia: Helia + let helia: Helia> let kubo: Controller let name: IPNS beforeEach(async () => { helia = await createHeliaNode({ - pubsub: gossipsub() + services: { + identify: identifyService(), + pubsub: gossipsub() + } }) kubo = await createKuboNode({ ipfsOptions: { @@ -130,7 +136,7 @@ describe('pubsub routing', () => { // wait for helia to be subscribed to the topic for record updates await waitFor(async () => { - return helia.libp2p.pubsub.getTopics().includes(subscriptionName) + return helia.libp2p.services.pubsub.getTopics().includes(subscriptionName) }, { timeout: 30000, message: 'Helia did not register for record updates' diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 07327db9..86fb344f 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -160,9 +160,8 @@ }, "dependencies": { "@libp2p/interface-content-routing": "^2.1.0", - "@libp2p/interface-dht": "^2.0.1", "@libp2p/interface-peer-id": "^2.0.1", - "@libp2p/interface-pubsub": "^3.0.6", + "@libp2p/interface-pubsub": "^4.0.1", "@libp2p/interfaces": "^3.3.1", "@libp2p/logger": "^2.0.6", "@libp2p/peer-id": "^2.0.1", @@ -178,7 +177,7 @@ }, "devDependencies": { "@libp2p/peer-id-factory": "^2.0.1", - "aegir": "^38.1.0", + "aegir": "^39.0.8", "datastore-core": "^9.0.3", "sinon": "^15.0.1", "sinon-ts": "^1.0.0" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 932361b4..9d89b630 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -62,24 +62,24 @@ * ``` */ -import type { AbortOptions } from '@libp2p/interfaces' -import { isPeerId, PeerId } from '@libp2p/interface-peer-id' +import { isPeerId, type PeerId } from '@libp2p/interface-peer-id' +import { CodeError } from '@libp2p/interfaces/errors' +import { logger } from '@libp2p/logger' +import { peerIdFromString } from '@libp2p/peer-id' import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' -import type { IPNSEntry } from 'ipns' -import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' -import { ipnsValidator } from 'ipns/validator' import { ipnsSelector } from 'ipns/selector' +import { ipnsValidator } from 'ipns/validator' import { CID } from 'multiformats/cid' -import { resolveDnslink } from './utils/resolve-dns-link.js' -import { logger } from '@libp2p/logger' -import { peerIdFromString } from '@libp2p/peer-id' -import type { ProgressEvent, ProgressOptions } from 'progress-events' import { CustomProgressEvent } from 'progress-events' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { localStore, type LocalStore } from './routing/local-store.js' +import { resolveDnslink } from './utils/resolve-dns-link.js' +import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' +import type { AbortOptions } from '@libp2p/interfaces' import type { Datastore } from 'interface-datastore' -import { localStore, LocalStore } from './routing/local-store.js' -import { CodeError } from '@libp2p/interfaces/errors' +import type { IPNSEntry } from 'ipns' +import type { ProgressEvent, ProgressOptions } from 'progress-events' const log = logger('helia:ipns') @@ -223,13 +223,13 @@ class DefaultIPNS implements IPNS { const record = await this.#findIpnsRecord(routingKey, options) const str = uint8ArrayToString(record.value) - return await this.#resolve(str, options) + return this.#resolve(str, options) } async resolveDns (domain: string, options: ResolveDNSOptions = {}): Promise { const dnslink = await resolveDnslink(domain, options) - return await this.#resolve(dnslink, options) + return this.#resolve(dnslink, options) } republish (options: RepublishOptions = {}): void { @@ -275,7 +275,7 @@ class DefaultIPNS implements IPNS { const scheme = parts[1] if (scheme === 'ipns') { - return await this.resolve(peerIdFromString(parts[2]), options) + return this.resolve(peerIdFromString(parts[2]), options) } else if (scheme === 'ipfs') { return CID.parse(parts[2]) } diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/dht.ts index 93215f9d..2b970cb3 100644 --- a/packages/ipns/src/routing/dht.ts +++ b/packages/ipns/src/routing/dht.ts @@ -1,6 +1,6 @@ -import type { IPNSRouting } from '../index.js' +import { CustomProgressEvent, type ProgressEvent } from 'progress-events' import type { GetOptions, PutOptions } from './index.js' -import { CustomProgressEvent, ProgressEvent } from 'progress-events' +import type { IPNSRouting } from '../index.js' import type { ContentRouting } from '@libp2p/interface-content-routing' export interface DHTRoutingComponents { diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts index 38374874..bc0659b6 100644 --- a/packages/ipns/src/routing/index.ts +++ b/packages/ipns/src/routing/index.ts @@ -1,8 +1,8 @@ -import type { ProgressOptions } from 'progress-events' -import type { AbortOptions } from '@libp2p/interfaces' import type { DHTProgressEvents } from './dht.js' import type { DatastoreProgressEvents } from './local-store.js' import type { PubSubProgressEvents } from './pubsub.js' +import type { AbortOptions } from '@libp2p/interfaces' +import type { ProgressOptions } from 'progress-events' export interface PutOptions extends AbortOptions, ProgressOptions { diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index f9c1dbc8..eef3f41e 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -1,9 +1,9 @@ -import { CustomProgressEvent, ProgressEvent } from 'progress-events' -import type { AbortOptions } from '@libp2p/interfaces' import { Libp2pRecord } from '@libp2p/record' -import { Datastore, Key } from 'interface-datastore' +import { type Datastore, Key } from 'interface-datastore' +import { CustomProgressEvent, type ProgressEvent } from 'progress-events' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import type { GetOptions, IPNSRouting, PutOptions } from '../routing' +import type { AbortOptions } from '@libp2p/interfaces' function dhtRoutingKey (key: Uint8Array): Key { return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) @@ -57,7 +57,7 @@ export function localStore (datastore: Datastore): LocalStore { }, async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { const key = dhtRoutingKey(routingKey) - return await datastore.has(key, options) + return datastore.has(key, options) } } } diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts index bb10e300..83a57e43 100644 --- a/packages/ipns/src/routing/pubsub.ts +++ b/packages/ipns/src/routing/pubsub.ts @@ -1,17 +1,17 @@ +import { CodeError } from '@libp2p/interfaces/errors' +import { logger } from '@libp2p/logger' import { peerIdToRoutingKey } from 'ipns' +import { ipnsSelector } from 'ipns/selector' +import { ipnsValidator } from 'ipns/validator' +import { CustomProgressEvent, type ProgressEvent } from 'progress-events' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { logger } from '@libp2p/logger' +import { localStore, type LocalStore } from './local-store.js' +import type { GetOptions, IPNSRouting, PutOptions } from './index.js' import type { PeerId } from '@libp2p/interface-peer-id' import type { Message, PublishResult, PubSub } from '@libp2p/interface-pubsub' import type { Datastore } from 'interface-datastore' -import type { GetOptions, IPNSRouting, PutOptions } from './index.js' -import { CodeError } from '@libp2p/interfaces/errors' -import { localStore, LocalStore } from './local-store.js' -import { ipnsValidator } from 'ipns/validator' -import { ipnsSelector } from 'ipns/selector' -import { equals as uint8ArrayEquals } from 'uint8arrays/equals' -import { CustomProgressEvent, ProgressEvent } from 'progress-events' const log = logger('helia:ipns:routing:pubsub') @@ -19,7 +19,9 @@ export interface PubsubRoutingComponents { datastore: Datastore libp2p: { peerId: PeerId - pubsub: PubSub + services: { + pubsub: PubSub + } } } @@ -46,7 +48,7 @@ class PubSubRouting implements IPNSRouting { this.subscriptions = [] this.localStore = localStore(components.datastore) this.peerId = components.libp2p.peerId - this.pubsub = components.libp2p.pubsub + this.pubsub = components.libp2p.services.pubsub this.pubsub.addEventListener('message', (evt) => { const message = evt.detail diff --git a/packages/ipns/src/utils/resolve-dns-link.browser.ts b/packages/ipns/src/utils/resolve-dns-link.browser.ts index 459f387c..4035ca6a 100644 --- a/packages/ipns/src/utils/resolve-dns-link.browser.ts +++ b/packages/ipns/src/utils/resolve-dns-link.browser.ts @@ -1,7 +1,7 @@ /* eslint-env browser */ -import { TLRU } from './tlru.js' import PQueue from 'p-queue' +import { TLRU } from './tlru.js' import type { AbortOptions } from '@libp2p/interfaces' // Avoid sending multiple queries for the same hostname by caching results @@ -57,5 +57,5 @@ export async function resolveDnslink (fqdn: string, opts: ResolveDnsLinkOptions return ipfsPath(response) } - return await resolve(fqdn, opts) + return resolve(fqdn, opts) } diff --git a/packages/ipns/src/utils/resolve-dns-link.ts b/packages/ipns/src/utils/resolve-dns-link.ts index 34ec0341..d77cf2d7 100644 --- a/packages/ipns/src/utils/resolve-dns-link.ts +++ b/packages/ipns/src/utils/resolve-dns-link.ts @@ -1,12 +1,12 @@ import dns from 'dns' import { promisify } from 'util' -import type { AbortOptions } from '@libp2p/interfaces' import * as isIPFS from 'is-ipfs' +import type { AbortOptions } from '@libp2p/interfaces' const MAX_RECURSIVE_DEPTH = 32 export async function resolveDnslink (domain: string, options: AbortOptions = {}): Promise { - return await recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, options) + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, options) } async function recursiveResolveDnslink (domain: string, depth: number, options: AbortOptions = {}): Promise { @@ -44,7 +44,7 @@ async function recursiveResolveDnslink (domain: string, depth: number, options: return result } - return await recursiveResolveDnslink(domainOrCID, depth - 1, options) + return recursiveResolveDnslink(domainOrCID, depth - 1, options) } async function resolve (domain: string, options: AbortOptions = {}): Promise { diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index 3e8b7aab..184e835b 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -1,13 +1,13 @@ /* eslint-env mocha */ +import { createEd25519PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' -import type { IPNS, IPNSRouting } from '../src/index.js' -import { ipns } from '../src/index.js' import { CID } from 'multiformats/cid' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' import Sinon from 'sinon' -import { StubbedInstance, stubInterface } from 'sinon-ts' +import { type StubbedInstance, stubInterface } from 'sinon-ts' +import { ipns } from '../src/index.js' +import type { IPNS, IPNSRouting } from '../src/index.js' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index 051e68bc..f3b23e13 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -1,18 +1,18 @@ /* eslint-env mocha */ +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { Libp2pRecord } from '@libp2p/record' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' -import type { IPNS, IPNSRouting } from '../src/index.js' -import { ipns } from '../src/index.js' +import { type Datastore, Key } from 'interface-datastore' +import { create, marshal, peerIdToRoutingKey } from 'ipns' import { CID } from 'multiformats/cid' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' import Sinon from 'sinon' -import { StubbedInstance, stubInterface } from 'sinon-ts' -import { create, marshal, peerIdToRoutingKey } from 'ipns' -import { Datastore, Key } from 'interface-datastore' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { type StubbedInstance, stubInterface } from 'sinon-ts' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Libp2pRecord } from '@libp2p/record' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { ipns } from '../src/index.js' +import type { IPNS, IPNSRouting } from '../src/index.js' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') From 25b38fd32359e9414c0bc758af215f2ba57d1d87 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 24 May 2023 11:38:35 +0000 Subject: [PATCH 083/253] chore(release): 1.1.3 [skip ci] ## [@helia/ipns-v1.1.3](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.2...@helia/ipns-v1.1.3) (2023-05-24) ### Dependencies * update all deps and fix linting ([4cdba4f](https://github.com/ipfs/helia-ipns/commit/4cdba4fda743e7805725f4155242b93bc74ba4ae)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 33f543e1..79f228ef 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.3](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.2...@helia/ipns-v1.1.3) (2023-05-24) + + +### Dependencies + +* update all deps and fix linting ([4cdba4f](https://github.com/ipfs/helia-ipns/commit/4cdba4fda743e7805725f4155242b93bc74ba4ae)) + ## [@helia/ipns-v1.1.2](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.1...@helia/ipns-v1.1.2) (2023-05-09) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 86fb344f..06ab1446 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.2", + "version": "1.1.3", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From d4d6515f023db339874d34871e69fb7c3fc47f6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 13:31:19 +0100 Subject: [PATCH 084/253] deps: update all deps and fix linting * deps(dev): bump aegir from 38.1.8 to 39.0.8 Bumps [aegir](https://github.com/ipfs/aegir) from 38.1.8 to 39.0.8. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v38.1.8...v39.0.8) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * chore: update all deps, fix linting * chore: add identify * chore: update fixtures * chore: deps again --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 2 +- packages/interop/package.json | 14 ++++---- packages/interop/test/bitswap.spec.ts | 9 +++-- packages/interop/test/files.spec.ts | 10 +++--- .../test/fixtures/create-helia.browser.ts | 21 ++++++----- .../interop/test/fixtures/create-helia.ts | 11 +++--- packages/interop/test/fixtures/create-kubo.ts | 6 ++-- packages/unixfs/package.json | 4 +-- packages/unixfs/src/commands/add.ts | 8 ++--- packages/unixfs/src/commands/cat.ts | 8 ++--- packages/unixfs/src/commands/chmod.ts | 30 ++++++++-------- packages/unixfs/src/commands/cp.ts | 10 +++--- packages/unixfs/src/commands/ls.ts | 10 +++--- packages/unixfs/src/commands/mkdir.ts | 14 ++++---- packages/unixfs/src/commands/rm.ts | 10 +++--- packages/unixfs/src/commands/stat.ts | 16 ++++----- packages/unixfs/src/commands/touch.ts | 30 ++++++++-------- .../unixfs/src/commands/utils/add-link.ts | 28 +++++++-------- .../src/commands/utils/cid-to-directory.ts | 8 ++--- .../src/commands/utils/cid-to-pblink.ts | 8 ++--- .../unixfs/src/commands/utils/dir-sharded.ts | 12 +++---- .../unixfs/src/commands/utils/hamt-utils.ts | 18 +++++----- .../commands/utils/is-over-shard-threshold.ts | 6 ++-- packages/unixfs/src/commands/utils/persist.ts | 4 +-- .../unixfs/src/commands/utils/remove-link.ts | 22 ++++++------ packages/unixfs/src/commands/utils/resolve.ts | 8 ++--- packages/unixfs/src/index.ts | 36 +++++++++---------- packages/unixfs/test/add.spec.ts | 4 +-- packages/unixfs/test/cat.spec.ts | 8 ++--- packages/unixfs/test/chmod.spec.ts | 6 ++-- packages/unixfs/test/cp.spec.ts | 12 +++---- .../test/fixtures/create-sharded-directory.ts | 4 +-- .../fixtures/create-subsharded-directory.ts | 8 ++--- packages/unixfs/test/fixtures/print-tree.ts | 6 ++-- packages/unixfs/test/ls.spec.ts | 8 ++--- packages/unixfs/test/mkdir.spec.ts | 8 ++--- packages/unixfs/test/rm.spec.ts | 8 ++--- packages/unixfs/test/stat.spec.ts | 8 ++--- packages/unixfs/test/touch.spec.ts | 6 ++-- 39 files changed, 227 insertions(+), 222 deletions(-) diff --git a/package.json b/package.json index d8234687..91c3b6b5 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^38.1.0" + "aegir": "^39.0.8" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 565bc3eb..001e0e3d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,23 +52,23 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^11.0.0", - "@chainsafe/libp2p-yamux": "^3.0.5", + "@chainsafe/libp2p-noise": "^12.0.1", + "@chainsafe/libp2p-yamux": "^4.0.2", "@helia/interface": "^1.0.0", "@helia/unixfs": "^1.0.0", - "@libp2p/tcp": "^6.1.2", - "@libp2p/websockets": "^5.0.3", - "aegir": "^38.1.0", + "@libp2p/tcp": "^7.0.1", + "@libp2p/websockets": "^6.0.1", + "aegir": "^39.0.8", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.19.0", + "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^4.0.1", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.43.2", + "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "wherearewe": "^2.0.1" diff --git a/packages/interop/test/bitswap.spec.ts b/packages/interop/test/bitswap.spec.ts index 13a5fe96..9104246b 100644 --- a/packages/interop/test/bitswap.spec.ts +++ b/packages/interop/test/bitswap.spec.ts @@ -1,13 +1,13 @@ /* eslint-env mocha */ +import { type UnixFS, unixfs } from '@helia/unixfs' import { expect } from 'aegir/chai' +import toBuffer from 'it-to-buffer' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' -import type { Controller } from 'ipfsd-ctl' -import { UnixFS, unixfs } from '@helia/unixfs' import type { FileCandidate } from 'ipfs-unixfs-importer' -import toBuffer from 'it-to-buffer' +import type { Controller } from 'ipfsd-ctl' describe('unixfs bitswap interop', () => { let helia: Helia @@ -20,8 +20,7 @@ describe('unixfs bitswap interop', () => { kubo = await createKuboNode() // connect helia to kubo - await helia.libp2p.peerStore.addressBook.add(kubo.peer.id, kubo.peer.addresses) - await helia.libp2p.dial(kubo.peer.id) + await helia.libp2p.dial(kubo.peer.addresses) }) afterEach(async () => { diff --git a/packages/interop/test/files.spec.ts b/packages/interop/test/files.spec.ts index 94f75604..4988f3ac 100644 --- a/packages/interop/test/files.spec.ts +++ b/packages/interop/test/files.spec.ts @@ -1,16 +1,16 @@ /* eslint-env mocha */ +import { type AddOptions, type UnixFS, unixfs } from '@helia/unixfs' import { expect } from 'aegir/chai' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import { balanced } from 'ipfs-unixfs-importer/layout' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' -import type { Controller } from 'ipfsd-ctl' -import { AddOptions, UnixFS, unixfs } from '@helia/unixfs' -import { balanced } from 'ipfs-unixfs-importer/layout' -import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js' import type { FileCandidate } from 'ipfs-unixfs-importer' +import type { Controller } from 'ipfsd-ctl' import type { CID } from 'multiformats/cid' -import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js' describe('unixfs interop', () => { let helia: Helia diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index d28c6efa..867de492 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,14 +1,15 @@ -import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' -import { webSockets } from '@libp2p/websockets' -import { all } from '@libp2p/websockets/filters' import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -25,10 +26,14 @@ export async function createHeliaNode (): Promise { streamMuxers: [ yamux() ], + services: { + identify: identifyService() + }, datastore, - nat: { - enabled: false - } + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config }) const helia = await createHelia({ diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index 7a7201b0..f7a25c3f 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,10 +1,11 @@ -import { createHelia } from 'helia' -import { createLibp2p, Libp2pOptions } from 'libp2p' -import { tcp } from '@libp2p/tcp' import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -22,8 +23,8 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise = {}): Promise { const opts = mergeOptions({ @@ -24,5 +24,5 @@ export async function createKuboNode (options: ControllerOptions<'go'> = {}): Pr } }, options) - return await createController(opts) + return createController(opts) } diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 6c8bb09d..0da91828 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -157,9 +157,9 @@ "sparse-array": "^1.3.2" }, "devDependencies": { - "aegir": "^38.1.0", + "aegir": "^39.0.8", "blockstore-core": "^4.0.1", - "delay": "^5.0.0", + "delay": "^6.0.0", "it-all": "^3.0.1", "it-drain": "^3.0.1", "it-first": "^3.0.1", diff --git a/packages/unixfs/src/commands/add.ts b/packages/unixfs/src/commands/add.ts index 33e334d7..5bc60cb6 100644 --- a/packages/unixfs/src/commands/add.ts +++ b/packages/unixfs/src/commands/add.ts @@ -1,8 +1,8 @@ -import type { CID } from 'multiformats/cid' -import type { Blocks } from '@helia/interface/blocks' -import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer' -import { balanced } from 'ipfs-unixfs-importer/layout' +import { type ByteStream, type DirectoryCandidate, type FileCandidate, importBytes, importByteStream, type ImportCandidateStream, importDirectory, importer, type ImporterOptions, importFile, type ImportResult } from 'ipfs-unixfs-importer' import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import { balanced } from 'ipfs-unixfs-importer/layout' +import type { Blocks } from '@helia/interface/blocks' +import type { CID } from 'multiformats/cid' /** * Default importer settings match Filecoin diff --git a/packages/unixfs/src/commands/cat.ts b/packages/unixfs/src/commands/cat.ts index 328cdae9..1360281f 100644 --- a/packages/unixfs/src/commands/cat.ts +++ b/packages/unixfs/src/commands/cat.ts @@ -1,10 +1,10 @@ import { exporter } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' -import type { CatOptions } from '../index.js' -import { resolve } from './utils/resolve.js' import mergeOpts from 'merge-options' -import type { Blocks } from '@helia/interface/blocks' import { NoContentError, NotAFileError } from './utils/errors.js' +import { resolve } from './utils/resolve.js' +import type { CatOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' +import type { CID } from 'multiformats/cid' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) diff --git a/packages/unixfs/src/commands/chmod.ts b/packages/unixfs/src/commands/chmod.ts index 3785f058..b0092d39 100644 --- a/packages/unixfs/src/commands/chmod.ts +++ b/packages/unixfs/src/commands/chmod.ts @@ -1,21 +1,21 @@ -import { recursive } from 'ipfs-unixfs-exporter' -import { CID } from 'multiformats/cid' -import type { ChmodOptions } from '../index.js' -import mergeOpts from 'merge-options' +import * as dagPB from '@ipld/dag-pb' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' -import { pipe } from 'it-pipe' -import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' -import * as dagPB from '@ipld/dag-pb' -import type { PBNode, PBLink } from '@ipld/dag-pb' +import { recursive } from 'ipfs-unixfs-exporter' import { importer } from 'ipfs-unixfs-importer' -import { persist } from './utils/persist.js' -import type { Blocks } from '@helia/interface/blocks' import last from 'it-last' -import { sha256 } from 'multiformats/hashes/sha2' -import { resolve, updatePathCids } from './utils/resolve.js' +import { pipe } from 'it-pipe' +import mergeOpts from 'merge-options' +import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' +import { persist } from './utils/persist.js' +import { resolve, updatePathCids } from './utils/resolve.js' +import type { ChmodOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' +import type { PBNode, PBLink } from '@ipld/dag-pb' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:chmod') @@ -94,14 +94,14 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options } } }), - async (nodes) => await last(nodes) + async (nodes) => last(nodes) ) if (root == null) { throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`) } - return await updatePathCids(root.cid, resolved, blockstore, opts) + return updatePathCids(root.cid, resolved, blockstore, opts) } const block = await blockstore.get(resolved.cid) @@ -133,5 +133,5 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options await blockstore.put(updatedCid, updatedBlock) - return await updatePathCids(updatedCid, resolved, blockstore, opts) + return updatePathCids(updatedCid, resolved, blockstore, opts) } diff --git a/packages/unixfs/src/commands/cp.ts b/packages/unixfs/src/commands/cp.ts index 1069b4ab..010a9612 100644 --- a/packages/unixfs/src/commands/cp.ts +++ b/packages/unixfs/src/commands/cp.ts @@ -1,13 +1,13 @@ -import type { Blocks } from '@helia/interface/blocks' -import type { CID } from 'multiformats/cid' -import type { CpOptions } from '../index.js' -import mergeOpts from 'merge-options' import { logger } from '@libp2p/logger' +import mergeOpts from 'merge-options' import { addLink } from './utils/add-link.js' -import { cidToPBLink } from './utils/cid-to-pblink.js' import { cidToDirectory } from './utils/cid-to-directory.js' +import { cidToPBLink } from './utils/cid-to-pblink.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' import { InvalidParametersError } from './utils/errors.js' +import type { CpOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' +import type { CID } from 'multiformats/cid' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:cp') diff --git a/packages/unixfs/src/commands/ls.ts b/packages/unixfs/src/commands/ls.ts index f27f2e25..7525b66f 100644 --- a/packages/unixfs/src/commands/ls.ts +++ b/packages/unixfs/src/commands/ls.ts @@ -1,10 +1,10 @@ -import { exporter, UnixFSEntry } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' -import type { LsOptions } from '../index.js' -import { resolve } from './utils/resolve.js' +import { exporter, type UnixFSEntry } from 'ipfs-unixfs-exporter' import mergeOpts from 'merge-options' -import type { Blocks } from '@helia/interface/blocks' import { NoContentError, NotADirectoryError } from './utils/errors.js' +import { resolve } from './utils/resolve.js' +import type { LsOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' +import type { CID } from 'multiformats/cid' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) diff --git a/packages/unixfs/src/commands/mkdir.ts b/packages/unixfs/src/commands/mkdir.ts index b2281023..272a94ea 100644 --- a/packages/unixfs/src/commands/mkdir.ts +++ b/packages/unixfs/src/commands/mkdir.ts @@ -1,17 +1,17 @@ -import { CID } from 'multiformats/cid' -import mergeOpts from 'merge-options' -import { logger } from '@libp2p/logger' -import type { MkdirOptions } from '../index.js' import * as dagPB from '@ipld/dag-pb' -import { addLink } from './utils/add-link.js' -import type { Blocks } from '@helia/interface/blocks' +import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' -import { sha256 } from 'multiformats/hashes/sha2' import { exporter } from 'ipfs-unixfs-exporter' +import mergeOpts from 'merge-options' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import { addLink } from './utils/add-link.js' import { cidToDirectory } from './utils/cid-to-directory.js' import { cidToPBLink } from './utils/cid-to-pblink.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' import { InvalidParametersError, NotADirectoryError } from './utils/errors.js' +import type { MkdirOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:mkdir') diff --git a/packages/unixfs/src/commands/rm.ts b/packages/unixfs/src/commands/rm.ts index 8f091b8a..8da00de4 100644 --- a/packages/unixfs/src/commands/rm.ts +++ b/packages/unixfs/src/commands/rm.ts @@ -1,12 +1,12 @@ -import type { Blocks } from '@helia/interface/blocks' -import type { CID } from 'multiformats/cid' -import type { RmOptions } from '../index.js' -import mergeOpts from 'merge-options' import { logger } from '@libp2p/logger' -import { removeLink } from './utils/remove-link.js' +import mergeOpts from 'merge-options' import { cidToDirectory } from './utils/cid-to-directory.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' import { InvalidParametersError } from './utils/errors.js' +import { removeLink } from './utils/remove-link.js' +import type { RmOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' +import type { CID } from 'multiformats/cid' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:rm') diff --git a/packages/unixfs/src/commands/stat.ts b/packages/unixfs/src/commands/stat.ts index d350319f..6955418e 100644 --- a/packages/unixfs/src/commands/stat.ts +++ b/packages/unixfs/src/commands/stat.ts @@ -1,16 +1,16 @@ -import { exporter } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' -import type { StatOptions, UnixFSStats } from '../index.js' -import mergeOpts from 'merge-options' +import * as dagPb from '@ipld/dag-pb' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' +import { exporter } from 'ipfs-unixfs-exporter' +import mergeOpts from 'merge-options' +import * as raw from 'multiformats/codecs/raw' import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' -import * as dagPb from '@ipld/dag-pb' -import type { AbortOptions } from '@libp2p/interfaces' -import type { Mtime } from 'ipfs-unixfs' import { resolve } from './utils/resolve.js' -import * as raw from 'multiformats/codecs/raw' +import type { StatOptions, UnixFSStats } from '../index.js' import type { Blocks } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { Mtime } from 'ipfs-unixfs' +import type { CID } from 'multiformats/cid' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:stat') diff --git a/packages/unixfs/src/commands/touch.ts b/packages/unixfs/src/commands/touch.ts index 57b3f396..7482f6f7 100644 --- a/packages/unixfs/src/commands/touch.ts +++ b/packages/unixfs/src/commands/touch.ts @@ -1,21 +1,21 @@ -import { recursive } from 'ipfs-unixfs-exporter' -import { CID } from 'multiformats/cid' -import type { TouchOptions } from '../index.js' -import mergeOpts from 'merge-options' +import * as dagPB from '@ipld/dag-pb' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' -import { pipe } from 'it-pipe' -import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' -import * as dagPB from '@ipld/dag-pb' -import type { PBNode, PBLink } from '@ipld/dag-pb' +import { recursive } from 'ipfs-unixfs-exporter' import { importer } from 'ipfs-unixfs-importer' -import { persist } from './utils/persist.js' -import type { Blocks } from '@helia/interface/blocks' import last from 'it-last' -import { sha256 } from 'multiformats/hashes/sha2' -import { resolve, updatePathCids } from './utils/resolve.js' +import { pipe } from 'it-pipe' +import mergeOpts from 'merge-options' +import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' +import { persist } from './utils/persist.js' +import { resolve, updatePathCids } from './utils/resolve.js' +import type { TouchOptions } from '../index.js' +import type { Blocks } from '@helia/interface/blocks' +import type { PBNode, PBLink } from '@ipld/dag-pb' const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) const log = logger('helia:unixfs:touch') @@ -98,14 +98,14 @@ export async function touch (cid: CID, blockstore: Blocks, options: Partial await last(nodes) + async (nodes) => last(nodes) ) if (root == null) { throw new UnknownError(`Could not chmod ${resolved.cid.toString()}`) } - return await updatePathCids(root.cid, resolved, blockstore, opts) + return updatePathCids(root.cid, resolved, blockstore, opts) } const block = await blockstore.get(resolved.cid) @@ -136,5 +136,5 @@ export async function touch (cid: CID, blockstore: Blocks, options: Partial, block if (meta.type === 'hamt-sharded-directory') { log('adding link to sharded directory') - return await addToShardedDirectory(parent, child, blockstore, options) + return addToShardedDirectory(parent, child, blockstore, options) } log(`adding ${child.Name} (${child.Hash}) to regular directory`) @@ -253,5 +253,5 @@ const addToShardedDirectory = async (parent: Directory, child: Required, log('adding %s to existing sub-shard', linkName) } - return await updateShardedDirectory(path, blockstore, options) + return updateShardedDirectory(path, blockstore, options) } diff --git a/packages/unixfs/src/commands/utils/cid-to-directory.ts b/packages/unixfs/src/commands/utils/cid-to-directory.ts index 04282be0..afe5638a 100644 --- a/packages/unixfs/src/commands/utils/cid-to-directory.ts +++ b/packages/unixfs/src/commands/utils/cid-to-directory.ts @@ -1,8 +1,8 @@ -import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' -import type { PBNode } from '@ipld/dag-pb' -import type { Blocks } from '@helia/interface/blocks' +import { exporter, type ExporterOptions } from 'ipfs-unixfs-exporter' import { NotADirectoryError } from './errors.js' +import type { Blocks } from '@helia/interface/blocks' +import type { PBNode } from '@ipld/dag-pb' +import type { CID } from 'multiformats/cid' export interface Directory { cid: CID diff --git a/packages/unixfs/src/commands/utils/cid-to-pblink.ts b/packages/unixfs/src/commands/utils/cid-to-pblink.ts index af351fc1..513ea0ab 100644 --- a/packages/unixfs/src/commands/utils/cid-to-pblink.ts +++ b/packages/unixfs/src/commands/utils/cid-to-pblink.ts @@ -1,9 +1,9 @@ -import { exporter, ExporterOptions } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' -import { NotUnixFSError } from './errors.js' import * as dagPb from '@ipld/dag-pb' -import type { PBNode, PBLink } from '@ipld/dag-pb' +import { exporter, type ExporterOptions } from 'ipfs-unixfs-exporter' +import { NotUnixFSError } from './errors.js' import type { Blocks } from '@helia/interface/blocks' +import type { PBNode, PBLink } from '@ipld/dag-pb' +import type { CID } from 'multiformats/cid' export async function cidToPBLink (cid: CID, name: string, blockstore: Blocks, options?: ExporterOptions): Promise> { const sourceEntry = await exporter(cid, blockstore, options) diff --git a/packages/unixfs/src/commands/utils/dir-sharded.ts b/packages/unixfs/src/commands/utils/dir-sharded.ts index 66721742..4fba36cb 100644 --- a/packages/unixfs/src/commands/utils/dir-sharded.ts +++ b/packages/unixfs/src/commands/utils/dir-sharded.ts @@ -1,14 +1,14 @@ -import { encode, PBLink, prepare } from '@ipld/dag-pb' +import { encode, type PBLink, prepare } from '@ipld/dag-pb' +import { createHAMT, Bucket, type BucketChild } from 'hamt-sharding' import { UnixFS } from 'ipfs-unixfs' -import { persist, PersistOptions } from './persist.js' -import { createHAMT, Bucket, BucketChild } from 'hamt-sharding' +import { CID } from 'multiformats/cid' import { hamtHashCode, hamtHashFn } from './hamt-constants.js' -import { CID } from 'multiformats/cid' -import type { Mtime } from 'ipfs-unixfs' +import { persist, type PersistOptions } from './persist.js' import type { Blocks } from '@helia/interface/blocks' +import type { Mtime } from 'ipfs-unixfs' interface InProgressImportResult extends ImportResult { single?: boolean @@ -95,7 +95,7 @@ export class DirSharded extends Dir { } async get (name: string): Promise { - return await this._bucket.get(name) + return this._bucket.get(name) } childCount (): number { diff --git a/packages/unixfs/src/commands/utils/hamt-utils.ts b/packages/unixfs/src/commands/utils/hamt-utils.ts index cbbf6d32..aed968f2 100644 --- a/packages/unixfs/src/commands/utils/hamt-utils.ts +++ b/packages/unixfs/src/commands/utils/hamt-utils.ts @@ -1,24 +1,24 @@ import * as dagPB from '@ipld/dag-pb' -import { DirSharded } from './dir-sharded.js' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' import last from 'it-last' -import type { CID, Version } from 'multiformats/cid' +// @ts-expect-error no types +import SparseArray from 'sparse-array' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { type InfiniteHash, wrapHash } from './consumable-hash.js' +import { DirSharded } from './dir-sharded.js' import { hamtHashCode, hamtHashFn, hamtBucketBits } from './hamt-constants.js' +import { persist } from './persist.js' +import type { PersistOptions } from './persist.js' import type { Blocks } from '@helia/interface/blocks' -import type { Mtime } from 'ipfs-unixfs' import type { AbortOptions } from '@libp2p/interfaces' +import type { Mtime } from 'ipfs-unixfs' import type { ImportResult } from 'ipfs-unixfs-importer' -import { persist } from './persist.js' -import { InfiniteHash, wrapHash } from './consumable-hash.js' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -// @ts-expect-error no types -import SparseArray from 'sparse-array' -import type { PersistOptions } from './persist.js' +import type { CID, Version } from 'multiformats/cid' const log = logger('helia:unixfs:commands:utils:hamt-utils') diff --git a/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts index 093dee6d..c14741dc 100644 --- a/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts +++ b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts @@ -1,8 +1,8 @@ -import type { PBNode } from '@ipld/dag-pb' -import type { Blocks } from '@helia/interface/blocks' -import { UnixFS } from 'ipfs-unixfs' import * as dagPb from '@ipld/dag-pb' +import { UnixFS } from 'ipfs-unixfs' import { CID_V0, CID_V1 } from './dir-sharded.js' +import type { Blocks } from '@helia/interface/blocks' +import type { PBNode } from '@ipld/dag-pb' /** * Estimate node size only based on DAGLink name and CID byte lengths diff --git a/packages/unixfs/src/commands/utils/persist.ts b/packages/unixfs/src/commands/utils/persist.ts index ba14330d..a0f918a4 100644 --- a/packages/unixfs/src/commands/utils/persist.ts +++ b/packages/unixfs/src/commands/utils/persist.ts @@ -1,9 +1,9 @@ -import { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' +import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' import type { Blocks } from '@helia/interface/blocks' -import type { BlockCodec } from 'multiformats/codecs/interface' import type { Version as CIDVersion } from 'multiformats/cid' +import type { BlockCodec } from 'multiformats/codecs/interface' export interface PersistOptions { codec?: BlockCodec diff --git a/packages/unixfs/src/commands/utils/remove-link.ts b/packages/unixfs/src/commands/utils/remove-link.ts index f74e2c04..a60f6412 100644 --- a/packages/unixfs/src/commands/utils/remove-link.ts +++ b/packages/unixfs/src/commands/utils/remove-link.ts @@ -1,21 +1,21 @@ import * as dagPB from '@ipld/dag-pb' -import type { CID, Version } from 'multiformats/cid' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' +import { exporter } from 'ipfs-unixfs-exporter' +import { InvalidParametersError, InvalidPBNodeError } from './errors.js' import { recreateShardedDirectory, - UpdateHamtDirectoryOptions, + type UpdateHamtDirectoryOptions, updateShardedDirectory } from './hamt-utils.js' -import type { PBNode } from '@ipld/dag-pb' -import type { Blocks } from '@helia/interface/blocks' +import { isOverShardThreshold } from './is-over-shard-threshold.js' +import { persist } from './persist.js' import type { Directory } from './cid-to-directory.js' +import type { Blocks } from '@helia/interface/blocks' +import type { PBNode } from '@ipld/dag-pb' import type { AbortOptions } from '@libp2p/interfaces' -import { InvalidParametersError, InvalidPBNodeError } from './errors.js' -import { exporter } from 'ipfs-unixfs-exporter' -import { persist } from './persist.js' -import { isOverShardThreshold } from './is-over-shard-threshold.js' +import type { CID, Version } from 'multiformats/cid' const log = logger('helia:unixfs:utils:remove-link') @@ -44,7 +44,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) { log('converting shard to flat directory %c', parent.cid) - return await convertToFlatDirectory(result, blockstore, options) + return convertToFlatDirectory(result, blockstore, options) } return result @@ -52,7 +52,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B log(`removing link ${name} regular directory`) - return await removeFromDirectory(parent, name, blockstore, options) + return removeFromDirectory(parent, name, blockstore, options) } const removeFromDirectory = async (parent: Directory, name: string, blockstore: Blocks, options: AbortOptions): Promise => { @@ -129,7 +129,7 @@ const removeFromShardedDirectory = async (parent: Directory, name: string, block } } - return await updateShardedDirectory(path, blockstore, options) + return updateShardedDirectory(path, blockstore, options) } const convertToFlatDirectory = async (parent: Directory, blockstore: Blocks, options: RmLinkOptions): Promise => { diff --git a/packages/unixfs/src/commands/utils/resolve.ts b/packages/unixfs/src/commands/utils/resolve.ts index 08094fb6..a2391d93 100644 --- a/packages/unixfs/src/commands/utils/resolve.ts +++ b/packages/unixfs/src/commands/utils/resolve.ts @@ -1,12 +1,12 @@ -import type { CID } from 'multiformats/cid' -import { exporter } from 'ipfs-unixfs-exporter' -import type { AbortOptions } from '@libp2p/interfaces' import { logger } from '@libp2p/logger' -import { DoesNotExistError, InvalidParametersError } from './errors.js' +import { exporter } from 'ipfs-unixfs-exporter' import { addLink } from './add-link.js' import { cidToDirectory } from './cid-to-directory.js' import { cidToPBLink } from './cid-to-pblink.js' +import { DoesNotExistError, InvalidParametersError } from './errors.js' import type { Blocks } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { CID } from 'multiformats/cid' const log = logger('helia:unixfs:components:utils:resolve') diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index 2f71c2a8..f2fc4855 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -31,21 +31,21 @@ * ``` */ -import type { CID, Version } from 'multiformats/cid' -import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' -import type { AbortOptions } from '@libp2p/interfaces' import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js' import { cat } from './commands/cat.js' -import { mkdir } from './commands/mkdir.js' -import type { Mtime } from 'ipfs-unixfs' +import { chmod } from './commands/chmod.js' import { cp } from './commands/cp.js' +import { ls } from './commands/ls.js' +import { mkdir } from './commands/mkdir.js' import { rm } from './commands/rm.js' import { stat } from './commands/stat.js' import { touch } from './commands/touch.js' -import { chmod } from './commands/chmod.js' +import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' +import type { AbortOptions } from '@libp2p/interfaces' +import type { Mtime, UnixFS as IPFSUnixFS } from 'ipfs-unixfs' import type { ExporterProgressEvents, UnixFSEntry } from 'ipfs-unixfs-exporter' -import { ls } from './commands/ls.js' import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImporterProgressEvents, ImportResult } from 'ipfs-unixfs-importer' +import type { CID, Version } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' export interface UnixFSComponents { @@ -248,7 +248,7 @@ export interface UnixFSStats { * UnixFS metadata about this file or directory. Will not be present * if the node is a `raw` type. */ - unixfs?: import('ipfs-unixfs').UnixFS + unixfs?: IPFSUnixFS } /** @@ -505,19 +505,19 @@ class DefaultUnixFS implements UnixFS { } async addBytes (bytes: Uint8Array, options: Partial = {}): Promise { - return await addBytes(bytes, this.components.blockstore, options) + return addBytes(bytes, this.components.blockstore, options) } async addByteStream (bytes: ByteStream, options: Partial = {}): Promise { - return await addByteStream(bytes, this.components.blockstore, options) + return addByteStream(bytes, this.components.blockstore, options) } async addFile (file: FileCandidate, options: Partial = {}): Promise { - return await addFile(file, this.components.blockstore, options) + return addFile(file, this.components.blockstore, options) } async addDirectory (dir: Partial = {}, options: Partial = {}): Promise { - return await addDirectory(dir, this.components.blockstore, options) + return addDirectory(dir, this.components.blockstore, options) } async * cat (cid: CID, options: Partial = {}): AsyncIterable { @@ -525,11 +525,11 @@ class DefaultUnixFS implements UnixFS { } async chmod (cid: CID, mode: number, options: Partial = {}): Promise { - return await chmod(cid, mode, this.components.blockstore, options) + return chmod(cid, mode, this.components.blockstore, options) } async cp (source: CID, target: CID, name: string, options: Partial = {}): Promise { - return await cp(source, target, name, this.components.blockstore, options) + return cp(source, target, name, this.components.blockstore, options) } async * ls (cid: CID, options: Partial = {}): AsyncIterable { @@ -537,19 +537,19 @@ class DefaultUnixFS implements UnixFS { } async mkdir (cid: CID, dirname: string, options: Partial = {}): Promise { - return await mkdir(cid, dirname, this.components.blockstore, options) + return mkdir(cid, dirname, this.components.blockstore, options) } async rm (cid: CID, path: string, options: Partial = {}): Promise { - return await rm(cid, path, this.components.blockstore, options) + return rm(cid, path, this.components.blockstore, options) } async stat (cid: CID, options: Partial = {}): Promise { - return await stat(cid, this.components.blockstore, options) + return stat(cid, this.components.blockstore, options) } async touch (cid: CID, options: Partial = {}): Promise { - return await touch(cid, this.components.blockstore, options) + return touch(cid, this.components.blockstore, options) } } diff --git a/packages/unixfs/test/add.spec.ts b/packages/unixfs/test/add.spec.ts index ebc4ba84..384f8819 100644 --- a/packages/unixfs/test/add.spec.ts +++ b/packages/unixfs/test/add.spec.ts @@ -1,10 +1,10 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' import all from 'it-all' +import { unixfs, type UnixFS } from '../src/index.js' +import type { Blockstore } from 'interface-blockstore' describe('addAll', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/cat.spec.ts b/packages/unixfs/test/cat.spec.ts index 30b5d482..0c0946a6 100644 --- a/packages/unixfs/test/cat.spec.ts +++ b/packages/unixfs/test/cat.spec.ts @@ -1,14 +1,14 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' -import type { CID } from 'multiformats/cid' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' -import toBuffer from 'it-to-buffer' import drain from 'it-drain' +import toBuffer from 'it-to-buffer' +import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' describe('cat', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/chmod.spec.ts b/packages/unixfs/test/chmod.spec.ts index e76db519..a584cfd6 100644 --- a/packages/unixfs/test/chmod.spec.ts +++ b/packages/unixfs/test/chmod.spec.ts @@ -1,12 +1,12 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' -import type { Blockstore } from 'interface-blockstore' import { MemoryBlockstore } from 'blockstore-core' -import { UnixFS, unixfs } from '../src/index.js' -import type { CID } from 'multiformats/cid' +import { type UnixFS, unixfs } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' describe('chmod', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/cp.spec.ts b/packages/unixfs/test/cp.spec.ts index 51fc9e9e..4527fafc 100644 --- a/packages/unixfs/test/cp.spec.ts +++ b/packages/unixfs/test/cp.spec.ts @@ -1,16 +1,16 @@ /* eslint-env mocha */ -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { expect } from 'aegir/chai' -import { identity } from 'multiformats/hashes/identity' -import { CID } from 'multiformats/cid' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' +import first from 'it-first' import toBuffer from 'it-to-buffer' +import { CID } from 'multiformats/cid' +import { identity } from 'multiformats/hashes/identity' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' -import first from 'it-first' import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' +import type { Blockstore } from 'interface-blockstore' describe('cp', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/fixtures/create-sharded-directory.ts b/packages/unixfs/test/fixtures/create-sharded-directory.ts index 0ba0d1b6..3c8fd627 100644 --- a/packages/unixfs/test/fixtures/create-sharded-directory.ts +++ b/packages/unixfs/test/fixtures/create-sharded-directory.ts @@ -1,8 +1,8 @@ import { expect } from 'aegir/chai' -import last from 'it-last' import { importer } from 'ipfs-unixfs-importer' -import type { CID } from 'multiformats/cid' +import last from 'it-last' import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' export async function createShardedDirectory (blockstore: Blockstore, files = 1001): Promise { const result = await last(importer((function * () { diff --git a/packages/unixfs/test/fixtures/create-subsharded-directory.ts b/packages/unixfs/test/fixtures/create-subsharded-directory.ts index 1ae1561c..697623a0 100644 --- a/packages/unixfs/test/fixtures/create-subsharded-directory.ts +++ b/packages/unixfs/test/fixtures/create-subsharded-directory.ts @@ -1,9 +1,9 @@ -import type { Blockstore } from 'interface-blockstore' -import { importer } from 'ipfs-unixfs-importer' -import type { CID } from 'multiformats/cid' -import { unixfs } from '../../src/index.js' import * as dagPb from '@ipld/dag-pb' +import { importer } from 'ipfs-unixfs-importer' import last from 'it-last' +import { unixfs } from '../../src/index.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' export async function createSubshardedDirectory (blockstore: Blockstore, depth: number = 1, files: number = 5000): Promise<{ importerCid: CID diff --git a/packages/unixfs/test/fixtures/print-tree.ts b/packages/unixfs/test/fixtures/print-tree.ts index 72c0aa33..e41ac6ab 100644 --- a/packages/unixfs/test/fixtures/print-tree.ts +++ b/packages/unixfs/test/fixtures/print-tree.ts @@ -1,8 +1,8 @@ -import type { Blockstore } from 'interface-blockstore' -import { UnixFS } from 'ipfs-unixfs' -import type { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' +import { UnixFS } from 'ipfs-unixfs' import * as raw from 'multiformats/codecs/raw' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' type GetStore = Pick diff --git a/packages/unixfs/test/ls.spec.ts b/packages/unixfs/test/ls.spec.ts index 84ad860b..8c626101 100644 --- a/packages/unixfs/test/ls.spec.ts +++ b/packages/unixfs/test/ls.spec.ts @@ -1,12 +1,12 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' -import type { CID } from 'multiformats/cid' -import all from 'it-all' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' +import all from 'it-all' +import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' describe('ls', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/mkdir.spec.ts b/packages/unixfs/test/mkdir.spec.ts index 5d402fd1..a2cf6de5 100644 --- a/packages/unixfs/test/mkdir.spec.ts +++ b/packages/unixfs/test/mkdir.spec.ts @@ -1,13 +1,13 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' import all from 'it-all' +import { unixfs, type UnixFS } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' -import { MemoryBlockstore } from 'blockstore-core' -import type { CID } from 'multiformats/cid' import type { Mtime } from 'ipfs-unixfs' -import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import type { CID } from 'multiformats/cid' describe('mkdir', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/rm.spec.ts b/packages/unixfs/test/rm.spec.ts index 5cfa84fe..181b2dcd 100644 --- a/packages/unixfs/test/rm.spec.ts +++ b/packages/unixfs/test/rm.spec.ts @@ -1,15 +1,15 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' -import type { CID } from 'multiformats/cid' import { importer } from 'ipfs-unixfs-importer' -import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import last from 'it-last' +import { unixfs, type UnixFS } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' import { smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' describe('rm', () => { let blockstore: Blockstore diff --git a/packages/unixfs/test/stat.spec.ts b/packages/unixfs/test/stat.spec.ts index 32aa7478..a87a3bec 100644 --- a/packages/unixfs/test/stat.spec.ts +++ b/packages/unixfs/test/stat.spec.ts @@ -1,13 +1,13 @@ /* eslint-env mocha */ +import * as dagPb from '@ipld/dag-pb' import { expect } from 'aegir/chai' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' -import type { CID } from 'multiformats/cid' -import * as dagPb from '@ipld/dag-pb' +import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { largeFile, smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' describe('stat', function () { this.timeout(120 * 1000) diff --git a/packages/unixfs/test/touch.spec.ts b/packages/unixfs/test/touch.spec.ts index 0ccbead8..8b9e4ab9 100644 --- a/packages/unixfs/test/touch.spec.ts +++ b/packages/unixfs/test/touch.spec.ts @@ -1,12 +1,12 @@ /* eslint-env mocha */ import { expect } from 'aegir/chai' -import type { Blockstore } from 'interface-blockstore' -import { unixfs, UnixFS } from '../src/index.js' import { MemoryBlockstore } from 'blockstore-core' -import type { CID } from 'multiformats/cid' import delay from 'delay' +import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' describe('.files.touch', () => { let blockstore: Blockstore From 312381cd9d31500651a6c35043460ef3bce0fd67 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 25 May 2023 16:45:02 +0100 Subject: [PATCH 085/253] test: add tests for different key types, where possible (#52) Expands the tests to test using `secp256k1` and `RSA` keys as well as `Ed25519` where possible. Kubo doesn't support generating or importing `secp256k1` keys so those tests are skipped, as are `RSA` keys for pubsub resolution since the DHT needs to be enabled to resolve the public key component. --- packages/interop/test/dht.spec.ts | 271 +++++++++++--------- packages/interop/test/fixtures/key-types.ts | 7 + packages/interop/test/pubsub.spec.ts | 263 +++++++++---------- 3 files changed, 284 insertions(+), 257 deletions(-) create mode 100644 packages/interop/test/fixtures/key-types.ts diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index 638d15c0..5dbff01c 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -3,7 +3,7 @@ import { ipns } from '@helia/ipns' import { dht } from '@helia/ipns/routing' import { type KadDHT, kadDHT } from '@libp2p/kad-dht' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { createEd25519PeerId, createRSAPeerId, createSecp256k1PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' @@ -19,6 +19,7 @@ import { connect } from './fixtures/connect.js' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import { sortClosestPeers } from './fixtures/create-peer-ids.js' +import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' @@ -26,163 +27,177 @@ import type { Libp2p } from '@libp2p/interface-libp2p' import type { Controller } from 'ipfsd-ctl' import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' -describe('dht routing', () => { - let helia: Helia> - let kubo: Controller - let name: IPNS - - // the CID we are going to publish - let value: CID - - // the public key we will use to publish the value - let key: PeerId - - /** - * Ensure that for the CID we are going to publish, the resolver has a peer ID that - * is KAD-closer to the routing key so we can predict the the resolver will receive - * the DHT record containing the IPNS record - */ - async function createNodes (resolver: 'kubo' | 'helia'): Promise { - const input = Uint8Array.from([0, 1, 2, 3, 4]) - const digest = await sha256.digest(input) - value = CID.createV1(raw.code, digest) - - helia = await createHeliaNode({ - services: { - identify: identifyService(), - dht: kadDHT({ - validators: { - ipns: ipnsValidator - }, - selectors: { - ipns: ipnsSelector - }, - // skips waiting for the initial self-query to find peers - allowQueryWithZeroPeers: true - }) - } - }) - kubo = await createKuboNode() - - // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key - while (true) { - key = await createEd25519PeerId() - const routingKey = uint8ArrayConcat([ - uint8ArrayFromString('/ipns/'), - key.toBytes() - ]) +keyTypes.forEach(type => { + describe(`dht routing with ${type} keys`, () => { + let helia: Helia> + let kubo: Controller + let name: IPNS + + // the CID we are going to publish + let value: CID + + // the public key we will use to publish the value + let key: PeerId + + /** + * Ensure that for the CID we are going to publish, the resolver has a peer ID that + * is KAD-closer to the routing key so we can predict the the resolver will receive + * the DHT record containing the IPNS record + */ + async function createNodes (resolver: 'kubo' | 'helia'): Promise { + const input = Uint8Array.from([0, 1, 2, 3, 4]) + const digest = await sha256.digest(input) + value = CID.createV1(raw.code, digest) + + helia = await createHeliaNode({ + services: { + identify: identifyService(), + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + }, + // skips waiting for the initial self-query to find peers + allowQueryWithZeroPeers: true + }) + } + }) + kubo = await createKuboNode() + + // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key + while (true) { + if (type === 'Ed25519') { + key = await createEd25519PeerId() + } else if (type === 'secp256k1') { + key = await createSecp256k1PeerId() + } else { + key = await createRSAPeerId() + } - const [closest] = await sortClosestPeers(routingKey, [ - helia.libp2p.peerId, - kubo.peer.id - ]) + const routingKey = uint8ArrayConcat([ + uint8ArrayFromString('/ipns/'), + key.toBytes() + ]) - if (resolver === 'kubo' && closest.equals(kubo.peer.id)) { - break - } + const [closest] = await sortClosestPeers(routingKey, [ + helia.libp2p.peerId, + kubo.peer.id + ]) + + if (resolver === 'kubo' && closest.equals(kubo.peer.id)) { + break + } - if (resolver === 'helia' && closest.equals(helia.libp2p.peerId)) { - break + if (resolver === 'helia' && closest.equals(helia.libp2p.peerId)) { + break + } } - } - // connect the two nodes over the KAD-DHT protocol, this should ensure - // both nodes have each other in their KAD buckets - await connect(helia, kubo, '/ipfs/lan/kad/1.0.0') + // connect the two nodes over the KAD-DHT protocol, this should ensure + // both nodes have each other in their KAD buckets + await connect(helia, kubo, '/ipfs/lan/kad/1.0.0') - await waitFor(async () => { - let found = false + await waitFor(async () => { + let found = false - for await (const event of helia.libp2p.services.dht.findPeer(kubo.peer.id)) { - if (event.name === 'FINAL_PEER') { - found = true + for await (const event of helia.libp2p.services.dht.findPeer(kubo.peer.id)) { + if (event.name === 'FINAL_PEER') { + found = true + } } - } - return found - }, { - timeout: 30000, - delay: 1000, - message: 'Helia could not find Kubo on the DHT' - }) + return found + }, { + timeout: 30000, + delay: 1000, + message: 'Helia could not find Kubo on the DHT' + }) - await waitFor(async () => { - let found = false + await waitFor(async () => { + let found = false - for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) { - if (event.name === 'FINAL_PEER') { - found = true + for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) { + if (event.name === 'FINAL_PEER') { + found = true + } } + + return found + }, { + timeout: 30000, + delay: 1000, + message: 'Kubo could not find Helia on the DHT' + }) + + name = ipns(helia, [ + dht(helia) + ]) + } + + afterEach(async () => { + if (helia != null) { + await helia.stop() } - return found - }, { - timeout: 30000, - delay: 1000, - message: 'Kubo could not find Helia on the DHT' + if (kubo != null) { + await kubo.stop() + } }) - name = ipns(helia, [ - dht(helia) - ]) - } - - afterEach(async () => { - if (helia != null) { - await helia.stop() - } + it(`should publish on helia and resolve on kubo using a ${type} key`, async () => { + await createNodes('kubo') - if (kubo != null) { - await kubo.stop() - } - }) + const keyName = 'my-ipns-key' + await helia.libp2p.keychain.importPeer(keyName, key) - it('should publish on helia and resolve on kubo', async () => { - await createNodes('kubo') + await name.publish(key, value) - const keyName = 'my-ipns-key' - await helia.libp2p.keychain.importPeer(keyName, key) + const resolved = await last(kubo.api.name.resolve(key)) - await name.publish(key, value) + if (resolved == null) { + throw new Error('kubo failed to resolve name') + } - const resolved = await last(kubo.api.name.resolve(key)) + expect(resolved).to.equal(`/ipfs/${value.toString()}`) + }) - if (resolved == null) { - throw new Error('kubo failed to resolve name') - } + it('should publish on kubo and resolve on helia', async function () { + if (isElectronMain) { + // electron main does not have fetch, FormData or Blob APIs + // can revisit when kubo-rpc-client supports the key.import API + return this.skip() + } - expect(resolved).to.equal(`/ipfs/${value.toString()}`) - }) + if (type === 'secp256k1') { + // Kubo cannot import secp256k1 keys + return this.skip() + } - it('should publish on kubo and resolve on helia', async function () { - if (isElectronMain) { - // electron main does not have fetch, FormData or Blob APIs - // can revisit when kubo-rpc-client supports the key.import API - return this.skip() - } + await createNodes('helia') - await createNodes('helia') + const keyName = 'my-ipns-key' + const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) - const keyName = 'my-ipns-key' - const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) + // ensure the key is in the kubo keychain so we can use it to publish the IPNS record + const body = new FormData() + body.append('key', new Blob([key.privateKey ?? new Uint8Array(0)])) - // ensure the key is in the kubo keychain so we can use it to publish the IPNS record - const body = new FormData() - body.append('key', new Blob([key.privateKey ?? new Uint8Array(0)])) + // can't use the kubo-rpc-api for this call yet + const response = await fetch(`http://${kubo.api.apiHost}:${kubo.api.apiPort}/api/v0/key/import?arg=${keyName}`, { + method: 'POST', + body + }) - // can't use the kubo-rpc-api for this call yet - const response = await fetch(`http://${kubo.api.apiHost}:${kubo.api.apiPort}/api/v0/key/import?arg=${keyName}`, { - method: 'POST', - body - }) + expect(response).to.have.property('status', 200) - expect(response).to.have.property('status', 200) + await kubo.api.name.publish(cid, { + key: keyName + }) - await kubo.api.name.publish(cid, { - key: keyName + const resolvedCid = await name.resolve(key) + expect(resolvedCid.toString()).to.equal(cid.toString()) }) - - const resolvedCid = await name.resolve(key) - expect(resolvedCid.toString()).to.equal(cid.toString()) }) }) diff --git a/packages/interop/test/fixtures/key-types.ts b/packages/interop/test/fixtures/key-types.ts new file mode 100644 index 00000000..78b60e33 --- /dev/null +++ b/packages/interop/test/fixtures/key-types.ts @@ -0,0 +1,7 @@ +import type { PeerIdType } from '@libp2p/interface-peer-id' + +export const keyTypes: PeerIdType[] = [ + 'Ed25519', + 'secp256k1', + 'RSA' +] diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index 5e315736..9180d35d 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -1,4 +1,5 @@ /* eslint-env mocha */ +/* eslint max-nested-callbacks: ["error", 5] */ import { gossipsub } from '@chainsafe/libp2p-gossipsub' import { ipns } from '@helia/ipns' @@ -18,6 +19,7 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { connect } from './fixtures/connect.js' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' +import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' @@ -27,156 +29,159 @@ import type { Libp2p } from 'libp2p' const LIBP2P_KEY_CODEC = 0x72 -describe('pubsub routing', () => { - let helia: Helia> - let kubo: Controller - let name: IPNS - - beforeEach(async () => { - helia = await createHeliaNode({ - services: { - identify: identifyService(), - pubsub: gossipsub() - } - }) - kubo = await createKuboNode({ - ipfsOptions: { - config: { - Routing: { - Type: 'none' - } +// skip RSA tests because we need the DHT enabled to find the public key +// component of the keypair, but that means we can't test pubsub +// resolution because Kubo will use the DHT as well +keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { + describe(`pubsub routing with ${keyType} keys`, () => { + let helia: Helia> + let kubo: Controller + let name: IPNS + + beforeEach(async () => { + helia = await createHeliaNode({ + services: { + identify: identifyService(), + pubsub: gossipsub() } - }, - args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] - }) - - // connect the two nodes - await connect(helia, kubo, '/meshsub/1.1.0') - - name = ipns(helia, [ - pubsub(helia) - ]) - }) + }) + kubo = await createKuboNode({ + args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] + }) - afterEach(async () => { - if (helia != null) { - await helia.stop() - } + // connect the two nodes + await connect(helia, kubo, '/meshsub/1.1.0') - if (kubo != null) { - await kubo.stop() - } - }) - - it('should publish on helia and resolve on kubo', async () => { - const input = Uint8Array.from([0, 1, 2, 3, 4]) - const digest = await sha256.digest(input) - const cid = CID.createV1(raw.code, digest) - - const keyName = 'my-ipns-key' - await helia.libp2p.keychain.createKey(keyName, 'Ed25519') - const peerId = await helia.libp2p.keychain.exportPeerId(keyName) - - if (peerId.publicKey == null) { - throw new Error('No public key present') - } + name = ipns(helia, [ + pubsub(helia) + ]) + }) - // first publish should fail because kubo isn't subscribed to key update channel - await expect(name.publish(peerId, cid)).to.eventually.be.rejected() - .with.property('message', 'PublishError.InsufficientPeers') + afterEach(async () => { + if (helia != null) { + await helia.stop() + } - // should fail to resolve the first time as kubo was not subscribed to the pubsub channel - await expect(last(kubo.api.name.resolve(peerId, { - timeout: 100 - }))).to.eventually.be.undefined() + if (kubo != null) { + await kubo.stop() + } + }) - // magic pubsub subscription name - const subscriptionName = `/ipns/${CID.createV1(LIBP2P_KEY_CODEC, identity.digest(peerId.publicKey)).toString(base36)}` + it('should publish on helia and resolve on kubo', async () => { + const input = Uint8Array.from([0, 1, 2, 3, 4]) + const digest = await sha256.digest(input) + const cid = CID.createV1(raw.code, digest) - // wait for kubo to be subscribed to updates - await waitFor(async () => { - const subs = await kubo.api.name.pubsub.subs() + const keyName = 'my-ipns-key' + await helia.libp2p.keychain.createKey(keyName, keyType) + const peerId = await helia.libp2p.keychain.exportPeerId(keyName) - return subs.includes(subscriptionName) - }, { - timeout: 30000 - }) + if (peerId.publicKey == null) { + throw new Error('No public key present') + } - // publish should now succeed - await name.publish(peerId, cid) + // first publish should fail because kubo isn't subscribed to key update channel + await expect(name.publish(peerId, cid)).to.eventually.be.rejected() + .with.property('message', 'PublishError.InsufficientPeers') - // kubo should now be able to resolve IPNS name - const resolved = await last(kubo.api.name.resolve(peerId, { - timeout: 100 - })) + // should fail to resolve the first time as kubo was not subscribed to the pubsub channel + await expect(last(kubo.api.name.resolve(peerId, { + timeout: 100 + }))).to.eventually.be.undefined() - expect(resolved).to.equal(`/ipfs/${cid.toString()}`) - }) + // magic pubsub subscription name + const subscriptionName = `/ipns/${CID.createV1(LIBP2P_KEY_CODEC, identity.digest(peerId.publicKey)).toString(base36)}` - it('should publish on kubo and resolve on helia', async () => { - const keyName = 'my-ipns-key' - const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) - const result = await kubo.api.key.gen(keyName, { - // @ts-expect-error kubo needs this in lower case - type: 'ed25519' - }) + // wait for kubo to be subscribed to updates + await waitFor(async () => { + const subs = await kubo.api.name.pubsub.subs() - // the generated id is libp2p-key CID with the public key as an identity multihash - const peerCid = CID.parse(result.id, base36) - const peerId = await peerIdFromKeys(peerCid.multihash.digest) - - // first call to pubsub resolver should fail but we should now be subscribed for updates - await expect(name.resolve(peerId)).to.eventually.be.rejected() - - // actual pubsub subscription name - const subscriptionName = `/record/${uint8ArrayToString(uint8ArrayConcat([ - uint8ArrayFromString('/ipns/'), - peerId.toBytes() - ]), 'base64url')}` - - // wait for helia to be subscribed to the topic for record updates - await waitFor(async () => { - return helia.libp2p.services.pubsub.getTopics().includes(subscriptionName) - }, { - timeout: 30000, - message: 'Helia did not register for record updates' - }) + return subs.includes(subscriptionName) + }, { + timeout: 30000 + }) - // wait for kubo to see that helia is subscribed to the topic for record updates - await waitFor(async () => { - const peers = await kubo.api.pubsub.peers(subscriptionName) + // publish should now succeed + await name.publish(peerId, cid) - return peers.map(p => p.toString()).includes(helia.libp2p.peerId.toString()) - }, { - timeout: 30000, - message: 'Kubo did not see that Helia was registered for record updates' - }) + // kubo should now be able to resolve IPNS name + const resolved = await last(kubo.api.name.resolve(peerId, { + timeout: 100 + })) - // now publish, this should cause a pubsub message on the topic for record updates - await kubo.api.name.publish(cid, { - key: keyName + expect(resolved).to.equal(`/ipfs/${cid.toString()}`) }) - let resolvedCid: CID | undefined + it('should publish on kubo and resolve on helia', async function () { + if (keyType === 'secp256k1') { + // Kubo cannot generate secp256k1 keys + return this.skip() + } - // we should get an update eventually - await waitFor(async () => { - try { - resolvedCid = await name.resolve(peerId) + const keyName = 'my-ipns-key' + const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) + const result = await kubo.api.key.gen(keyName, { + // @ts-expect-error kubo needs this in lower case + type: keyType.toLowerCase() + }) + + // the generated id is libp2p-key CID with the public key as an identity multihash + const peerCid = CID.parse(result.id, base36) + const peerId = await peerIdFromKeys(peerCid.multihash.digest) + + // first call to pubsub resolver should fail but we should now be subscribed for updates + await expect(name.resolve(peerId)).to.eventually.be.rejected() + + // actual pubsub subscription name + const subscriptionName = `/record/${uint8ArrayToString(uint8ArrayConcat([ + uint8ArrayFromString('/ipns/'), + peerId.toBytes() + ]), 'base64url')}` + + // wait for helia to be subscribed to the topic for record updates + await waitFor(async () => { + return helia.libp2p.services.pubsub.getTopics().includes(subscriptionName) + }, { + timeout: 30000, + message: 'Helia did not register for record updates' + }) + + // wait for kubo to see that helia is subscribed to the topic for record updates + await waitFor(async () => { + const peers = await kubo.api.pubsub.peers(subscriptionName) + + return peers.map(p => p.toString()).includes(helia.libp2p.peerId.toString()) + }, { + timeout: 30000, + message: 'Kubo did not see that Helia was registered for record updates' + }) + + // now publish, this should cause a pubsub message on the topic for record updates + await kubo.api.name.publish(cid, { + key: keyName + }) + + let resolvedCid: CID | undefined + + // we should get an update eventually + await waitFor(async () => { + try { + resolvedCid = await name.resolve(peerId) + + return true + } catch { + return false + } + }, { + timeout: 10000, + message: 'Helia could not resolve the IPNS record' + }) - return true - } catch { - return false + if (resolvedCid == null) { + throw new Error('Failed to resolve CID') } - }, { - timeout: 10000, - message: 'Helia could not resolve the IPNS record' - }) - - if (resolvedCid == null) { - throw new Error('Failed to resolve CID') - } - expect(resolvedCid.toString()).to.equal(cid.toString()) + expect(resolvedCid.toString()).to.equal(cid.toString()) + }) }) }) From 3d3627e92ee99446cd0eaf69de84187ceee653e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 18:03:08 +0100 Subject: [PATCH 086/253] deps(dev): bump libp2p from 0.44.0 to 0.45.3 (#13) * deps(dev): bump libp2p from 0.44.0 to 0.45.3 Bumps [libp2p](https://github.com/libp2p/js-libp2p) from 0.44.0 to 0.45.3. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/CHANGELOG.md) - [Commits](https://github.com/libp2p/js-libp2p/compare/v0.44.0...v0.45.3) --- updated-dependencies: - dependency-name: libp2p dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: fix deps and linting * chore: add identify * chore: revert * chore: revert * chore: debug browsers * chore: remove debug --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 8 ++++---- .../interop/test/fixtures/create-helia.browser.ts | 15 ++++++++++----- packages/interop/test/fixtures/create-helia.ts | 5 +++-- packages/interop/test/strings.spec.ts | 3 ++- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 87a9446e..99bdd12f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,21 +52,21 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/interface": "^1.0.0", "@helia/strings": "^1.0.0", - "@libp2p/tcp": "^6.1.2", + "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", "aegir": "^39.0.3", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.19.0", + "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.44.0", + "libp2p": "^0.45.3", "merge-options": "^3.0.4", "uint8arrays": "^4.0.3", "wherearewe": "^2.0.1" diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 6baa5299..867de492 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -5,10 +5,11 @@ import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -25,10 +26,14 @@ export async function createHeliaNode (): Promise { streamMuxers: [ yamux() ], + services: { + identify: identifyService() + }, datastore, - nat: { - enabled: false - } + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config }) const helia = await createHelia({ diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index c92d309b..f7a25c3f 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -22,8 +23,8 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise { str = strings(helia) kubo = await createKuboNode() - await helia.libp2p.dial((await (kubo.api.id())).addresses) + const id = await kubo.api.id() + await helia.libp2p.dial(id.addresses) }) afterEach(async () => { From f70bbcbd6d7f3175e0d96fdd64aa745b79ca10c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 18:11:06 +0100 Subject: [PATCH 087/253] deps(dev): bump libp2p from 0.44.0 to 0.45.3 (#7) * deps(dev): bump libp2p from 0.44.0 to 0.45.3 Bumps [libp2p](https://github.com/libp2p/js-libp2p) from 0.44.0 to 0.45.3. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/CHANGELOG.md) - [Commits](https://github.com/libp2p/js-libp2p/compare/v0.44.0...v0.45.3) --- updated-dependencies: - dependency-name: libp2p dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: update deps and fix linting --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 6 +++--- .../interop/test/fixtures/create-helia.browser.ts | 15 ++++++++++----- packages/interop/test/fixtures/create-helia.ts | 5 +++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 8f48336f..406403dc 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/interface": "^1.0.0", "@helia/json": "^1.0.0", @@ -61,12 +61,12 @@ "aegir": "^39.0.3", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.19.0", + "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.44.0", + "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.2", "wherearewe": "^2.0.1" diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 6baa5299..867de492 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -5,10 +5,11 @@ import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -25,10 +26,14 @@ export async function createHeliaNode (): Promise { streamMuxers: [ yamux() ], + services: { + identify: identifyService() + }, datastore, - nat: { - enabled: false - } + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config }) const helia = await createHelia({ diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index c92d309b..f7a25c3f 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -22,8 +23,8 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise Date: Thu, 25 May 2023 18:11:28 +0100 Subject: [PATCH 088/253] deps(dev): bump libp2p from 0.44.0 to 0.45.3 (#7) * deps(dev): bump libp2p from 0.44.0 to 0.45.3 Bumps [libp2p](https://github.com/libp2p/js-libp2p) from 0.44.0 to 0.45.3. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/CHANGELOG.md) - [Commits](https://github.com/libp2p/js-libp2p/compare/v0.44.0...v0.45.3) --- updated-dependencies: - dependency-name: libp2p dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: update deps and linting --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 6 +++--- .../interop/test/fixtures/create-helia.browser.ts | 15 ++++++++++----- packages/interop/test/fixtures/create-helia.ts | 5 +++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index ca450fbe..c6b01fb4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/dag-json": "^1.0.0", "@helia/interface": "^1.0.0", @@ -61,12 +61,12 @@ "aegir": "^39.0.3", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.19.0", + "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.44.0", + "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.2", "wherearewe": "^2.0.1" diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 6baa5299..867de492 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -5,10 +5,11 @@ import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -25,10 +26,14 @@ export async function createHeliaNode (): Promise { streamMuxers: [ yamux() ], + services: { + identify: identifyService() + }, datastore, - nat: { - enabled: false - } + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config }) const helia = await createHelia({ diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index c92d309b..f7a25c3f 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -22,8 +23,8 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise Date: Thu, 25 May 2023 18:12:15 +0100 Subject: [PATCH 089/253] deps(dev): bump libp2p from 0.44.0 to 0.45.3 (#6) * deps(dev): bump libp2p from 0.44.0 to 0.45.3 Bumps [libp2p](https://github.com/libp2p/js-libp2p) from 0.44.0 to 0.45.3. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/CHANGELOG.md) - [Commits](https://github.com/libp2p/js-libp2p/compare/v0.44.0...v0.45.3) --- updated-dependencies: - dependency-name: libp2p dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: update deps and fix linting --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 6 +++--- .../interop/test/fixtures/create-helia.browser.ts | 15 ++++++++++----- packages/interop/test/fixtures/create-helia.ts | 5 +++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index f9f31332..50e05cca 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/dag-cbor": "~0.0.0", "@helia/interface": "^1.0.0", @@ -62,12 +62,12 @@ "aegir": "^39.0.3", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.19.0", + "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.44.0", + "libp2p": "^0.45.3", "merge-options": "^3.0.4", "wherearewe": "^2.0.1" }, diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 6baa5299..867de492 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -5,10 +5,11 @@ import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -25,10 +26,14 @@ export async function createHeliaNode (): Promise { streamMuxers: [ yamux() ], + services: { + identify: identifyService() + }, datastore, - nat: { - enabled: false - } + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config }) const helia = await createHelia({ diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index c92d309b..f7a25c3f 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -22,8 +23,8 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise Date: Wed, 7 Jun 2023 09:50:46 +0000 Subject: [PATCH 090/253] chore(release): 1.2.3 [skip ci] ## [@helia/unixfs-v1.2.3](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.2...@helia/unixfs-v1.2.3) (2023-06-07) ### Dependencies * update all deps and fix linting ([d4d6515](https://github.com/ipfs/helia-unixfs/commit/d4d6515f023db339874d34871e69fb7c3fc47f6c)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 7220d239..4682a4b4 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.2.3](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.2...@helia/unixfs-v1.2.3) (2023-06-07) + + +### Dependencies + +* update all deps and fix linting ([d4d6515](https://github.com/ipfs/helia-unixfs/commit/d4d6515f023db339874d34871e69fb7c3fc47f6c)) + ## [@helia/unixfs-v1.2.2](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.1...@helia/unixfs-v1.2.2) (2023-04-12) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 0da91828..ddb00c1e 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.2.2", + "version": "1.2.3", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 95e68a12ac7f829b7aa455b571f942dfc82394ed Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 7 Jun 2023 10:50:57 +0100 Subject: [PATCH 091/253] feat: initial import --- .github/dependabot.yml | 11 + .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 181 +++++++++++++ .github/workflows/stale.yml | 26 ++ .gitignore | 9 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ README.md | 49 ++++ package.json | 44 ++++ packages/car/LICENSE | 4 + packages/car/LICENSE-APACHE | 5 + packages/car/LICENSE-MIT | 19 ++ packages/car/README.md | 59 +++++ packages/car/package.json | 181 +++++++++++++ packages/car/src/index.ts | 239 ++++++++++++++++++ packages/car/src/utils/dag-walkers.ts | 176 +++++++++++++ packages/car/test/fixtures/files.ts | 5 + packages/car/test/fixtures/memory-car.ts | 33 +++ packages/car/test/index.spec.ts | 121 +++++++++ packages/car/tsconfig.json | 10 + packages/interop/.aegir.js | 45 ++++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 59 +++++ packages/interop/package.json | 82 ++++++ packages/interop/src/index.ts | 1 + .../test/fixtures/create-helia.browser.ts | 46 ++++ .../interop/test/fixtures/create-helia.ts | 39 +++ packages/interop/test/fixtures/create-kubo.ts | 28 ++ packages/interop/test/fixtures/memory-car.ts | 33 +++ packages/interop/test/index.spec.ts | 96 +++++++ packages/interop/tsconfig.json | 15 ++ 34 files changed, 1680 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 package.json create mode 100644 packages/car/LICENSE create mode 100644 packages/car/LICENSE-APACHE create mode 100644 packages/car/LICENSE-MIT create mode 100644 packages/car/README.md create mode 100644 packages/car/package.json create mode 100644 packages/car/src/index.ts create mode 100644 packages/car/src/utils/dag-walkers.ts create mode 100644 packages/car/test/fixtures/files.ts create mode 100644 packages/car/test/fixtures/memory-car.ts create mode 100644 packages/car/test/index.spec.ts create mode 100644 packages/car/tsconfig.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/test/fixtures/memory-car.ts create mode 100644 packages/interop/test/index.spec.ts create mode 100644 packages/interop/tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..51686414 --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,181 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox-webworker + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit-webworker: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7ad9e674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..3386fefe --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +

+ + Helia logo + +

+ +# @helia/car + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-car.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-car) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-car/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-car/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Import/export car files from Helia + +## Table of contents + +- [Structure](#structure) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) + +## Structure + +- [`/packages/car`](./packages/car) Import/export car files from Helia +- [`/packages/interop`](./packages/interop) Interop tests for @helia/car + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-car/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/package.json b/package.json new file mode 100644 index 00000000..92535a03 --- /dev/null +++ b/package.json @@ -0,0 +1,44 @@ +{ + "name": "@helia/car", + "version": "0.0.0", + "description": "Import/export car files from Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-car#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-car.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-car/issues" + }, + "keywords": [ + "ipfs" + ], + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && aegir run release && npm run docs" + }, + "devDependencies": { + "aegir": "^39.0.8" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/car/LICENSE b/packages/car/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/car/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/car/LICENSE-APACHE b/packages/car/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/car/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/car/LICENSE-MIT b/packages/car/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/car/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/car/README.md b/packages/car/README.md new file mode 100644 index 00000000..72de958e --- /dev/null +++ b/packages/car/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/car + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-car.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-car) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-car/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-car/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Import/export car files from Helia + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-car/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/car/package.json b/packages/car/package.json new file mode 100644 index 00000000..7c96dc3b --- /dev/null +++ b/packages/car/package.json @@ -0,0 +1,181 @@ +{ + "name": "@helia/car", + "version": "0.0.0", + "description": "Import/export car files from Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-car.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-car/issues" + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./errors": { + "types": "./dist/src/errors.d.ts", + "import": "./dist/src/errors.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "^1.0.0", + "@ipld/car": "^5.1.1", + "@ipld/dag-pb": "^4.0.0", + "@libp2p/interfaces": "^3.3.1", + "cborg": "^1.10.2", + "interface-blockstore": "^5.0.0", + "it-map": "^3.0.3", + "multiformats": "^11.0.1", + "p-defer": "^4.0.0", + "p-queue": "^7.3.4", + "progress-events": "^1.0.0" + }, + "devDependencies": { + "@helia/unixfs": "^1.2.2", + "aegir": "^39.0.8", + "blockstore-core": "^4.0.1", + "ipfs-unixfs-importer": "^15.1.0", + "it-drain": "^3.0.1", + "it-to-buffer": "^4.0.2" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/car/src/index.ts b/packages/car/src/index.ts new file mode 100644 index 00000000..63812274 --- /dev/null +++ b/packages/car/src/index.ts @@ -0,0 +1,239 @@ +/** + * @packageDocumentation + * + * `@helia/car` provides `import` and `export` methods to read/write Car files to {@link https://github.com/ipfs/helia Helia}'s blockstore. + * + * See the {@link Car Car interface} for all available operations. + * + * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more esoteric DAG walkers can be passed as an init option. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { unixfs } from '@helia/unixfs' + * import { car } from '@helia/car' + * import { CarWriter } from '@ipld/car' + * import { Readable } from 'node:stream' + * import fs from 'node:fs' + * + * const helia = createHelia({ + * // ... helia config + * }) + * const fs = unixfs(helia) + * + * // add some UnixFS data + * const cid = await fs.addBytes(fileData1) + * + * // export it as a Car + * const c = car(helia) + * const { writer, out } = await CarWriter.create(cid) + * + * // `out` needs to be directed somewhere, see the @ipld/car docs for more information + * Readable.from(out).pipe(fs.createWriteStream('example.car')) + * + * // write the DAG behind `cid` into the writer + * await c.export(cid, writer) + * ``` + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { unixfs } from '@helia/unixfs' + * import { car } from '@helia/car' + * import { CarReader } from '@ipld/car' + * import { Readable } from 'node:stream' + * import fs from 'node:fs' + * + * const helia = createHelia({ + * // ... helia config + * }) + * + * // import the car + * const inStream = fs.createReadStream('example.car') + * const reader = await CarReader.fromIterable(inStream) + * + * await c.import(reader) + * ``` + */ + +import drain from 'it-drain' +import map from 'it-map' +import defer from 'p-defer' +import PQueue from 'p-queue' +import { cborWalker, dagPbWalker, jsonWalker, rawWalker } from './utils/dag-walkers.js' +import type { Blocks, GetBlockProgressEvents, PutManyBlocksProgressEvents } from '@helia/interface/blocks' +import type { CarReader, CarWriter } from '@ipld/car' +import type { AbortOptions } from '@libp2p/interfaces' +import type { CID } from 'multiformats/cid' +import type { ProgressOptions } from 'progress-events' + +export interface CarComponents { + blockstore: Blocks +} + +export interface CarInit { + /** + * In order to export CIDs that correspond to a DAG, it's necessary to know + * how to traverse that DAG. DAGWalkers take a block and yield any CIDs + * encoded within that block. + */ + dagWalkers?: DAGWalker[] +} + +/** + * DAGWalkers take a block and yield CIDs encoded in that block + */ +export interface DAGWalker { + codec: number + walk: (block: Uint8Array) => AsyncGenerator +} + +/** + * The Car interface provides operations for importing and exporting Car files + * from Helia's underlying blockstore. + */ +export interface Car { + /** + * Add all blocks in the passed CarReader to the blockstore. + * + * @example + * + * ```typescript + * import fs from 'node:fs' + * import { createHelia } from 'helia' + * import { car } from '@helia/car + * import { CarReader } from '@ipld/car' + * + * const helia = await createHelia() + * + * const inStream = fs.createReadStream('example.car') + * const reader = await CarReader.fromIterable(inStream) + * + * const c = car(helia) + * await c.import(reader) + * ``` + */ + import: (reader: Pick, options?: AbortOptions & ProgressOptions) => Promise + + /** + * Store all blocks that make up one or more DAGs in a car file. + * + * @example + * + * ```typescript + * import fs from 'node:fs' + * import { Readable } from 'stream' + * import { createHelia } from 'helia' + * import { car } from '@helia/car + * import { CID } from 'multiformats/cid' + * import pEvent from 'p-event' + * + * const helia = await createHelia() + * const cid = CID.parse('QmFoo...') + * + * const c = car(helia) + * + * const byteStream = await c.export(cid) + * const output = fs.createWriteStream('example.car') + * const eventPromise = pEvent(output, 'end') + * Readable.from(byteStream).pipe(output) + * + * await eventPromise + * ``` + */ + export: (root: CID | CID[], writer: Pick, options?: AbortOptions & ProgressOptions) => Promise +} + +const DEFAULT_DAG_WALKERS = [ + rawWalker, + dagPbWalker, + cborWalker, + jsonWalker +] + +const DAG_WALK_QUEUE_CONCURRENCY = 1 + +class DefaultCar implements Car { + private readonly components: CarComponents + private dagWalkers: Record + + constructor (components: CarComponents, init: CarInit) { + this.components = components + + this.dagWalkers = {} + + ;[...DEFAULT_DAG_WALKERS, ...(init.dagWalkers ?? [])].forEach(dagWalker => { + this.dagWalkers[dagWalker.codec] = dagWalker + }) + } + + async import (reader: Pick, options?: AbortOptions & ProgressOptions): Promise { + await drain(this.components.blockstore.putMany( + map(reader.blocks(), ({ cid, bytes }) => ({ cid, block: bytes })), + options + )) + } + + async export (root: CID | CID[], writer: Pick, options?: AbortOptions & ProgressOptions): Promise { + const deferred = defer() + const roots = Array.isArray(root) ? root : [root] + + // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs + const queue = new PQueue({ + concurrency: DAG_WALK_QUEUE_CONCURRENCY + }) + queue.on('idle', () => { + deferred.resolve() + }) + queue.on('error', (err) => { + deferred.resolve(err) + }) + + for (const root of roots) { + void queue.add(async () => { + await this.#walkDag(root, queue, async (cid, bytes) => { + await writer.put({ cid, bytes }) + }, options) + }) + } + + // wait for the writer to end + try { + await deferred.promise + } finally { + await writer.close() + } + } + + /** + * Walk the DAG behind the passed CID, ensure all blocks are present in the blockstore + * and update the pin count for them + */ + async #walkDag (cid: CID, queue: PQueue, withBlock: (cid: CID, block: Uint8Array) => Promise, options?: AbortOptions & ProgressOptions): Promise { + const dagWalker = this.dagWalkers[cid.code] + + if (dagWalker == null) { + throw new Error(`No dag walker found for cid codec ${cid.code}`) + } + + const block = await this.components.blockstore.get(cid, options) + + await withBlock(cid, block) + + // walk dag, ensure all blocks are present + for await (const cid of dagWalker.walk(block)) { + void queue.add(async () => { + await this.#walkDag(cid, queue, withBlock, options) + }) + } + } +} + +/** + * Create a {@link Car} instance for use with {@link https://github.com/ipfs/helia Helia} + */ +export function car (helia: { blockstore: Blocks }, init: CarInit = {}): Car { + return new DefaultCar(helia, init) +} diff --git a/packages/car/src/utils/dag-walkers.ts b/packages/car/src/utils/dag-walkers.ts new file mode 100644 index 00000000..1216138c --- /dev/null +++ b/packages/car/src/utils/dag-walkers.ts @@ -0,0 +1,176 @@ +/* eslint max-depth: ["error", 7] */ + +import * as dagPb from '@ipld/dag-pb' +import * as cborg from 'cborg' +import { Type, Token } from 'cborg' +import * as cborgJson from 'cborg/json' +import { CID } from 'multiformats' +import { base64 } from 'multiformats/bases/base64' +import * as raw from 'multiformats/codecs/raw' + +/** + * DAGWalkers take a block and yield CIDs encoded in that block + */ +export interface DAGWalker { + codec: number + walk: (block: Uint8Array) => AsyncGenerator +} + +/** + * Dag walker for dag-pb CIDs + */ +export const dagPbWalker: DAGWalker = { + codec: dagPb.code, + async * walk (block) { + const node = dagPb.decode(block) + + yield * node.Links.map(l => l.Hash) + } +} + +/** + * Dag walker for raw CIDs + */ +export const rawWalker: DAGWalker = { + codec: raw.code, + async * walk () { + // no embedded CIDs in a raw block + } +} + +// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692 +const CID_TAG = 42 + +/** + * Dag walker for dag-cbor CIDs. Does not actually use dag-cbor since + * all we are interested in is extracting the the CIDs from the block + * so we can just use cborg for that. + */ +export const cborWalker: DAGWalker = { + codec: 0x71, + async * walk (block) { + const cids: CID[] = [] + const tags: cborg.TagDecoder[] = [] + tags[CID_TAG] = (bytes) => { + if (bytes[0] !== 0) { + throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00') + } + + const cid = CID.decode(bytes.subarray(1)) // ignore leading 0x00 + + cids.push(cid) + + return cid + } + + cborg.decode(block, { + tags + }) + + yield * cids + } +} + +/** + * Borrowed from @ipld/dag-json + */ +class DagJsonTokenizer extends cborgJson.Tokenizer { + private readonly tokenBuffer: cborg.Token[] + + constructor (data: Uint8Array, options?: cborg.DecodeOptions) { + super(data, options) + + this.tokenBuffer = [] + } + + done (): boolean { + return this.tokenBuffer.length === 0 && super.done() + } + + _next (): cborg.Token { + if (this.tokenBuffer.length > 0) { + // @ts-expect-error https://github.com/Microsoft/TypeScript/issues/30406 + return this.tokenBuffer.pop() + } + return super.next() + } + + /** + * Implements rules outlined in https://github.com/ipld/specs/pull/356 + */ + next (): cborg.Token { + const token = this._next() + + if (token.type === Type.map) { + const keyToken = this._next() + if (keyToken.type === Type.string && keyToken.value === '/') { + const valueToken = this._next() + if (valueToken.type === Type.string) { // *must* be a CID + const breakToken = this._next() // swallow the end-of-map token + if (breakToken.type !== Type.break) { + throw new Error('Invalid encoded CID form') + } + this.tokenBuffer.push(valueToken) // CID.parse will pick this up after our tag token + return new Token(Type.tag, 42, 0) + } + if (valueToken.type === Type.map) { + const innerKeyToken = this._next() + if (innerKeyToken.type === Type.string && innerKeyToken.value === 'bytes') { + const innerValueToken = this._next() + if (innerValueToken.type === Type.string) { // *must* be Bytes + for (let i = 0; i < 2; i++) { + const breakToken = this._next() // swallow two end-of-map tokens + if (breakToken.type !== Type.break) { + throw new Error('Invalid encoded Bytes form') + } + } + const bytes = base64.decode(`m${innerValueToken.value}`) + return new Token(Type.bytes, bytes, innerValueToken.value.length) + } + this.tokenBuffer.push(innerValueToken) // bail + } + this.tokenBuffer.push(innerKeyToken) // bail + } + this.tokenBuffer.push(valueToken) // bail + } + this.tokenBuffer.push(keyToken) // bail + } + return token + } +} + +/** + * Dag walker for dag-json CIDs. Does not actually use dag-json since + * all we are interested in is extracting the the CIDs from the block + * so we can just use cborg/json for that. + */ +export const jsonWalker: DAGWalker = { + codec: 0x0129, + async * walk (block) { + const cids: CID[] = [] + const tags: cborg.TagDecoder[] = [] + tags[CID_TAG] = (string) => { + const cid = CID.parse(string) + + cids.push(cid) + + return cid + } + + cborgJson.decode(block, { + tags, + tokenizer: new DagJsonTokenizer(block, { + tags, + allowIndefinite: true, + allowUndefined: true, + allowNaN: true, + allowInfinity: true, + allowBigInt: true, + strict: false, + rejectDuplicateMapKeys: false + }) + }) + + yield * cids + } +} diff --git a/packages/car/test/fixtures/files.ts b/packages/car/test/fixtures/files.ts new file mode 100644 index 00000000..cce002d2 --- /dev/null +++ b/packages/car/test/fixtures/files.ts @@ -0,0 +1,5 @@ +const ONE_MEG = 1024 * 1024 + +export const largeFile = Uint8Array.from(new Array(ONE_MEG * 5).fill(0).map(() => Math.random() * 100)) + +export const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) diff --git a/packages/car/test/fixtures/memory-car.ts b/packages/car/test/fixtures/memory-car.ts new file mode 100644 index 00000000..54debcac --- /dev/null +++ b/packages/car/test/fixtures/memory-car.ts @@ -0,0 +1,33 @@ +import { CarWriter } from '@ipld/car' +import toBuffer from 'it-to-buffer' +import defer from 'p-defer' +import type { CID } from 'multiformats/cid' + +export interface MemoryCar extends Pick { + bytes: () => Promise +} + +export function memoryCarWriter (root: CID | CID[]): MemoryCar { + const deferred = defer() + const { writer, out } = CarWriter.create(Array.isArray(root) ? root : [root]) + + Promise.resolve() + .then(async () => { + deferred.resolve(await toBuffer(out)) + }) + .catch(err => { + deferred.reject(err) + }) + + return { + async put (block: { cid: CID, bytes: Uint8Array }): Promise { + await writer.put(block) + }, + async close (): Promise { + await writer.close() + }, + async bytes (): Promise { + return deferred.promise + } + } +} diff --git a/packages/car/test/index.spec.ts b/packages/car/test/index.spec.ts new file mode 100644 index 00000000..9fba0ed4 --- /dev/null +++ b/packages/car/test/index.spec.ts @@ -0,0 +1,121 @@ +/* eslint-env mocha */ + +import { type UnixFS, unixfs } from '@helia/unixfs' +import { CarReader } from '@ipld/car' +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import toBuffer from 'it-to-buffer' +import { car, type Car } from '../src/index.js' +import { largeFile, smallFile } from './fixtures/files.js' +import { memoryCarWriter } from './fixtures/memory-car.js' +import type { Blockstore } from 'interface-blockstore' + +describe('import', () => { + let blockstore: Blockstore + let c: Car + let u: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + c = car({ blockstore }) + u = unixfs({ blockstore }) + }) + + it('exports and imports a car file', async () => { + const otherBlockstore = new MemoryBlockstore() + const otherUnixFS = unixfs({ blockstore: otherBlockstore }) + const otherCar = car({ blockstore: otherBlockstore }) + const cid = await otherUnixFS.addBytes(smallFile) + + const writer = memoryCarWriter(cid) + await otherCar.export(cid, writer) + + const reader = await CarReader.fromBytes(await writer.bytes()) + + await c.import(reader) + + expect(await blockstore.get(cid)).to.equalBytes(smallFile) + }) + + it('exports and imports a multiple root car file', async () => { + const fileData1 = Uint8Array.from([0, 1, 2, 3]) + const fileData2 = Uint8Array.from([4, 5, 6, 7]) + const fileData3 = Uint8Array.from([8, 9, 0, 1]) + + const otherBlockstore = new MemoryBlockstore() + const otherUnixFS = unixfs({ blockstore: otherBlockstore }) + const otherCar = car({ blockstore: otherBlockstore }) + const cid1 = await otherUnixFS.addBytes(fileData1) + const cid2 = await otherUnixFS.addBytes(fileData2) + const cid3 = await otherUnixFS.addBytes(fileData3) + + const writer = memoryCarWriter([cid1, cid2, cid3]) + await otherCar.export([cid1, cid2, cid3], writer) + + const reader = await CarReader.fromBytes(await writer.bytes()) + + await c.import(reader) + + expect(await toBuffer(u.cat(cid1))).to.equalBytes(fileData1) + expect(await toBuffer(u.cat(cid2))).to.equalBytes(fileData2) + expect(await toBuffer(u.cat(cid3))).to.equalBytes(fileData3) + }) + + it('exports and imports a multiple block car file', async () => { + const otherBlockstore = new MemoryBlockstore() + const otherUnixFS = unixfs({ blockstore: otherBlockstore }) + const otherCar = car({ blockstore: otherBlockstore }) + const cid = await otherUnixFS.addBytes(largeFile, { + chunker: fixedSize({ + chunkSize: 1024 + }) + }) + + const writer = memoryCarWriter(cid) + await otherCar.export(cid, writer) + + const reader = await CarReader.fromBytes(await writer.bytes()) + + await c.import(reader) + + expect(await toBuffer(u.cat(cid))).to.equalBytes(largeFile) + }) + + it('exports and imports a multiple block, multiple root car file', async () => { + const fileData1 = Uint8Array.from([0, 1, 2, 3, 4, 5, 6, 7]) + const fileData2 = Uint8Array.from([4, 5, 6, 7, 8, 9, 0, 1]) + const fileData3 = Uint8Array.from([8, 9, 0, 1, 2, 3, 4, 5]) + + const otherBlockstore = new MemoryBlockstore() + const otherUnixFS = unixfs({ blockstore: otherBlockstore }) + const otherCar = car({ blockstore: otherBlockstore }) + const cid1 = await otherUnixFS.addBytes(fileData1, { + chunker: fixedSize({ + chunkSize: 2 + }) + }) + const cid2 = await otherUnixFS.addBytes(fileData2, { + chunker: fixedSize({ + chunkSize: 2 + }) + }) + const cid3 = await otherUnixFS.addBytes(fileData3, { + chunker: fixedSize({ + chunkSize: 2 + }) + }) + + const writer = memoryCarWriter([cid1, cid2, cid3]) + await otherCar.export([cid1, cid2, cid3], writer) + + const reader = await CarReader.fromBytes(await writer.bytes()) + + await c.import(reader) + + expect(await toBuffer(u.cat(cid1))).to.equalBytes(fileData1) + expect(await toBuffer(u.cat(cid2))).to.equalBytes(fileData2) + expect(await toBuffer(u.cat(cid3))).to.equalBytes(fileData3) + }) +}) diff --git a/packages/car/tsconfig.json b/packages/car/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/car/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..254d35ee --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/car-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-car.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-car) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-car/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-car/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/car + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-car/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..080b2934 --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,82 @@ +{ + "name": "@helia/car-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/car", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-car.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-car/issues" + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^12.0.1", + "@chainsafe/libp2p-yamux": "^4.0.2", + "@helia/car": "~0.0.0", + "@helia/interface": "^1.0.0", + "@helia/unixfs": "^1.2.2", + "@ipld/car": "^5.1.1", + "@libp2p/tcp": "^7.0.1", + "@libp2p/websockets": "^6.0.1", + "aegir": "^39.0.8", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.0.3", + "go-ipfs": "^0.20.0", + "helia": "^1.0.0", + "ipfs-unixfs-importer": "^15.1.0", + "ipfsd-ctl": "^13.0.0", + "it-to-buffer": "^4.0.2", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.45.3", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1", + "p-defer": "^4.0.0", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..867de492 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,46 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + services: { + identify: identifyService() + }, + datastore, + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..f7a25c3f --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,39 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + services: { + identify: identifyService() + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..52ed76e1 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import mergeOptions from 'merge-options' +import { isElectronMain, isNode } from 'wherearewe' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return createController(opts) +} diff --git a/packages/interop/test/fixtures/memory-car.ts b/packages/interop/test/fixtures/memory-car.ts new file mode 100644 index 00000000..54debcac --- /dev/null +++ b/packages/interop/test/fixtures/memory-car.ts @@ -0,0 +1,33 @@ +import { CarWriter } from '@ipld/car' +import toBuffer from 'it-to-buffer' +import defer from 'p-defer' +import type { CID } from 'multiformats/cid' + +export interface MemoryCar extends Pick { + bytes: () => Promise +} + +export function memoryCarWriter (root: CID | CID[]): MemoryCar { + const deferred = defer() + const { writer, out } = CarWriter.create(Array.isArray(root) ? root : [root]) + + Promise.resolve() + .then(async () => { + deferred.resolve(await toBuffer(out)) + }) + .catch(err => { + deferred.reject(err) + }) + + return { + async put (block: { cid: CID, bytes: Uint8Array }): Promise { + await writer.put(block) + }, + async close (): Promise { + await writer.close() + }, + async bytes (): Promise { + return deferred.promise + } + } +} diff --git a/packages/interop/test/index.spec.ts b/packages/interop/test/index.spec.ts new file mode 100644 index 00000000..ca992bf7 --- /dev/null +++ b/packages/interop/test/index.spec.ts @@ -0,0 +1,96 @@ +/* eslint-env mocha */ + +import { car } from '@helia/car' +import { type UnixFS, unixfs } from '@helia/unixfs' +import { CarReader } from '@ipld/car' +import { expect } from 'aegir/chai' +import toBuffer from 'it-to-buffer' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import { memoryCarWriter } from './fixtures/memory-car.js' +import type { Car } from '@helia/car' +import type { Helia } from '@helia/interface' +import type { FileCandidate } from 'ipfs-unixfs-importer' +import type { Controller } from 'ipfsd-ctl' + +describe('car interop', () => { + let helia: Helia + let c: Car + let u: UnixFS + let kubo: Controller + + beforeEach(async () => { + helia = await createHeliaNode() + c = car(helia) + u = unixfs(helia) + kubo = await createKuboNode() + + // connect helia to kubo + await helia.libp2p.dial(kubo.peer.addresses) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should export a car from Helia, import and read the contents from kubo', async () => { + const chunkSize = 1024 * 1024 + const size = chunkSize * 10 + const input: Uint8Array[] = [] + + const candidate: FileCandidate = { + content: (async function * () { + for (let i = 0; i < size; i += chunkSize) { + const buf = new Uint8Array(chunkSize) + input.push(buf) + + yield buf + } + }()) + } + + const cid = await u.addFile(candidate) + const writer = memoryCarWriter(cid) + await c.export(cid, writer) + + const buf = await writer.bytes() + + kubo.api.dag.import([buf]) + + const bytes = await toBuffer(kubo.api.cat(cid)) + + expect(bytes).to.equalBytes(toBuffer(input)) + }) + + it('should export a car from kubo, import and read the contents from Helia', async () => { + const chunkSize = 1024 * 1024 + const size = chunkSize * 10 + const input: Uint8Array[] = [] + + const candidate: FileCandidate = { + content: (async function * () { + for (let i = 0; i < size; i += chunkSize) { + const buf = new Uint8Array(chunkSize) + input.push(buf) + + yield buf + } + }()) + } + + const { cid } = await kubo.api.add(candidate.content) + const bytes = await toBuffer(kubo.api.dag.export(cid)) + + const reader = await CarReader.fromBytes(bytes) + + await c.import(reader) + + expect(await toBuffer(u.cat(cid))).to.equalBytes(toBuffer(input)) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..3b3b8539 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../car" + } + ] +} From 0857d1f76cd7403dbea46cf3d9c891543fc83fe1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 7 Jun 2023 11:03:35 +0100 Subject: [PATCH 092/253] fix: import from multiformats/cid for smaller bundles --- packages/car/src/utils/dag-walkers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/car/src/utils/dag-walkers.ts b/packages/car/src/utils/dag-walkers.ts index 1216138c..2e364c07 100644 --- a/packages/car/src/utils/dag-walkers.ts +++ b/packages/car/src/utils/dag-walkers.ts @@ -4,7 +4,7 @@ import * as dagPb from '@ipld/dag-pb' import * as cborg from 'cborg' import { Type, Token } from 'cborg' import * as cborgJson from 'cborg/json' -import { CID } from 'multiformats' +import { CID } from 'multiformats/cid' import { base64 } from 'multiformats/bases/base64' import * as raw from 'multiformats/codecs/raw' From 3803a378c0d7a556248e39e7a3c756e94e69888d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 7 Jun 2023 11:07:26 +0100 Subject: [PATCH 093/253] chore: fix linting --- packages/car/src/utils/dag-walkers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/car/src/utils/dag-walkers.ts b/packages/car/src/utils/dag-walkers.ts index 2e364c07..62de926f 100644 --- a/packages/car/src/utils/dag-walkers.ts +++ b/packages/car/src/utils/dag-walkers.ts @@ -4,8 +4,8 @@ import * as dagPb from '@ipld/dag-pb' import * as cborg from 'cborg' import { Type, Token } from 'cborg' import * as cborgJson from 'cborg/json' -import { CID } from 'multiformats/cid' import { base64 } from 'multiformats/bases/base64' +import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' /** From e0f815c1279793777f12ac4537f93986c8bda190 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 7 Jun 2023 10:23:07 +0000 Subject: [PATCH 094/253] chore(release): 1.0.0 [skip ci] ## @helia/car-v1.0.0 (2023-06-07) ### Bug Fixes * import from multiformats/cid for smaller bundles ([0857d1f](https://github.com/ipfs/helia-car/commit/0857d1f76cd7403dbea46cf3d9c891543fc83fe1)) ### Trivial Changes * fix linting ([3803a37](https://github.com/ipfs/helia-car/commit/3803a378c0d7a556248e39e7a3c756e94e69888d)) --- packages/car/CHANGELOG.md | 11 +++++++++++ packages/car/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 packages/car/CHANGELOG.md diff --git a/packages/car/CHANGELOG.md b/packages/car/CHANGELOG.md new file mode 100644 index 00000000..8d80597f --- /dev/null +++ b/packages/car/CHANGELOG.md @@ -0,0 +1,11 @@ +## @helia/car-v1.0.0 (2023-06-07) + + +### Bug Fixes + +* import from multiformats/cid for smaller bundles ([0857d1f](https://github.com/ipfs/helia-car/commit/0857d1f76cd7403dbea46cf3d9c891543fc83fe1)) + + +### Trivial Changes + +* fix linting ([3803a37](https://github.com/ipfs/helia-car/commit/3803a378c0d7a556248e39e7a3c756e94e69888d)) diff --git a/packages/car/package.json b/packages/car/package.json index 7c96dc3b..3855696b 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -1,6 +1,6 @@ { "name": "@helia/car", - "version": "0.0.0", + "version": "1.0.0", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", From 7e3815e00dba19488e7b2b7a262559c32bfa32bc Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Wed, 7 Jun 2023 10:23:28 +0000 Subject: [PATCH 095/253] deps: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 080b2934..49fd5942 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -50,7 +50,7 @@ "devDependencies": { "@chainsafe/libp2p-noise": "^12.0.1", "@chainsafe/libp2p-yamux": "^4.0.2", - "@helia/car": "~0.0.0", + "@helia/car": "^1.0.0", "@helia/interface": "^1.0.0", "@helia/unixfs": "^1.2.2", "@ipld/car": "^5.1.1", From 8426d650ae4645b7b975331c5fd02f56e390cab6 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 7 Jun 2023 11:45:10 +0100 Subject: [PATCH 096/253] fix: export unixfs errors (#50) Export error defs for reuse in other unixfs related modules --- package.json | 4 ---- packages/interop/package.json | 4 ---- packages/unixfs/package.json | 24 +++++++++++++++---- packages/unixfs/src/commands/cat.ts | 2 +- packages/unixfs/src/commands/chmod.ts | 2 +- packages/unixfs/src/commands/cp.ts | 2 +- packages/unixfs/src/commands/ls.ts | 2 +- packages/unixfs/src/commands/mkdir.ts | 2 +- packages/unixfs/src/commands/rm.ts | 2 +- packages/unixfs/src/commands/stat.ts | 2 +- packages/unixfs/src/commands/touch.ts | 2 +- .../unixfs/src/commands/utils/add-link.ts | 2 +- .../src/commands/utils/cid-to-directory.ts | 2 +- .../src/commands/utils/cid-to-pblink.ts | 2 +- .../unixfs/src/commands/utils/remove-link.ts | 2 +- packages/unixfs/src/commands/utils/resolve.ts | 2 +- .../unixfs/src/{commands/utils => }/errors.ts | 0 17 files changed, 33 insertions(+), 25 deletions(-) rename packages/unixfs/src/{commands/utils => }/errors.ts (100%) diff --git a/package.json b/package.json index 91c3b6b5..880bc567 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ "keywords": [ "ipfs" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "private": true, "scripts": { "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", diff --git a/packages/interop/package.json b/packages/interop/package.json index 001e0e3d..585d781d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index ddb00c1e..583831c3 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -14,12 +14,24 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, "files": [ "src", "dist", @@ -30,6 +42,10 @@ ".": { "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js" + }, + "./errors": { + "types": "./dist/src/errors.d.ts", + "import": "./dist/src/errors.js" } }, "eslintConfig": { diff --git a/packages/unixfs/src/commands/cat.ts b/packages/unixfs/src/commands/cat.ts index 1360281f..18269a80 100644 --- a/packages/unixfs/src/commands/cat.ts +++ b/packages/unixfs/src/commands/cat.ts @@ -1,6 +1,6 @@ import { exporter } from 'ipfs-unixfs-exporter' import mergeOpts from 'merge-options' -import { NoContentError, NotAFileError } from './utils/errors.js' +import { NoContentError, NotAFileError } from '../errors.js' import { resolve } from './utils/resolve.js' import type { CatOptions } from '../index.js' import type { Blocks } from '@helia/interface/blocks' diff --git a/packages/unixfs/src/commands/chmod.ts b/packages/unixfs/src/commands/chmod.ts index b0092d39..fc72c57a 100644 --- a/packages/unixfs/src/commands/chmod.ts +++ b/packages/unixfs/src/commands/chmod.ts @@ -9,8 +9,8 @@ import mergeOpts from 'merge-options' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' -import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' import { persist } from './utils/persist.js' import { resolve, updatePathCids } from './utils/resolve.js' import type { ChmodOptions } from '../index.js' diff --git a/packages/unixfs/src/commands/cp.ts b/packages/unixfs/src/commands/cp.ts index 010a9612..f8bbfdf1 100644 --- a/packages/unixfs/src/commands/cp.ts +++ b/packages/unixfs/src/commands/cp.ts @@ -1,10 +1,10 @@ import { logger } from '@libp2p/logger' import mergeOpts from 'merge-options' +import { InvalidParametersError } from '../errors.js' import { addLink } from './utils/add-link.js' import { cidToDirectory } from './utils/cid-to-directory.js' import { cidToPBLink } from './utils/cid-to-pblink.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' -import { InvalidParametersError } from './utils/errors.js' import type { CpOptions } from '../index.js' import type { Blocks } from '@helia/interface/blocks' import type { CID } from 'multiformats/cid' diff --git a/packages/unixfs/src/commands/ls.ts b/packages/unixfs/src/commands/ls.ts index 7525b66f..c27566bd 100644 --- a/packages/unixfs/src/commands/ls.ts +++ b/packages/unixfs/src/commands/ls.ts @@ -1,6 +1,6 @@ import { exporter, type UnixFSEntry } from 'ipfs-unixfs-exporter' import mergeOpts from 'merge-options' -import { NoContentError, NotADirectoryError } from './utils/errors.js' +import { NoContentError, NotADirectoryError } from '../errors.js' import { resolve } from './utils/resolve.js' import type { LsOptions } from '../index.js' import type { Blocks } from '@helia/interface/blocks' diff --git a/packages/unixfs/src/commands/mkdir.ts b/packages/unixfs/src/commands/mkdir.ts index 272a94ea..36c968c5 100644 --- a/packages/unixfs/src/commands/mkdir.ts +++ b/packages/unixfs/src/commands/mkdir.ts @@ -5,11 +5,11 @@ import { exporter } from 'ipfs-unixfs-exporter' import mergeOpts from 'merge-options' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' +import { InvalidParametersError, NotADirectoryError } from '../errors.js' import { addLink } from './utils/add-link.js' import { cidToDirectory } from './utils/cid-to-directory.js' import { cidToPBLink } from './utils/cid-to-pblink.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' -import { InvalidParametersError, NotADirectoryError } from './utils/errors.js' import type { MkdirOptions } from '../index.js' import type { Blocks } from '@helia/interface/blocks' diff --git a/packages/unixfs/src/commands/rm.ts b/packages/unixfs/src/commands/rm.ts index 8da00de4..e640ac4a 100644 --- a/packages/unixfs/src/commands/rm.ts +++ b/packages/unixfs/src/commands/rm.ts @@ -1,8 +1,8 @@ import { logger } from '@libp2p/logger' import mergeOpts from 'merge-options' +import { InvalidParametersError } from '../errors.js' import { cidToDirectory } from './utils/cid-to-directory.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' -import { InvalidParametersError } from './utils/errors.js' import { removeLink } from './utils/remove-link.js' import type { RmOptions } from '../index.js' import type { Blocks } from '@helia/interface/blocks' diff --git a/packages/unixfs/src/commands/stat.ts b/packages/unixfs/src/commands/stat.ts index 6955418e..e7a84f86 100644 --- a/packages/unixfs/src/commands/stat.ts +++ b/packages/unixfs/src/commands/stat.ts @@ -4,7 +4,7 @@ import { UnixFS } from 'ipfs-unixfs' import { exporter } from 'ipfs-unixfs-exporter' import mergeOpts from 'merge-options' import * as raw from 'multiformats/codecs/raw' -import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js' import { resolve } from './utils/resolve.js' import type { StatOptions, UnixFSStats } from '../index.js' import type { Blocks } from '@helia/interface/blocks' diff --git a/packages/unixfs/src/commands/touch.ts b/packages/unixfs/src/commands/touch.ts index 7482f6f7..1cdae6de 100644 --- a/packages/unixfs/src/commands/touch.ts +++ b/packages/unixfs/src/commands/touch.ts @@ -9,8 +9,8 @@ import mergeOpts from 'merge-options' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' +import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js' import { SHARD_SPLIT_THRESHOLD_BYTES } from './utils/constants.js' -import { InvalidPBNodeError, NotUnixFSError, UnknownError } from './utils/errors.js' import { persist } from './utils/persist.js' import { resolve, updatePathCids } from './utils/resolve.js' import type { TouchOptions } from '../index.js' diff --git a/packages/unixfs/src/commands/utils/add-link.ts b/packages/unixfs/src/commands/utils/add-link.ts index 350c7ddc..a48b2053 100644 --- a/packages/unixfs/src/commands/utils/add-link.ts +++ b/packages/unixfs/src/commands/utils/add-link.ts @@ -6,8 +6,8 @@ import { sha256 } from 'multiformats/hashes/sha2' // @ts-expect-error no types import SparseArray from 'sparse-array' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from '../../errors.js' import { wrapHash } from './consumable-hash.js' -import { AlreadyExistsError, InvalidParametersError, InvalidPBNodeError } from './errors.js' import { hamtBucketBits, hamtHashFn } from './hamt-constants.js' import { createShard, diff --git a/packages/unixfs/src/commands/utils/cid-to-directory.ts b/packages/unixfs/src/commands/utils/cid-to-directory.ts index afe5638a..dc072c2c 100644 --- a/packages/unixfs/src/commands/utils/cid-to-directory.ts +++ b/packages/unixfs/src/commands/utils/cid-to-directory.ts @@ -1,5 +1,5 @@ import { exporter, type ExporterOptions } from 'ipfs-unixfs-exporter' -import { NotADirectoryError } from './errors.js' +import { NotADirectoryError } from '../../errors.js' import type { Blocks } from '@helia/interface/blocks' import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' diff --git a/packages/unixfs/src/commands/utils/cid-to-pblink.ts b/packages/unixfs/src/commands/utils/cid-to-pblink.ts index 513ea0ab..a1a460dd 100644 --- a/packages/unixfs/src/commands/utils/cid-to-pblink.ts +++ b/packages/unixfs/src/commands/utils/cid-to-pblink.ts @@ -1,6 +1,6 @@ import * as dagPb from '@ipld/dag-pb' import { exporter, type ExporterOptions } from 'ipfs-unixfs-exporter' -import { NotUnixFSError } from './errors.js' +import { NotUnixFSError } from '../../errors.js' import type { Blocks } from '@helia/interface/blocks' import type { PBNode, PBLink } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' diff --git a/packages/unixfs/src/commands/utils/remove-link.ts b/packages/unixfs/src/commands/utils/remove-link.ts index a60f6412..a39256f4 100644 --- a/packages/unixfs/src/commands/utils/remove-link.ts +++ b/packages/unixfs/src/commands/utils/remove-link.ts @@ -3,7 +3,7 @@ import * as dagPB from '@ipld/dag-pb' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' import { exporter } from 'ipfs-unixfs-exporter' -import { InvalidParametersError, InvalidPBNodeError } from './errors.js' +import { InvalidParametersError, InvalidPBNodeError } from '../../errors.js' import { recreateShardedDirectory, type UpdateHamtDirectoryOptions, diff --git a/packages/unixfs/src/commands/utils/resolve.ts b/packages/unixfs/src/commands/utils/resolve.ts index a2391d93..9f8cecf2 100644 --- a/packages/unixfs/src/commands/utils/resolve.ts +++ b/packages/unixfs/src/commands/utils/resolve.ts @@ -1,9 +1,9 @@ import { logger } from '@libp2p/logger' import { exporter } from 'ipfs-unixfs-exporter' +import { DoesNotExistError, InvalidParametersError } from '../../errors.js' import { addLink } from './add-link.js' import { cidToDirectory } from './cid-to-directory.js' import { cidToPBLink } from './cid-to-pblink.js' -import { DoesNotExistError, InvalidParametersError } from './errors.js' import type { Blocks } from '@helia/interface/blocks' import type { AbortOptions } from '@libp2p/interfaces' import type { CID } from 'multiformats/cid' diff --git a/packages/unixfs/src/commands/utils/errors.ts b/packages/unixfs/src/errors.ts similarity index 100% rename from packages/unixfs/src/commands/utils/errors.ts rename to packages/unixfs/src/errors.ts From 60514b8d5aec788f59603148fd22fe1ab405fe9c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 7 Jun 2023 10:53:20 +0000 Subject: [PATCH 097/253] chore(release): 1.2.4 [skip ci] ## [@helia/unixfs-v1.2.4](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.3...@helia/unixfs-v1.2.4) (2023-06-07) ### Bug Fixes * export unixfs errors ([#50](https://github.com/ipfs/helia-unixfs/issues/50)) ([8426d65](https://github.com/ipfs/helia-unixfs/commit/8426d650ae4645b7b975331c5fd02f56e390cab6)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 4682a4b4..29dcfdd2 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.2.4](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.3...@helia/unixfs-v1.2.4) (2023-06-07) + + +### Bug Fixes + +* export unixfs errors ([#50](https://github.com/ipfs/helia-unixfs/issues/50)) ([8426d65](https://github.com/ipfs/helia-unixfs/commit/8426d650ae4645b7b975331c5fd02f56e390cab6)) + ## [@helia/unixfs-v1.2.3](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.2...@helia/unixfs-v1.2.3) (2023-06-07) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 583831c3..281a37d7 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.2.3", + "version": "1.2.4", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 444c8bd0dd40d8cad7ca12f3fbffaaf19f8e75fc Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 7 Jun 2023 15:34:09 +0100 Subject: [PATCH 098/253] feat: add offline option to all operations (#51) Adds an `offline` option that means Helia won't go to the network when blocks are missing, instead throwing `ERR_NOT_FOUND`. --- packages/unixfs/src/commands/chmod.ts | 2 +- packages/unixfs/src/commands/touch.ts | 2 +- .../unixfs/src/commands/utils/add-link.ts | 4 +- .../commands/utils/is-over-shard-threshold.ts | 11 +++-- .../unixfs/src/commands/utils/remove-link.ts | 2 +- packages/unixfs/src/index.ts | 48 +++++++++++++++++++ packages/unixfs/test/cat.spec.ts | 12 +++++ packages/unixfs/test/chmod.spec.ts | 12 +++++ packages/unixfs/test/cp.spec.ts | 13 +++++ packages/unixfs/test/ls.spec.ts | 14 ++++++ packages/unixfs/test/mkdir.spec.ts | 13 +++++ packages/unixfs/test/rm.spec.ts | 12 +++++ packages/unixfs/test/stat.spec.ts | 12 +++++ packages/unixfs/test/touch.spec.ts | 13 +++++ 14 files changed, 160 insertions(+), 10 deletions(-) diff --git a/packages/unixfs/src/commands/chmod.ts b/packages/unixfs/src/commands/chmod.ts index fc72c57a..e9d04ec7 100644 --- a/packages/unixfs/src/commands/chmod.ts +++ b/packages/unixfs/src/commands/chmod.ts @@ -104,7 +104,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options return updatePathCids(root.cid, resolved, blockstore, opts) } - const block = await blockstore.get(resolved.cid) + const block = await blockstore.get(resolved.cid, options) let metadata: UnixFS let links: PBLink[] = [] diff --git a/packages/unixfs/src/commands/touch.ts b/packages/unixfs/src/commands/touch.ts index 1cdae6de..05b3aead 100644 --- a/packages/unixfs/src/commands/touch.ts +++ b/packages/unixfs/src/commands/touch.ts @@ -108,7 +108,7 @@ export async function touch (cid: CID, blockstore: Blocks, options: Partial, block const result = await addToDirectory(parent, child, blockstore, options) - if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) { + if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options)) { log('converting directory to sharded directory') const converted = await convertToShardedDirectory(result, blockstore) result.cid = converted.cid - result.node = dagPB.decode(await blockstore.get(converted.cid)) + result.node = dagPB.decode(await blockstore.get(converted.cid, options)) } return result diff --git a/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts index c14741dc..312b8ec9 100644 --- a/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts +++ b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts @@ -3,6 +3,7 @@ import { UnixFS } from 'ipfs-unixfs' import { CID_V0, CID_V1 } from './dir-sharded.js' import type { Blocks } from '@helia/interface/blocks' import type { PBNode } from '@ipld/dag-pb' +import type { AbortOptions } from '@libp2p/interfaces' /** * Estimate node size only based on DAGLink name and CID byte lengths @@ -10,7 +11,7 @@ import type { PBNode } from '@ipld/dag-pb' * * If the node is a hamt sharded directory the calculation is based on if it was a regular directory. */ -export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number): Promise { +export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number, options: AbortOptions): Promise { if (node.Data == null) { throw new Error('DagPB node had no data') } @@ -21,7 +22,7 @@ export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, th if (unixfs.type === 'directory') { size = estimateNodeSize(node) } else if (unixfs.type === 'hamt-sharded-directory') { - size = await estimateShardSize(node, 0, threshold, blockstore) + size = await estimateShardSize(node, 0, threshold, blockstore, options) } else { throw new Error('Can only estimate the size of directories or shards') } @@ -42,7 +43,7 @@ function estimateNodeSize (node: PBNode): number { return size } -async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks): Promise { +async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks, options: AbortOptions): Promise { if (current > max) { return max } @@ -67,10 +68,10 @@ async function estimateShardSize (node: PBNode, current: number, max: number, bl current += link.Hash.bytes.byteLength if (link.Hash.code === dagPb.code) { - const block = await blockstore.get(link.Hash) + const block = await blockstore.get(link.Hash, options) const node = dagPb.decode(block) - current += await estimateShardSize(node, current, max, blockstore) + current += await estimateShardSize(node, current, max, blockstore, options) } } diff --git a/packages/unixfs/src/commands/utils/remove-link.ts b/packages/unixfs/src/commands/utils/remove-link.ts index a39256f4..23542486 100644 --- a/packages/unixfs/src/commands/utils/remove-link.ts +++ b/packages/unixfs/src/commands/utils/remove-link.ts @@ -41,7 +41,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B const result = await removeFromShardedDirectory(parent, name, blockstore, options) - if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) { + if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options))) { log('converting shard to flat directory %c', parent.cid) return convertToFlatDirectory(result, blockstore, options) diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index f2fc4855..11f341c1 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -80,6 +80,12 @@ export interface CatOptions extends AbortOptions, ProgressOptions { * An optional path to allow reading files inside directories */ path?: string + + /** + * If true, do not perform any network operations and throw if blocks are + * missing from the local store. (default: false) + */ + offline?: boolean } /** @@ -102,6 +108,12 @@ export interface ChmodOptions extends AbortOptions, ProgressOptions { * Stop reading the directory contents after this many directory entries */ length?: number + + /** + * If true, do not perform any network operations and throw if blocks are + * missing from the local store. (default: false) + */ + offline?: boolean } /** @@ -171,6 +195,12 @@ export interface MkdirOptions extends AbortOptions, ProgressOptions { * An optional path to allow statting paths inside directories */ path?: string + + /** + * If true, do not perform any network operations and throw if blocks are + * missing from the local store. (default: false) + */ + offline?: boolean } /** @@ -275,6 +317,12 @@ export interface TouchOptions extends AbortOptions, ProgressOptions { .with.property('code', 'ERR_NOT_A_FILE') }) + it('refuses to read missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(drain(fs.cat(cid, { + offline: true + }))).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + it('reads file from inside a sharded directory', async () => { const dirCid = await createShardedDirectory(blockstore) const fileCid = await fs.addBytes(smallFile) diff --git a/packages/unixfs/test/chmod.spec.ts b/packages/unixfs/test/chmod.spec.ts index a584cfd6..e0fdc6b4 100644 --- a/packages/unixfs/test/chmod.spec.ts +++ b/packages/unixfs/test/chmod.spec.ts @@ -89,4 +89,16 @@ describe('chmod', () => { expect(updatedMode).to.not.equal(originalMode) expect(updatedMode).to.equal(0o777) }) + + it('refuses to chmod missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(fs.chmod(cid, 0o777, { + offline: true + })).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) diff --git a/packages/unixfs/test/cp.spec.ts b/packages/unixfs/test/cp.spec.ts index 4527fafc..53f18321 100644 --- a/packages/unixfs/test/cp.spec.ts +++ b/packages/unixfs/test/cp.spec.ts @@ -10,6 +10,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' +import { smallFile } from './fixtures/files.js' import type { Blockstore } from 'interface-blockstore' describe('cp', () => { @@ -179,4 +180,16 @@ describe('cp', () => { expect(finalDirCid).to.eql(containingDirCid, 'adding a file to the imported dir did not result in the same CID') }) + + it('refuses to copy missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(fs.cp(cid, cid, 'file.txt', { + offline: true + })).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) diff --git a/packages/unixfs/test/ls.spec.ts b/packages/unixfs/test/ls.spec.ts index 8c626101..2d5b680f 100644 --- a/packages/unixfs/test/ls.spec.ts +++ b/packages/unixfs/test/ls.spec.ts @@ -3,8 +3,10 @@ import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' import all from 'it-all' +import drain from 'it-drain' import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { smallFile } from './fixtures/files.js' import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' @@ -118,4 +120,16 @@ describe('ls', () => { expect(files.length).to.equal(1) expect(files.filter(file => file.name === fileName)).to.be.ok() }) + + it('refuses to list missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(drain(fs.ls(cid, { + offline: true + }))).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) diff --git a/packages/unixfs/test/mkdir.spec.ts b/packages/unixfs/test/mkdir.spec.ts index a2cf6de5..ce7e7dde 100644 --- a/packages/unixfs/test/mkdir.spec.ts +++ b/packages/unixfs/test/mkdir.spec.ts @@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core' import all from 'it-all' import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { smallFile } from './fixtures/files.js' import type { Blockstore } from 'interface-blockstore' import type { Mtime } from 'ipfs-unixfs' import type { CID } from 'multiformats/cid' @@ -116,4 +117,16 @@ describe('mkdir', () => { path: dirName })).to.eventually.have.nested.property('unixfs.type', 'directory') }) + + it('refuses to mkdir with missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(fs.mkdir(cid, 'dir', { + offline: true + })).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) diff --git a/packages/unixfs/test/rm.spec.ts b/packages/unixfs/test/rm.spec.ts index 181b2dcd..8b6227b5 100644 --- a/packages/unixfs/test/rm.spec.ts +++ b/packages/unixfs/test/rm.spec.ts @@ -217,4 +217,16 @@ describe('rm', () => { expect(containingDirCid).to.eql(importerCid) }) + + it('refuses to rm missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(fs.rm(cid, 'dir', { + offline: true + })).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) diff --git a/packages/unixfs/test/stat.spec.ts b/packages/unixfs/test/stat.spec.ts index a87a3bec..6b50d034 100644 --- a/packages/unixfs/test/stat.spec.ts +++ b/packages/unixfs/test/stat.spec.ts @@ -197,4 +197,16 @@ describe('stat', function () { expect(stats.type).to.equal('file') expect(stats.fileSize).to.equal(4n) }) + + it('refuses to stat missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(fs.stat(cid, { + offline: true + })).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) diff --git a/packages/unixfs/test/touch.spec.ts b/packages/unixfs/test/touch.spec.ts index 8b9e4ab9..d4296727 100644 --- a/packages/unixfs/test/touch.spec.ts +++ b/packages/unixfs/test/touch.spec.ts @@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core' import delay from 'delay' import { unixfs, type UnixFS } from '../src/index.js' import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { smallFile } from './fixtures/files.js' import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' @@ -145,4 +146,16 @@ describe('.files.touch', () => { .that.satisfies((s: bigint) => s > seconds) } }) + + it('refuses to touch missing blocks', async () => { + const cid = await fs.addBytes(smallFile) + + await blockstore.delete(cid) + expect(blockstore.has(cid)).to.be.false() + + await expect(fs.touch(cid, { + offline: true + })).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) }) From 9b4707efe111ab5bae13683f7587f516f74307e1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 7 Jun 2023 14:45:00 +0000 Subject: [PATCH 099/253] chore(release): 1.3.0 [skip ci] ## [@helia/unixfs-v1.3.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.4...@helia/unixfs-v1.3.0) (2023-06-07) ### Features * add offline option to all operations ([#51](https://github.com/ipfs/helia-unixfs/issues/51)) ([444c8bd](https://github.com/ipfs/helia-unixfs/commit/444c8bd0dd40d8cad7ca12f3fbffaaf19f8e75fc)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 29dcfdd2..4a4bcd91 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.3.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.4...@helia/unixfs-v1.3.0) (2023-06-07) + + +### Features + +* add offline option to all operations ([#51](https://github.com/ipfs/helia-unixfs/issues/51)) ([444c8bd](https://github.com/ipfs/helia-unixfs/commit/444c8bd0dd40d8cad7ca12f3fbffaaf19f8e75fc)) + ## [@helia/unixfs-v1.2.4](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.3...@helia/unixfs-v1.2.4) (2023-06-07) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 281a37d7..997794cd 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.2.4", + "version": "1.3.0", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 237c58898b4ccda7f2a9230fba57747b1ffabc24 Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Mon, 12 Jun 2023 10:10:53 +0200 Subject: [PATCH 100/253] ci: update Unified CI configuration (#2) --- .github/workflows/automerge.yml | 3 +++ .github/workflows/js-test-and-release.yml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index d57c2a02..3833fc22 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -1,3 +1,6 @@ +# File managed by web3-bot. DO NOT EDIT. +# See https://github.com/protocol/.github/ for details. + name: Automerge on: [ pull_request ] diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 51686414..ba514373 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,3 +1,6 @@ +# File managed by web3-bot. DO NOT EDIT. +# See https://github.com/protocol/.github/ for details. + name: test & maybe release on: push: @@ -17,6 +20,7 @@ jobs: - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present lint - run: npm run --if-present dep-check + - run: npm run --if-present doc-check test-node: needs: check From 6970cd1ff8df7c75c160c82aacd1904d7c0ee097 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Jun 2023 11:12:59 +0000 Subject: [PATCH 101/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..668bd07d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,21 +6,4 @@ on: jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 From b658042d2c0558e933ff66d25bb44e3cedc8fc79 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Jun 2023 11:13:53 +0000 Subject: [PATCH 102/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..668bd07d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,21 +6,4 @@ on: jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 From b9bc16d3da172fb27fdfd2135c6e753e3784f6d0 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Jun 2023 11:14:46 +0000 Subject: [PATCH 103/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..668bd07d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,21 +6,4 @@ on: jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 From e558424e930175351beb453af5dbc1ee975fd88e Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Jun 2023 11:15:57 +0000 Subject: [PATCH 104/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..668bd07d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,21 +6,4 @@ on: jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 From 7559fb0414bae7e23689b415a0586dac125a8ad9 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Jun 2023 11:16:03 +0000 Subject: [PATCH 105/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..668bd07d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,21 +6,4 @@ on: jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 From 9de4a05390566f0f9d103c486a6894ca6fad63a4 Mon Sep 17 00:00:00 2001 From: GitHub Date: Mon, 19 Jun 2023 12:28:30 +0000 Subject: [PATCH 106/253] chore: Update .github/workflows/stale.yml [skip ci] From 54822ed49a5aaefe0f164ef8750a8aee129172bd Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 28 Jun 2023 08:49:18 +0000 Subject: [PATCH 107/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 668bd07d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,11 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: From cbb47f29a219548647cdd4d2f426a234bb20b958 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 28 Jun 2023 08:49:39 +0000 Subject: [PATCH 108/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 668bd07d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,11 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: From 26b5cd3aa7debe0b36fde325f1e0b6b3eacaac5b Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 28 Jun 2023 08:50:19 +0000 Subject: [PATCH 109/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 668bd07d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,11 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: From b103705ff2a1a3ee9e3947a790f25478c8d98d05 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 28 Jun 2023 08:50:23 +0000 Subject: [PATCH 110/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 668bd07d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,11 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: From 9cd521d2e5a6ac2e34b760897270de157496ccf0 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 28 Jun 2023 08:52:03 +0000 Subject: [PATCH 111/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 668bd07d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,11 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: From b490a6e35cb521c0c29d0f1382fc2e4b3b662b9c Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 30 Jun 2023 16:24:37 +0200 Subject: [PATCH 112/253] feat: add globSource and urlSource (#53) Ports `globSource` and `urlSource` from `ipfs-utils` for use with `unixfs.add` and `unixfs.addAll`. --- packages/unixfs/.aegir.js | 23 ++ packages/unixfs/package.json | 12 +- packages/unixfs/src/index.ts | 15 + packages/unixfs/src/utils/glob-source.ts | 92 ++++++ packages/unixfs/src/utils/url-source.ts | 35 +++ packages/unixfs/test/add.spec.ts | 20 +- .../another-dir/another-nested-dir/other.txt | 0 .../test/fixtures/files/another-dir/hello.txt | 1 + .../test/fixtures/files/dir/.hidden.txt | 0 .../unixfs/test/fixtures/files/dir/file-1.txt | 0 .../unixfs/test/fixtures/files/dir/file-2.js | 0 .../unixfs/test/fixtures/files/dir/file-3.css | 0 .../fixtures/files/dir/nested-dir/other.txt | 0 .../unixfs/test/fixtures/files/file-0.html | 0 .../unixfs/test/utils/glob-source.spec.ts | 291 ++++++++++++++++++ packages/unixfs/test/utils/url-source.spec.ts | 21 ++ 16 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 packages/unixfs/.aegir.js create mode 100644 packages/unixfs/src/utils/glob-source.ts create mode 100644 packages/unixfs/src/utils/url-source.ts create mode 100644 packages/unixfs/test/fixtures/files/another-dir/another-nested-dir/other.txt create mode 100644 packages/unixfs/test/fixtures/files/another-dir/hello.txt create mode 100644 packages/unixfs/test/fixtures/files/dir/.hidden.txt create mode 100644 packages/unixfs/test/fixtures/files/dir/file-1.txt create mode 100755 packages/unixfs/test/fixtures/files/dir/file-2.js create mode 100644 packages/unixfs/test/fixtures/files/dir/file-3.css create mode 100644 packages/unixfs/test/fixtures/files/dir/nested-dir/other.txt create mode 100644 packages/unixfs/test/fixtures/files/file-0.html create mode 100644 packages/unixfs/test/utils/glob-source.spec.ts create mode 100644 packages/unixfs/test/utils/url-source.spec.ts diff --git a/packages/unixfs/.aegir.js b/packages/unixfs/.aegir.js new file mode 100644 index 00000000..90ff44c1 --- /dev/null +++ b/packages/unixfs/.aegir.js @@ -0,0 +1,23 @@ +import EchoServer from 'aegir/echo-server' +import { format } from 'iso-url' + +export default { + test: { + async before (options) { + let echoServer = new EchoServer() + await echoServer.start() + const { address, port } = echoServer.server.address() + let hostname = address + if(options.runner === 'react-native-android') { + hostname = '10.0.2.2' + } + return { + echoServer, + env: { ECHO_SERVER : format({ protocol: 'http:', hostname, port })} + } + }, + async after (options, before) { + await before.echoServer.stop() + } + } +} diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 997794cd..9b8d8dc0 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -165,6 +165,7 @@ "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^13.1.0", "ipfs-unixfs-importer": "^15.1.0", + "it-glob": "^2.0.4", "it-last": "^3.0.1", "it-pipe": "^3.0.1", "merge-options": "^3.0.4", @@ -176,13 +177,22 @@ "aegir": "^39.0.8", "blockstore-core": "^4.0.1", "delay": "^6.0.0", + "iso-url": "^1.2.1", "it-all": "^3.0.1", "it-drain": "^3.0.1", "it-first": "^3.0.1", "it-to-buffer": "^4.0.1", - "uint8arrays": "^4.0.3" + "uint8arrays": "^4.0.3", + "wherearewe": "^2.0.1" }, "typedoc": { "entryPoint": "./src/index.ts" + }, + "browser": { + "./dist/utils/glob-source.js": false, + "node:fs": false, + "node:fs/promises": false, + "node:path": false, + "node:url": false } } diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index 11f341c1..300a8862 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -29,6 +29,18 @@ * console.info(entry) * } * ``` + * + * @example + * + * Recursively adding a directory (Node.js-compatibly environments only): + * + * ```typescript + * import { globSource } from '@helia/unixfs' + * + * for await (const entry of fs.addAll(globSource('path/to/containing/dir', 'glob-pattern'))) { + * console.info(entry) + * } + * ``` */ import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js' @@ -607,3 +619,6 @@ class DefaultUnixFS implements UnixFS { export function unixfs (helia: { blockstore: Blocks }): UnixFS { return new DefaultUnixFS(helia) } + +export { globSource } from './utils/glob-source.js' +export { urlSource } from './utils/url-source.js' diff --git a/packages/unixfs/src/utils/glob-source.ts b/packages/unixfs/src/utils/glob-source.ts new file mode 100644 index 00000000..a834cc3a --- /dev/null +++ b/packages/unixfs/src/utils/glob-source.ts @@ -0,0 +1,92 @@ +import fs from 'node:fs' +import fsp from 'node:fs/promises' +import Path from 'node:path' +import glob from 'it-glob' +import { InvalidParametersError } from '../errors.js' +import type { MtimeLike } from 'ipfs-unixfs' +import type { ImportCandidateStream } from 'ipfs-unixfs-importer' + +export interface GlobSourceOptions { + /** + * Include .dot files in matched paths + */ + hidden?: boolean + + /** + * follow symlinks + */ + followSymlinks?: boolean + + /** + * Preserve mode + */ + preserveMode?: boolean + + /** + * Preserve mtime + */ + preserveMtime?: boolean + + /** + * mode to use - if preserveMode is true this will be ignored + */ + mode?: number + + /** + * mtime to use - if preserveMtime is true this will be ignored + */ + mtime?: MtimeLike +} + +export interface GlobSourceResult { + path: string + content: AsyncIterable | undefined + mode: number | undefined + mtime: MtimeLike | undefined +} + +/** + * Create an async iterator that yields paths that match requested glob pattern + */ +export async function * globSource (cwd: string, pattern: string, options: GlobSourceOptions = {}): ImportCandidateStream { + if (typeof pattern !== 'string') { + throw new InvalidParametersError('Pattern must be a string') + } + + if (!Path.isAbsolute(cwd)) { + cwd = Path.resolve(process.cwd(), cwd) + } + + const globOptions = Object.assign({}, { + nodir: false, + realpath: false, + absolute: true, + dot: Boolean(options.hidden), + follow: options.followSymlinks != null ? options.followSymlinks : true + }) + + for await (const p of glob(cwd, pattern, globOptions)) { + const stat = await fsp.stat(p) + + let mode = options.mode + + if (options.preserveMode === true) { + mode = stat.mode + } + + let mtime = options.mtime + + if (options.preserveMtime === true) { + mtime = stat.mtime + } + + yield { + path: toPosix(p.replace(cwd, '')), + content: stat.isFile() ? fs.createReadStream(p) : undefined, + mode, + mtime + } + } +} + +const toPosix = (path: string): string => path.replace(/\\/g, '/') diff --git a/packages/unixfs/src/utils/url-source.ts b/packages/unixfs/src/utils/url-source.ts new file mode 100644 index 00000000..501bc668 --- /dev/null +++ b/packages/unixfs/src/utils/url-source.ts @@ -0,0 +1,35 @@ +import { UnknownError } from '../errors.js' +import type { FileCandidate } from 'ipfs-unixfs-importer' + +export function urlSource (url: URL, options?: RequestInit): FileCandidate> { + return { + path: decodeURIComponent(new URL(url).pathname.split('/').pop() ?? ''), + content: readURLContent(url, options) + } +} + +async function * readURLContent (url: URL, options?: RequestInit): AsyncGenerator { + const response = await globalThis.fetch(url, options) + + if (response.body == null) { + throw new UnknownError('HTTP response did not have a body') + } + + const reader = response.body.getReader() + + try { + while (true) { + const { done, value } = await reader.read() + + if (done) { + return + } + + if (value != null) { + yield value + } + } + } finally { + reader.releaseLock() + } +} diff --git a/packages/unixfs/test/add.spec.ts b/packages/unixfs/test/add.spec.ts index 384f8819..dbc9e9be 100644 --- a/packages/unixfs/test/add.spec.ts +++ b/packages/unixfs/test/add.spec.ts @@ -3,7 +3,9 @@ import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' import all from 'it-all' -import { unixfs, type UnixFS } from '../src/index.js' +import last from 'it-last' +import { isNode } from 'wherearewe' +import { globSource, unixfs, urlSource, type UnixFS } from '../src/index.js' import type { Blockstore } from 'interface-blockstore' describe('addAll', () => { @@ -29,6 +31,16 @@ describe('addAll', () => { expect(output[0].cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') expect(output[1].cid.toString()).to.equal('bafkreidmuy2n45xj3cdknzprtzo2uvgm3hak6mzy5sllxty457agsftd34') }) + + it('recursively adds a directory', async function () { + if (!isNode) { + return this.skip() + } + + const res = await last(fs.addAll(globSource('./test/fixtures', 'files/**/*'))) + + expect(res?.cid.toString()).to.equal('bafybeievhllpjjjbyg53g74wcl5hckdccjjj7zgtexqcacjegoduegnkyu') + }) }) describe('addBytes', () => { @@ -82,6 +94,12 @@ describe('addFile', () => { expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') }) + + it('adds a file from a URL', async () => { + const cid = await fs.addFile(urlSource(new URL(`${process.env.ECHO_SERVER}/download?data=hello-world`))) + + expect(cid.toString()).to.equal('bafkreifpuj5ujvb3aku75ja5cphnylsac3h47b6f3p4zbzmtm2nkrtrinu') + }) }) describe('addDirectory', () => { diff --git a/packages/unixfs/test/fixtures/files/another-dir/another-nested-dir/other.txt b/packages/unixfs/test/fixtures/files/another-dir/another-nested-dir/other.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/unixfs/test/fixtures/files/another-dir/hello.txt b/packages/unixfs/test/fixtures/files/another-dir/hello.txt new file mode 100644 index 00000000..b6fc4c62 --- /dev/null +++ b/packages/unixfs/test/fixtures/files/another-dir/hello.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/packages/unixfs/test/fixtures/files/dir/.hidden.txt b/packages/unixfs/test/fixtures/files/dir/.hidden.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/unixfs/test/fixtures/files/dir/file-1.txt b/packages/unixfs/test/fixtures/files/dir/file-1.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/unixfs/test/fixtures/files/dir/file-2.js b/packages/unixfs/test/fixtures/files/dir/file-2.js new file mode 100755 index 00000000..e69de29b diff --git a/packages/unixfs/test/fixtures/files/dir/file-3.css b/packages/unixfs/test/fixtures/files/dir/file-3.css new file mode 100644 index 00000000..e69de29b diff --git a/packages/unixfs/test/fixtures/files/dir/nested-dir/other.txt b/packages/unixfs/test/fixtures/files/dir/nested-dir/other.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/unixfs/test/fixtures/files/file-0.html b/packages/unixfs/test/fixtures/files/file-0.html new file mode 100644 index 00000000..e69de29b diff --git a/packages/unixfs/test/utils/glob-source.spec.ts b/packages/unixfs/test/utils/glob-source.spec.ts new file mode 100644 index 00000000..cdf00fab --- /dev/null +++ b/packages/unixfs/test/utils/glob-source.spec.ts @@ -0,0 +1,291 @@ +/* eslint-env mocha */ + +import fs from 'node:fs' +import Path from 'node:path' +import { fileURLToPath } from 'node:url' +import { expect } from 'aegir/chai' +import all from 'it-all' +import { isNode } from 'wherearewe' +import { globSource } from '../../src/utils/glob-source.js' + +function fixtureDir (): string { + const filename = fileURLToPath(import.meta.url) + const dirname = Path.dirname(filename) + + return Path.resolve(Path.join(dirname, '..', '..', '..', 'test', 'fixtures', 'files')) +} + +function fixture (file: string): string { + return Path.resolve(Path.join(fixtureDir(), file)) +} + +function findMode (file: string): number { + return fs.statSync(fixture(file)).mode +} + +function findMtime (file: string): Date { + return fs.statSync(fixture(file)).mtime +} + +describe('glob-source', () => { + it('single file, relative path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource('./test/fixtures/files', 'file-0.html')) + + expect(result.length).to.equal(1) + expect(result[0].path).to.equal('/file-0.html') + }) + + it('single file, absolute path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), 'file-0.html')) + + expect(result.length).to.equal(1) + expect(result[0].path).to.equal('/file-0.html') + }) + + it('directory, relative path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), 'dir/**/*')) + + expect(result).to.have.lengthOf(5) + expect(result).to.containSubset([{ + path: '/dir/file-1.txt' + }, { + path: '/dir/file-2.js' + }, { + path: '/dir/file-3.css' + }, { + path: '/dir/nested-dir' + }, { + path: '/dir/nested-dir/other.txt' + }]) + }) + + it('multiple directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), '{dir/nested-dir,another-dir/another-nested-dir}/**/*')) + + expect(result).to.have.lengthOf(2) + expect(result).to.containSubset([{ + path: '/dir/nested-dir/other.txt' + }, { + path: '/another-dir/another-nested-dir/other.txt' + }]) + }) + + it('directory, hidden files', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), 'dir/**/*', { + hidden: true + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.containSubset([{ + path: '/dir/.hidden.txt' + }]) + }) + + it('directory, ignore files', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), 'dir/**/!(file-1.txt)')) + + expect(result).to.have.lengthOf(4) + expect(result).to.not.containSubset([{ + path: '/dir/file-1.txt' + }]) + }) + + it('multiple paths', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('dir'), 'file-{1,2}.*')) + + expect(result).to.have.lengthOf(2) + expect(result).to.not.containSubset([{ + path: '/dir/file-1.txt' + }, { + path: '/dir/file-2.js' + }]) + }) + + it('preserves mode for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), '{dir,dir/**/*}', { + preserveMode: true + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.containSubset([{ + path: '/dir', + mode: findMode('/dir') + }, { + path: '/dir/file-1.txt', + mode: findMode('/dir/file-1.txt') + }, { + path: '/dir/file-2.js', + mode: findMode('/dir/file-2.js') + }, { + path: '/dir/file-3.css', + mode: findMode('/dir/file-3.css') + }, { + path: '/dir/nested-dir', + mode: findMode('/dir/nested-dir') + }, { + path: '/dir/nested-dir/other.txt', + mode: findMode('/dir/nested-dir/other.txt') + }]) + }) + + it('overrides mode for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), '{dir,dir/**/*}', { + mode: 5 + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.containSubset([{ + path: '/dir', + mode: 5 + }, { + path: '/dir/file-1.txt', + mode: 5 + }, { + path: '/dir/file-2.js', + mode: 5 + }, { + path: '/dir/file-3.css', + mode: 5 + }, { + path: '/dir/nested-dir', + mode: 5 + }, { + path: '/dir/nested-dir/other.txt', + mode: 5 + }]) + }) + + it('preserves mtime for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), '{dir,dir/**/*}', { + preserveMtime: true + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.containSubset([{ + path: '/dir', + mtime: findMtime('/dir') + }, { + path: '/dir/file-1.txt', + mtime: findMtime('/dir/file-1.txt') + }, { + path: '/dir/file-2.js', + mtime: findMtime('/dir/file-2.js') + }, { + path: '/dir/file-3.css', + mtime: findMtime('/dir/file-3.css') + }, { + path: '/dir/nested-dir', + mtime: findMtime('/dir/nested-dir') + }, { + path: '/dir/nested-dir/other.txt', + mtime: findMtime('/dir/nested-dir/other.txt') + }]) + }) + + it('overrides mtime for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixtureDir(), '{dir,dir/**/*}', { + mtime: new Date(5) + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.containSubset([{ + path: '/dir', + mtime: new Date(5) + }, { + path: '/dir/file-1.txt', + mtime: new Date(5) + }, { + path: '/dir/file-2.js', + mtime: new Date(5) + }, { + path: '/dir/file-3.css', + mtime: new Date(5) + }, { + path: '/dir/nested-dir', + mtime: new Date(5) + }, { + path: '/dir/nested-dir/other.txt', + mtime: new Date(5) + }]) + }) + + it('overrides mtime for file with secs/nsecs', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('dir'), 'file-1.txt', { + mtime: { secs: 5n, nsecs: 0 } + })) + + expect(result).to.have.deep.nested.property('[0].mtime', { secs: 5n, nsecs: 0 }) + }) + + it('overrides mtime for file with hrtime', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('dir'), 'file-1.txt', { + mtime: [5, 0] + })) + + expect(result).to.have.deep.nested.property('[0].mtime', [5, 0]) + }) + + it('overrides mtime for file with UnixFS timespec', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('dir'), 'file-1.txt', { + mtime: { Seconds: 5, FractionalNanoseconds: 0 } + })) + + expect(result).to.have.deep.nested.property('[0].mtime', { Seconds: 5, FractionalNanoseconds: 0 }) + }) +}) diff --git a/packages/unixfs/test/utils/url-source.spec.ts b/packages/unixfs/test/utils/url-source.spec.ts new file mode 100644 index 00000000..5f5eb673 --- /dev/null +++ b/packages/unixfs/test/utils/url-source.spec.ts @@ -0,0 +1,21 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import all from 'it-all' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { urlSource } from '../../src/utils/url-source.js' + +describe('url-source', function () { + it('can get url content', async function () { + const content = 'foo' + const file = urlSource(new URL(`${process.env.ECHO_SERVER}/download?data=${content}`)) + + expect(file).to.have.property('path', 'download') + + if (file.content != null) { + await expect(all(file.content)).to.eventually.deep.equal([uint8ArrayFromString(content)]) + } else { + throw new Error('empty response') + } + }) +}) From e2184ef889f87add7adfc0432cf17f0a854fc9b4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 30 Jun 2023 14:31:28 +0000 Subject: [PATCH 113/253] chore(release): 1.4.0 [skip ci] ## [@helia/unixfs-v1.4.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.3.0...@helia/unixfs-v1.4.0) (2023-06-30) ### Features * add globSource and urlSource ([#53](https://github.com/ipfs/helia-unixfs/issues/53)) ([b490a6e](https://github.com/ipfs/helia-unixfs/commit/b490a6e35cb521c0c29d0f1382fc2e4b3b662b9c)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 4a4bcd91..512160a1 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.4.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.3.0...@helia/unixfs-v1.4.0) (2023-06-30) + + +### Features + +* add globSource and urlSource ([#53](https://github.com/ipfs/helia-unixfs/issues/53)) ([b490a6e](https://github.com/ipfs/helia-unixfs/commit/b490a6e35cb521c0c29d0f1382fc2e4b3b662b9c)) + ## [@helia/unixfs-v1.3.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.2.4...@helia/unixfs-v1.3.0) (2023-06-07) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 9b8d8dc0..5993a720 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.3.0", + "version": "1.4.0", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 356797a9493c7753178b5f343962951bc9cd3052 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 3 Jul 2023 14:08:56 -0700 Subject: [PATCH 114/253] docs: fs already defined in example (#1) Fixes non-compileable code example where we set both `node:fs` import and `unixfs(helia)` to a variable `fs`. --- packages/car/src/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/car/src/index.ts b/packages/car/src/index.ts index 63812274..8640a4b4 100644 --- a/packages/car/src/index.ts +++ b/packages/car/src/index.ts @@ -15,7 +15,7 @@ * import { car } from '@helia/car' * import { CarWriter } from '@ipld/car' * import { Readable } from 'node:stream' - * import fs from 'node:fs' + * import nodeFs from 'node:fs' * * const helia = createHelia({ * // ... helia config @@ -30,7 +30,7 @@ * const { writer, out } = await CarWriter.create(cid) * * // `out` needs to be directed somewhere, see the @ipld/car docs for more information - * Readable.from(out).pipe(fs.createWriteStream('example.car')) + * Readable.from(out).pipe(nodeFs.createWriteStream('example.car')) * * // write the DAG behind `cid` into the writer * await c.export(cid, writer) @@ -44,14 +44,14 @@ * import { car } from '@helia/car' * import { CarReader } from '@ipld/car' * import { Readable } from 'node:stream' - * import fs from 'node:fs' + * import nodeFs from 'node:fs' * * const helia = createHelia({ * // ... helia config * }) * * // import the car - * const inStream = fs.createReadStream('example.car') + * const inStream = nodeFs.createReadStream('example.car') * const reader = await CarReader.fromIterable(inStream) * * await c.import(reader) From fd0f33b2a66e2840b5a03f27a48240b3c5d2b67e Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 25 Jul 2023 07:43:37 +0100 Subject: [PATCH 115/253] fix: correct browser override path for glob-source (#60) The file is imported from `dist/src/utils` --- packages/unixfs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 5993a720..8dba7776 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -189,7 +189,7 @@ "entryPoint": "./src/index.ts" }, "browser": { - "./dist/utils/glob-source.js": false, + "./dist/src/utils/glob-source.js": false, "node:fs": false, "node:fs/promises": false, "node:path": false, From b7625c3426380e63052968b1476e2d689c9213de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=94=BF=E4=B8=9C?= <37199800+BetterWorld-Liuser@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:44:45 +0800 Subject: [PATCH 116/253] docs: fix typos in example code (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit path: './foo.txt' miss a comma, ’entry‘ was written as "etnry"... --- packages/unixfs/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index 300a8862..d6501b4f 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -401,7 +401,7 @@ export interface UnixFS { * * ```typescript * const cid = await fs.addFile({ - * path: './foo.txt' + * path: './foo.txt', * content: Uint8Array.from([0, 1, 2, 3]), * mode: 0x755, * mtime: { @@ -482,7 +482,7 @@ export interface UnixFS { * * ```typescript * for await (const entry of fs.ls(directoryCid)) { - * console.info(etnry) + * console.info(entry) * } * ``` */ From c5026ae67ee49c7c14806a68dd1b7948711a63db Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 25 Jul 2023 06:53:20 +0000 Subject: [PATCH 117/253] chore(release): 1.4.1 [skip ci] ## [@helia/unixfs-v1.4.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.0...@helia/unixfs-v1.4.1) (2023-07-25) ### Bug Fixes * correct browser override path for glob-source ([#60](https://github.com/ipfs/helia-unixfs/issues/60)) ([fd0f33b](https://github.com/ipfs/helia-unixfs/commit/fd0f33b2a66e2840b5a03f27a48240b3c5d2b67e)) ### Documentation * fix typos in example code ([#57](https://github.com/ipfs/helia-unixfs/issues/57)) ([b7625c3](https://github.com/ipfs/helia-unixfs/commit/b7625c3426380e63052968b1476e2d689c9213de)) --- packages/unixfs/CHANGELOG.md | 12 ++++++++++++ packages/unixfs/package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 512160a1..8ec7d3d5 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,15 @@ +## [@helia/unixfs-v1.4.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.0...@helia/unixfs-v1.4.1) (2023-07-25) + + +### Bug Fixes + +* correct browser override path for glob-source ([#60](https://github.com/ipfs/helia-unixfs/issues/60)) ([fd0f33b](https://github.com/ipfs/helia-unixfs/commit/fd0f33b2a66e2840b5a03f27a48240b3c5d2b67e)) + + +### Documentation + +* fix typos in example code ([#57](https://github.com/ipfs/helia-unixfs/issues/57)) ([b7625c3](https://github.com/ipfs/helia-unixfs/commit/b7625c3426380e63052968b1476e2d689c9213de)) + ## [@helia/unixfs-v1.4.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.3.0...@helia/unixfs-v1.4.0) (2023-06-30) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 8dba7776..80f4d7f0 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.4.0", + "version": "1.4.1", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 27b2b8cbd0ae140069acb414d14da9ed85d6bfe0 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 1 Aug 2023 18:16:31 +0100 Subject: [PATCH 118/253] chore: use ts-ignore for go-ipfs import (#72) The next version of npm-go-ipfs has types so the @ts-expect-error prevents using the pre-release version of npm-go-ipfs. --- packages/interop/test/fixtures/create-kubo.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 3871cfd4..518c8a68 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,5 +1,5 @@ - -// @ts-expect-error no types +/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */ +// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' From c19ee4626d389e74d33f281e66211007895e8a31 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 1 Aug 2023 18:16:36 +0100 Subject: [PATCH 119/253] chore: use ts-ignore for go-ipfs import (#68) The next version of npm-go-ipfs has types so the @ts-expect-error prevents using the pre-release version of npm-go-ipfs. --- packages/interop/test/fixtures/create-kubo.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 52ed76e1..f4612231 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,5 +1,6 @@ -// @ts-expect-error no types +/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */ +// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' From c5ebe429e350b6522a8d598bd8e5857bb1df80f6 Mon Sep 17 00:00:00 2001 From: web3-bot Date: Sun, 13 Aug 2023 18:56:32 +0200 Subject: [PATCH 120/253] chore: delete templates [skip ci] (#82) --- .github/workflows/automerge.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .github/workflows/automerge.yml diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index 3833fc22..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,11 +0,0 @@ -# File managed by web3-bot. DO NOT EDIT. -# See https://github.com/protocol/.github/ for details. - -name: Automerge -on: [ pull_request ] - -jobs: - automerge: - uses: protocol/.github/.github/workflows/automerge.yml@master - with: - job: 'automerge' From d7bca323e5ebc98bc18775f5b888a7304cd210ca Mon Sep 17 00:00:00 2001 From: web3-bot Date: Sun, 13 Aug 2023 18:56:33 +0200 Subject: [PATCH 121/253] chore: delete templates [skip ci] (#23) --- .github/workflows/automerge.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .github/workflows/automerge.yml diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index 3833fc22..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,11 +0,0 @@ -# File managed by web3-bot. DO NOT EDIT. -# See https://github.com/protocol/.github/ for details. - -name: Automerge -on: [ pull_request ] - -jobs: - automerge: - uses: protocol/.github/.github/workflows/automerge.yml@master - with: - job: 'automerge' From 84aec2bcea35dbf0126e235d2bae77ae3129cdfd Mon Sep 17 00:00:00 2001 From: web3-bot Date: Sun, 13 Aug 2023 19:13:44 +0200 Subject: [PATCH 122/253] chore: delete templates [skip ci] (#78) --- .github/workflows/automerge.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .github/workflows/automerge.yml diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index 3833fc22..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,11 +0,0 @@ -# File managed by web3-bot. DO NOT EDIT. -# See https://github.com/protocol/.github/ for details. - -name: Automerge -on: [ pull_request ] - -jobs: - automerge: - uses: protocol/.github/.github/workflows/automerge.yml@master - with: - job: 'automerge' From fd8f610529d36362e084105c724c4646004b316e Mon Sep 17 00:00:00 2001 From: web3-bot Date: Mon, 14 Aug 2023 08:21:26 +0200 Subject: [PATCH 123/253] chore: add or force update .github/workflows/js-test-and-release.yml (#79) --- .github/workflows/js-test-and-release.yml | 191 ++-------------------- 1 file changed, 16 insertions(+), 175 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 614a1d85..35d87d10 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,184 +1,25 @@ -# File managed by web3-bot. DO NOT EDIT. -# See https://github.com/protocol/.github/ for details. - name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + packages: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} From 379f6f06f0d7715789b0312f15debdef7880d22d Mon Sep 17 00:00:00 2001 From: web3-bot Date: Mon, 14 Aug 2023 08:21:28 +0200 Subject: [PATCH 124/253] chore: add or force update .github/workflows/js-test-and-release.yml (#83) --- .github/workflows/js-test-and-release.yml | 191 ++-------------------- 1 file changed, 16 insertions(+), 175 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 614a1d85..35d87d10 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,184 +1,25 @@ -# File managed by web3-bot. DO NOT EDIT. -# See https://github.com/protocol/.github/ for details. - name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + packages: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} From b5616f623a1b28855b70929cecb1e76e95eb2ad8 Mon Sep 17 00:00:00 2001 From: web3-bot Date: Mon, 14 Aug 2023 08:21:29 +0200 Subject: [PATCH 125/253] chore: add or force update .github/workflows/js-test-and-release.yml (#24) --- .github/workflows/js-test-and-release.yml | 192 ++-------------------- 1 file changed, 16 insertions(+), 176 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index ba514373..35d87d10 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,185 +1,25 @@ -# File managed by web3-bot. DO NOT EDIT. -# See https://github.com/protocol/.github/ for details. - name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - run: npm run --if-present doc-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + packages: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} From ea21951770bf14b8ccbd6cb41e5b0eaf4c2c6138 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 14 Aug 2023 07:04:01 +0000 Subject: [PATCH 126/253] chore(release): 1.0.1 [skip ci] ## [@helia/car-v1.0.1](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.0...@helia/car-v1.0.1) (2023-08-14) ### Documentation * fs already defined in example ([#1](https://github.com/ipfs/helia-car/issues/1)) ([356797a](https://github.com/ipfs/helia-car/commit/356797a9493c7753178b5f343962951bc9cd3052)) --- packages/car/CHANGELOG.md | 7 +++++++ packages/car/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/car/CHANGELOG.md b/packages/car/CHANGELOG.md index 8d80597f..c550554a 100644 --- a/packages/car/CHANGELOG.md +++ b/packages/car/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/car-v1.0.1](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.0...@helia/car-v1.0.1) (2023-08-14) + + +### Documentation + +* fs already defined in example ([#1](https://github.com/ipfs/helia-car/issues/1)) ([356797a](https://github.com/ipfs/helia-car/commit/356797a9493c7753178b5f343962951bc9cd3052)) + ## @helia/car-v1.0.0 (2023-06-07) diff --git a/packages/car/package.json b/packages/car/package.json index 3855696b..7d38bf43 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -1,6 +1,6 @@ { "name": "@helia/car", - "version": "1.0.0", + "version": "1.0.1", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", From a70f4eb982e377eeeeb6fd4a53f7baf40c09641b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 14 Aug 2023 16:52:02 +0100 Subject: [PATCH 127/253] feat: initial import --- .github/dependabot.yml | 11 + .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 182 ++++++ .github/workflows/stale.yml | 26 + .gitignore | 9 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 + README.md | 50 ++ package.json | 44 ++ packages/interop/.aegir.js | 45 ++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 + packages/interop/README.md | 38 ++ packages/interop/package.json | 77 +++ packages/interop/src/index.ts | 1 + .../test/fixtures/create-helia.browser.ts | 46 ++ .../interop/test/fixtures/create-helia.ts | 39 ++ packages/interop/test/fixtures/create-kubo.ts | 26 + packages/interop/test/index.spec.ts | 105 +++ packages/interop/tsconfig.json | 10 + packages/interop/typedoc.json | 5 + packages/mfs/LICENSE | 4 + packages/mfs/LICENSE-APACHE | 5 + packages/mfs/LICENSE-MIT | 19 + packages/mfs/README.md | 59 ++ packages/mfs/package.json | 167 +++++ packages/mfs/src/index.ts | 600 ++++++++++++++++++ packages/mfs/src/utils/basename.ts | 3 + packages/mfs/test/cat.spec.ts | 94 +++ packages/mfs/test/chmod.spec.ts | 90 +++ packages/mfs/test/cp.spec.ts | 189 ++++++ .../test/fixtures/create-sharded-directory.ts | 27 + .../fixtures/create-subsharded-directory.ts | 92 +++ packages/mfs/test/fixtures/files.ts | 5 + packages/mfs/test/fixtures/print-tree.ts | 35 + packages/mfs/test/ls.spec.ts | 179 ++++++ packages/mfs/test/mkdir.spec.ts | 113 ++++ packages/mfs/test/rm.spec.ts | 243 +++++++ packages/mfs/test/stat.spec.ts | 204 ++++++ packages/mfs/test/touch.spec.ts | 148 +++++ packages/mfs/tsconfig.json | 10 + packages/mfs/typedoc.json | 5 + 44 files changed, 3069 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 package.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/test/index.spec.ts create mode 100644 packages/interop/tsconfig.json create mode 100644 packages/interop/typedoc.json create mode 100644 packages/mfs/LICENSE create mode 100644 packages/mfs/LICENSE-APACHE create mode 100644 packages/mfs/LICENSE-MIT create mode 100644 packages/mfs/README.md create mode 100644 packages/mfs/package.json create mode 100644 packages/mfs/src/index.ts create mode 100644 packages/mfs/src/utils/basename.ts create mode 100644 packages/mfs/test/cat.spec.ts create mode 100644 packages/mfs/test/chmod.spec.ts create mode 100644 packages/mfs/test/cp.spec.ts create mode 100644 packages/mfs/test/fixtures/create-sharded-directory.ts create mode 100644 packages/mfs/test/fixtures/create-subsharded-directory.ts create mode 100644 packages/mfs/test/fixtures/files.ts create mode 100644 packages/mfs/test/fixtures/print-tree.ts create mode 100644 packages/mfs/test/ls.spec.ts create mode 100644 packages/mfs/test/mkdir.spec.ts create mode 100644 packages/mfs/test/rm.spec.ts create mode 100644 packages/mfs/test/stat.spec.ts create mode 100644 packages/mfs/test/touch.spec.ts create mode 100644 packages/mfs/tsconfig.json create mode 100644 packages/mfs/typedoc.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0bc3b42d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 00000000..d57c2a02 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 00000000..a15c3791 --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,182 @@ +name: test & maybe release +on: + push: + branches: + - main + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + - run: npm run --if-present doc-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: firefox-webworker + + test-webkit: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit + + test-webkit-webworker: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:webkit-webworker + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: webkit-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7ad9e674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..351df654 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +

+ + Helia logo + +

+ +# @helia/mfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-mfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-mfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-mfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-mfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> A mutable filesystem powered by Helia + +## Table of contents + +- - [Structure](#structure) +- [@helia/unixfs ](#heliaunixfs----omit-in-toc---) + - [API Docs](#api-docs) + - [License](#license) + - [Contribute](#contribute) + +## Structure + +- [`/packages/interop`](./packages/interop) Interop tests for @helia/mfs +- [`/packages/mfs`](./packages/mfs) A mutable filesystem powered by Helia + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-mfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/package.json b/package.json new file mode 100644 index 00000000..c050bf2a --- /dev/null +++ b/package.json @@ -0,0 +1,44 @@ +{ + "name": "@helia/mfs", + "version": "0.0.0", + "description": "A mutable filesystem powered by Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-mfs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-mfs.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-mfs/issues" + }, + "keywords": [ + "ipfs" + ], + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && aegir run release && npm run docs" + }, + "devDependencies": { + "aegir": "^40.0.9" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 00000000..498799ac --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 00000000..644f58c5 --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,38 @@ +

+ + Helia logo + +

+ +# @helia/mfs-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-mfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-mfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-mfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-mfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/mfs + +## Table of contents + +- [License](#license) +- [Contribute](#contribute) + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-mfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 00000000..e59404a1 --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,77 @@ +{ + "name": "@helia/mfs-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/mfs", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-mfs.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-mfs/issues" + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^13.0.0", + "@chainsafe/libp2p-yamux": "^5.0.0", + "@helia/interface": "next", + "@helia/unixfs": "^1.0.0", + "@libp2p/tcp": "^8.0.2", + "@libp2p/websockets": "^7.0.2", + "aegir": "^40.0.9", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.2.0", + "go-ipfs": "^0.22.0", + "helia": "next", + "ipfs-core-types": "^0.14.0", + "ipfs-unixfs-importer": "^15.1.0", + "ipfsd-ctl": "^13.0.0", + "it-to-buffer": "^4.0.1", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.46.3", + "merge-options": "^3.0.4", + "multiformats": "^12.0.1", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 00000000..867de492 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,46 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + services: { + identify: identifyService() + }, + datastore, + connectionGater: { + denyDialMultiaddr: async () => false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 00000000..f7a25c3f --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,39 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { tcp } from '@libp2p/tcp' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { createHelia } from 'helia' +import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { identifyService } from 'libp2p/identify' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + services: { + identify: identifyService() + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 00000000..c032e86d --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,26 @@ +import * as goIpfs from 'go-ipfs' +import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import mergeOptions from 'merge-options' +import { isElectronMain, isNode } from 'wherearewe' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return createController(opts) +} diff --git a/packages/interop/test/index.spec.ts b/packages/interop/test/index.spec.ts new file mode 100644 index 00000000..963f0474 --- /dev/null +++ b/packages/interop/test/index.spec.ts @@ -0,0 +1,105 @@ +/* eslint-env mocha */ + +import { type MFS, mfs } from '@helia/mfs' +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' + +describe('mfs interop', () => { + let helia: Helia + let fs: MFS + let kubo: Controller + + beforeEach(async () => { + helia = await createHeliaNode() + fs = mfs(helia) + kubo = await createKuboNode() + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should have the same CID initially', async () => { + const heliaStat = await fs.stat('/') + const kuboStat = await kubo.api.files.stat('/') + + expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString()) + }) + + it('should have the same CID after creating a directory', async () => { + const dirPath = '/foo' + await fs.mkdir(dirPath) + await kubo.api.files.mkdir(dirPath, { + cidVersion: 1 + }) + + const heliaStat = await fs.stat('/') + const kuboStat = await kubo.api.files.stat('/') + + expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString()) + }) + + it('should have the same CID after removing a directory', async () => { + const dirPath = '/foo' + await fs.mkdir(dirPath) + await fs.rm(dirPath) + await kubo.api.files.mkdir(dirPath, { + cidVersion: 1 + }) + await kubo.api.files.rm(dirPath, { + recursive: true + }) + + const heliaStat = await fs.stat('/') + const kuboStat = await kubo.api.files.stat('/') + + expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString()) + }) + + it('should have the same CID after creating a file', async () => { + const filePath = '/foo.txt' + const fileData = Uint8Array.from([0, 1, 2, 3, 4]) + await fs.writeBytes(fileData, filePath, { + rawLeaves: true, + reduceSingleLeafToSelf: false + }) + await kubo.api.files.write(filePath, fileData, { + cidVersion: 1, + create: true + }) + + const heliaStat = await fs.stat('/') + const kuboStat = await kubo.api.files.stat('/') + + expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString()) + }) + + it('should have the same CID after removing a file', async () => { + const filePath = '/foo.txt' + const fileData = Uint8Array.from([0, 1, 2, 3, 4]) + await fs.writeBytes(fileData, filePath, { + rawLeaves: true, + reduceSingleLeafToSelf: false + }) + await fs.rm(filePath) + await kubo.api.files.write(filePath, fileData, { + cidVersion: 1, + create: true + }) + await kubo.api.files.rm(filePath) + + const heliaStat = await fs.stat('/') + const kuboStat = await kubo.api.files.stat('/') + + expect(heliaStat.cid.toV1().toString()).to.equal(kuboStat.cid.toV1().toString()) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/interop/typedoc.json b/packages/interop/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/interop/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/mfs/LICENSE b/packages/mfs/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/mfs/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/mfs/LICENSE-APACHE b/packages/mfs/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/mfs/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/mfs/LICENSE-MIT b/packages/mfs/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/mfs/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/mfs/README.md b/packages/mfs/README.md new file mode 100644 index 00000000..f5c0318f --- /dev/null +++ b/packages/mfs/README.md @@ -0,0 +1,59 @@ +

+ + Helia logo + +

+ +# @helia/mfs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-mfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-mfs) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-mfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-mfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> A mutable filesystem powered by Helia + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-mfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/mfs/package.json b/packages/mfs/package.json new file mode 100644 index 00000000..adcc5d5d --- /dev/null +++ b/packages/mfs/package.json @@ -0,0 +1,167 @@ +{ + "name": "@helia/mfs", + "version": "0.0.0", + "description": "A mutable filesystem powered by Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/mfs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-mfs.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-mfs/issues" + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "next", + "@helia/unixfs": "^1.2.2", + "@ipld/dag-pb": "^4.0.0", + "@libp2p/interfaces": "^3.3.1", + "@libp2p/logger": "^3.0.1", + "@multiformats/murmur3": "^2.1.2", + "hamt-sharding": "^3.0.2", + "interface-blockstore": "^5.0.0", + "interface-datastore": "^8.2.2", + "ipfs-unixfs": "^11.0.0", + "ipfs-unixfs-exporter": "^13.1.0", + "ipfs-unixfs-importer": "^15.1.0", + "it-last": "^3.0.1", + "it-pipe": "^3.0.1", + "merge-options": "^3.0.4", + "multiformats": "^12.0.1", + "progress-events": "^1.0.0", + "sparse-array": "^1.3.2" + }, + "devDependencies": { + "aegir": "^40.0.9", + "blockstore-core": "^4.0.1", + "datastore-core": "^9.2.0", + "delay": "^6.0.0", + "it-all": "^3.0.1", + "it-drain": "^3.0.1", + "it-first": "^3.0.1", + "it-to-buffer": "^4.0.1", + "uint8arrays": "^4.0.3" + } +} diff --git a/packages/mfs/src/index.ts b/packages/mfs/src/index.ts new file mode 100644 index 00000000..1fa81b9b --- /dev/null +++ b/packages/mfs/src/index.ts @@ -0,0 +1,600 @@ +/** + * @packageDocumentation + * + * `@helia/mfs` is an implementation of a {@link https://docs.ipfs.tech/concepts/file-systems/ Mutable File System} powered by {@link https://github.com/ipfs/helia Helia}. + * + * See the {@link MFS MFS interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { mfs } from '@helia/mfs' + * + * const helia = createHelia({ + * // ... helia config + * }) + * const fs = mfs(helia) + * + * // create an empty directory + * await fs.mkdir('/my-directory') + * + * // add a file to the directory + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/my-directory/foo.txt') + * + * // read the file + * for await (const buf of fs.cat('/my-directory/foo.txt')) { + * console.info(buf) + * } + * ``` + */ + +import { unixfs } from '@helia/unixfs' +import { AlreadyExistsError, DoesNotExistError, InvalidParametersError, NotADirectoryError } from '@helia/unixfs/errors' +import { logger } from '@libp2p/logger' +import { Key } from 'interface-datastore' +import { UnixFS as IPFSUnixFS, type Mtime } from 'ipfs-unixfs' +import { CID } from 'multiformats/cid' +import { basename } from './utils/basename.js' +import type { Blocks } from '@helia/interface/blocks' +import type { AddOptions, CatOptions, ChmodOptions, CpOptions, LsOptions, MkdirOptions as UnixFsMkdirOptions, RmOptions as UnixFsRmOptions, StatOptions, TouchOptions, UnixFS, UnixFSStats } from '@helia/unixfs' +import type { AbortOptions } from '@libp2p/interfaces' +import type { Datastore } from 'interface-datastore' +import type { UnixFSEntry } from 'ipfs-unixfs-exporter' +import type { ByteStream } from 'ipfs-unixfs-importer' + +const log = logger('helia:mfs') + +export interface MFSComponents { + blockstore: Blocks + datastore: Datastore +} + +export interface MFSInit { + /** + * The key used to store the root CID in the datastore (default: '/local/filesroot') + */ + key?: string +} + +export type WriteOptions = AddOptions & CpOptions & { + /** + * An optional mode to set on the new file + */ + mode: number + + /** + * An optional mtime to set on the new file + */ + mtime: Mtime +} + +export type MkdirOptions = AddOptions & StatOptions & CpOptions & UnixFsMkdirOptions + +/** + * Options to pass to the rm command + */ +export interface RmOptions extends UnixFsRmOptions { + /** + * If true, allow attempts to delete files or directories that do not exist + * (default: false) + */ + force: boolean +} + +/** + * The MFS interface allows working with files and directories in a mutable file + * system. + */ +export interface MFS { + /** + * Add a single `Uint8Array` to your Helia node as a file. + * + * @example + * + * ```typescript + * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * + * console.info(cid) + * ``` + */ + writeBytes: (bytes: Uint8Array, path: string, options?: Partial) => Promise + + /** + * Add a stream of `Uint8Array` to your Helia node as a file. + * + * @example + * + * ```typescript + * import fs from 'node:fs' + * + * const stream = fs.createReadStream('./foo.txt') + * const cid = await fs.addByteStream(stream) + * + * console.info(cid) + * ``` + */ + writeByteStream: (bytes: ByteStream, path: string, options?: Partial) => Promise + + /** + * Retrieve the contents of a file from your Helia node. + * + * @example + * + * ```typescript + * for await (const buf of fs.cat(cid)) { + * console.info(buf) + * } + * ``` + */ + cat: (path: string, options?: Partial) => AsyncIterable + + /** + * Change the permissions on a file or directory in a DAG + * + * @example + * + * ```typescript + * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const beforeStats = await fs.stat(beforeCid) + * + * const afterCid = await fs.chmod(cid, 0x755) + * const afterStats = await fs.stat(afterCid) + * + * console.info(beforeCid, beforeStats) + * console.info(afterCid, afterStats) + * ``` + */ + chmod: (path: string, mode: number, options?: Partial) => Promise + + /** + * Add a file or directory to a target directory. + * + * @example + * + * ```typescript + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const directoryCid = await fs.addDirectory() + * + * const updatedCid = await fs.cp(fileCid, directoryCid, 'foo.txt') + * + * console.info(updatedCid) + * ``` + */ + cp: (source: CID | string, destination: string, options?: Partial) => Promise + + /** + * List directory contents. + * + * @example + * + * ```typescript + * for await (const entry of fs.ls(directoryCid)) { + * console.info(etnry) + * } + * ``` + */ + ls: (path?: string, options?: Partial) => AsyncIterable + + /** + * Make a new directory under an existing directory. + * + * @example + * + * ```typescript + * const directoryCid = await fs.addDirectory() + * + * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') + * + * console.info(updatedCid) + * ``` + */ + mkdir: (path: string, options?: Partial) => Promise + + /** + * Remove a file or directory from an existing directory. + * + * @example + * + * ```typescript + * const directoryCid = await fs.addDirectory() + * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') + * + * const finalCid = await fs.rm(updatedCid, 'new-dir') + * + * console.info(finalCid) + * ``` + */ + rm: (path: string, options?: Partial) => Promise + + /** + * Return statistics about a UnixFS DAG. + * + * @example + * + * ```typescript + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * + * const stats = await fs.stat(fileCid) + * + * console.info(stats) + * ``` + */ + stat: (path: string, options?: Partial) => Promise + + /** + * Update the mtime of a UnixFS DAG + * + * @example + * + * ```typescript + * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const beforeStats = await fs.stat(beforeCid) + * + * const afterCid = await fs.touch(beforeCid) + * const afterStats = await fs.stat(afterCid) + * + * console.info(beforeCid, beforeStats) + * console.info(afterCid, afterStats) + * ``` + */ + touch: (path: string, options?: Partial) => Promise +} + +interface PathEntry { + cid: CID + name: string + unixfs?: IPFSUnixFS +} + +interface WalkPathOptions extends AbortOptions { + createMissingDirectories: boolean + finalSegmentMustBeDirectory: boolean +} + +class DefaultMFS implements MFS { + private readonly components: MFSComponents + private readonly unixfs: UnixFS + private root?: CID + private readonly key: Key + + constructor (components: MFSComponents, init: MFSInit = {}) { + this.components = components + + this.key = new Key(init.key ?? '/locals/filesroot') + this.unixfs = unixfs(components) + } + + async #getRootCID (): Promise { + if (this.root == null) { + try { + const buf = await this.components.datastore.get(this.key) + this.root = CID.decode(buf) + } catch (err: any) { + if (err.code !== 'ERR_NOT_FOUND') { + throw err + } + + this.root = await this.unixfs.addDirectory() + } + } + + return this.root + } + + async writeBytes (bytes: Uint8Array, path: string, options?: Partial): Promise { + const cid = await this.unixfs.addFile({ + content: bytes, + mode: options?.mode, + mtime: options?.mtime + }, options) + + await this.cp(cid, path, options) + } + + async writeByteStream (bytes: ByteStream, path: string, options?: Partial): Promise { + const cid = await this.unixfs.addFile({ + content: bytes, + mode: options?.mode, + mtime: options?.mtime + }, options) + + await this.cp(cid, path, options) + } + + async * cat (path: string, options: Partial = {}): AsyncIterable { + const root = await this.#getRootCID() + const trail = await this.#walkPath(root, path, { + ...options, + createMissingDirectories: false, + finalSegmentMustBeDirectory: false + }) + + yield * this.unixfs.cat(trail[trail.length - 1].cid, options) + } + + async chmod (path: string, mode: number, options: Partial = {}): Promise { + const root = await this.#getRootCID() + + this.root = await this.unixfs.chmod(root, mode, { + ...options, + path + }) + } + + async cp (source: CID | string, destination: string, options?: Partial): Promise { + const root = await this.#getRootCID() + const force = options?.force ?? false + + if (typeof source === 'string') { + const stat = await this.stat(source, options) + + source = stat.cid + } + + if (!force) { + await this.#ensurePathDoesNotExist(destination, options) + } + + const fileName = basename(destination) + const containingDirectory = destination.substring(0, destination.length - `/${fileName}`.length) + + let trail: PathEntry[] = [{ + cid: root, + name: '' + }] + + if (containingDirectory !== '') { + trail = await this.#walkPath(root, containingDirectory, { + ...options, + createMissingDirectories: options?.force ?? false, + finalSegmentMustBeDirectory: true + }) + } + + trail.push({ + cid: source, + name: fileName + }) + + this.root = await this.#persistPath(trail, options) + } + + async * ls (path?: string, options?: Partial): AsyncIterable { + const root = await this.#getRootCID() + + if (options?.path != null) { + path = `${path}/${options.path}` + } + + yield * this.unixfs.ls(root, { + ...options, + path + }) + } + + async mkdir (path: string, options?: Partial): Promise { + const force = options?.force ?? false + + if (!force) { + await this.#ensurePathDoesNotExist(path, options) + } + + const dirName = basename(path) + const containingDirectory = path.substring(0, path.length - `/${dirName}`.length) + const root = await this.#getRootCID() + + let trail: PathEntry[] = [{ + cid: root, + name: '' + }] + + if (containingDirectory !== '') { + trail = await this.#walkPath(root, containingDirectory, { + ...options, + createMissingDirectories: force, + finalSegmentMustBeDirectory: true + }) + } + + trail.push({ + cid: await this.unixfs.addDirectory({ + mode: options?.mode, + mtime: options?.mtime + }, options), + name: basename(path) + }) + + this.root = await this.#persistPath(trail, options) + } + + async rm (path: string, options?: Partial): Promise { + const force = options?.force ?? false + + if (!force) { + await this.#ensurePathExists(path, options) + } + + const root = await this.#getRootCID() + + const trail = await this.#walkPath(root, path, { + ...options, + createMissingDirectories: false, + finalSegmentMustBeDirectory: false + }) + + const lastSegment = trail.pop() + + if (lastSegment == null) { + throw new InvalidParametersError('path was too short') + } + + // remove directory entry + const containingDir = trail[trail.length - 1] + containingDir.cid = await this.unixfs.rm(containingDir.cid, lastSegment.name, options) + + this.root = await this.#persistPath(trail, options) + } + + async stat (path: string, options?: Partial): Promise { + const root = await this.#getRootCID() + + const trail = await this.#walkPath(root, path, { + ...options, + createMissingDirectories: false, + finalSegmentMustBeDirectory: false + }) + + const finalEntry = trail.pop() + + if (finalEntry == null) { + throw new DoesNotExistError() + } + + return this.unixfs.stat(finalEntry.cid, { + ...options + }) + } + + async touch (path: string, options?: Partial): Promise { + const root = await this.#getRootCID() + const trail = await this.#walkPath(root, path, { + ...options, + createMissingDirectories: false, + finalSegmentMustBeDirectory: false + }) + + const finalEntry = trail[trail.length - 1] + + if (finalEntry == null) { + throw new DoesNotExistError() + } + + finalEntry.cid = await this.unixfs.touch(finalEntry.cid, options) + + this.root = await this.#persistPath(trail, options) + } + + async #walkPath (root: CID, path: string, opts: WalkPathOptions): Promise { + if (!path.startsWith('/')) { + throw new InvalidParametersError('path must be absolute') + } + + const stat = await this.unixfs.stat(root, { + ...opts, + offline: true + }) + + const output: PathEntry[] = [{ + cid: root, + name: '', + unixfs: stat.unixfs + }] + + let cid = root + const parts = path.split('/').filter(Boolean) + + for (let i = 0; i < parts.length; i++) { + const segment = parts[i] + + try { + const stat = await this.unixfs.stat(cid, { + ...opts, + offline: true, + path: segment + }) + + output.push({ + cid: stat.cid, + name: segment, + unixfs: stat.unixfs + }) + + cid = stat.cid + } catch (err) { + log.error('could not resolve path segment %s of %s under %c', segment, path, root) + + if (opts.createMissingDirectories) { + const cid = await this.unixfs.addDirectory() + + output.push({ + cid, + name: segment, + unixfs: new IPFSUnixFS({ type: 'directory' }) + }) + } else { + throw new DoesNotExistError(`${path} does not exist`) + } + } + } + + const lastSegment = output[output.length - 1] + + if (opts.finalSegmentMustBeDirectory && lastSegment.unixfs?.isDirectory() !== true) { + throw new NotADirectoryError(`${path} was not a directory`) + } + + return output + } + + async #persistPath (path: PathEntry[], options: Partial = {}): Promise { + let child = path.pop() + + if (child == null) { + throw new InvalidParametersError('path was too short') + } + + let cid = child.cid + + for (let i = path.length - 1; i > -1; i--) { + const segment = path[i] + segment.cid = await this.unixfs.cp(child.cid, segment.cid, child.name, { + ...options, + force: true + }) + + child = segment + cid = segment.cid + } + + await this.components.datastore.put(this.key, cid.bytes, options) + + return cid + } + + async #ensurePathExists (path: string, options: StatOptions = {}): Promise { + const exists = await this.#pathExists(path, options) + + if (!exists) { + throw new DoesNotExistError() + } + } + + async #ensurePathDoesNotExist (path: string, options: StatOptions = {}): Promise { + const exists = await this.#pathExists(path, options) + + if (exists) { + throw new AlreadyExistsError() + } + } + + async #pathExists (path: string, options: StatOptions = {}): Promise { + try { + await this.stat(path, { + ...options, + offline: true + }) + + return true + } catch { + return false + } + } +} + +/** + * Create a {@link MFS} instance powered by {@link https://github.com/ipfs/helia Helia} + */ +export function mfs (helia: { blockstore: Blocks, datastore: Datastore }, init: MFSInit = {}): MFS { + return new DefaultMFS(helia, init) +} diff --git a/packages/mfs/src/utils/basename.ts b/packages/mfs/src/utils/basename.ts new file mode 100644 index 00000000..265efca6 --- /dev/null +++ b/packages/mfs/src/utils/basename.ts @@ -0,0 +1,3 @@ +export function basename (path: string): string { + return path.split('/').pop() ?? '' +} diff --git a/packages/mfs/test/cat.spec.ts b/packages/mfs/test/cat.spec.ts new file mode 100644 index 00000000..b66530a3 --- /dev/null +++ b/packages/mfs/test/cat.spec.ts @@ -0,0 +1,94 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import drain from 'it-drain' +import toBuffer from 'it-to-buffer' +import { mfs, type MFS } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('cat', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('reads a small file', async () => { + const path = '/foo.txt' + await fs.writeBytes(smallFile, path) + + const bytes = await toBuffer(fs.cat(path)) + + expect(bytes).to.equalBytes(smallFile) + }) + + it('reads a file with an offset', async () => { + const path = '/foo.txt' + const offset = 10 + + await fs.writeBytes(smallFile, path) + const bytes = await toBuffer(fs.cat(path, { + offset + })) + + expect(bytes).to.equalBytes(smallFile.subarray(offset)) + }) + + it('reads a file with a length', async () => { + const path = '/foo.txt' + const length = 10 + await fs.writeBytes(smallFile, path) + + const bytes = await toBuffer(fs.cat(path, { + length + })) + + expect(bytes).to.equalBytes(smallFile.subarray(0, length)) + }) + + it('reads a file with an offset and a length', async () => { + const path = '/foo.txt' + const offset = 2 + const length = 5 + await fs.writeBytes(smallFile, path) + + const bytes = await toBuffer(fs.cat(path, { + offset, + length + })) + + expect(bytes).to.equalBytes(smallFile.subarray(offset, offset + length)) + }) + + it('refuses to read a directory', async () => { + const path = '/foo' + + await fs.mkdir(path) + + await expect(drain(fs.cat(path))).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_A_FILE') + }) + + it('reads file from inside a sharded directory', async () => { + const shardedDirPath = '/foo' + const shardedDirCid = await createShardedDirectory(blockstore) + const filePath = `${shardedDirPath}/new-file.txt` + + await fs.cp(shardedDirCid, shardedDirPath) + await fs.writeBytes(smallFile, filePath) + + const bytes = await toBuffer(fs.cat(filePath)) + + expect(bytes).to.deep.equal(smallFile) + }) +}) diff --git a/packages/mfs/test/chmod.spec.ts b/packages/mfs/test/chmod.spec.ts new file mode 100644 index 00000000..67c35ead --- /dev/null +++ b/packages/mfs/test/chmod.spec.ts @@ -0,0 +1,90 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('chmod', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('should update the mode for a raw node', async () => { + const path = '/foo.txt' + await fs.writeBytes(smallFile, path) + const originalMode = (await fs.stat(path)).mode + + await fs.chmod(path, 0o777) + + const updatedMode = (await fs.stat(path)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update the mode for a file', async () => { + const path = '/foo.txt' + await fs.writeBytes(smallFile, path, { + rawLeaves: false + }) + const originalMode = (await fs.stat(path)).mode + await fs.chmod(path, 0o777) + + const updatedMode = (await fs.stat(path)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update the mode for a directory', async () => { + const path = '/foo' + await fs.mkdir(path) + const originalMode = (await fs.stat(path)).mode + await fs.chmod(path, 0o777) + + const updatedMode = (await fs.stat(path)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update mode recursively', async () => { + const dirPath = '/foo' + const filePath = `${dirPath}/bar/baz.txt` + await fs.mkdir('/foo') + await fs.mkdir('/foo/bar') + await fs.writeBytes(smallFile, filePath) + + const originalMode = (await fs.stat(filePath)).mode + await fs.chmod(dirPath, 0o777, { + recursive: true + }) + + const updatedMode = (await fs.stat(filePath)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) + + it('should update the mode for a hamt-sharded-directory', async () => { + const shardedDirPath = '/foo' + const shardedDirCid = await createShardedDirectory(blockstore) + await fs.cp(shardedDirCid, shardedDirPath) + + const originalMode = (await fs.stat(shardedDirPath)).mode + + await fs.chmod(shardedDirPath, 0o777) + + const updatedMode = (await fs.stat(shardedDirPath)).mode + expect(updatedMode).to.not.equal(originalMode) + expect(updatedMode).to.equal(0o777) + }) +}) diff --git a/packages/mfs/test/cp.spec.ts b/packages/mfs/test/cp.spec.ts new file mode 100644 index 00000000..71dc2610 --- /dev/null +++ b/packages/mfs/test/cp.spec.ts @@ -0,0 +1,189 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import first from 'it-first' +import toBuffer from 'it-to-buffer' +import { CID } from 'multiformats/cid' +import { identity } from 'multiformats/hashes/identity' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('cp', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('refuses to copy files without a source', async () => { + // @ts-expect-error invalid args + await expect(fs.cp()).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files without a source, even with options', async () => { + // @ts-expect-error invalid args + await expect(fs.cp({})).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files without a destination', async () => { + // @ts-expect-error invalid args + await expect(fs.cp('/source')).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy files without a destination, even with options', async () => { + // @ts-expect-error invalid args + await expect(fs.cp('/source', {})).to.eventually.be.rejected.with('Please supply at least one source') + }) + + it('refuses to copy an unreadable node', async () => { + const hash = identity.digest(uint8ArrayFromString('derp')) + const source = CID.createV1(identity.code, hash) + + await expect(fs.cp(source, '/foo.txt')).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_UNIXFS') + }) + + it('refuses to copy a non-existant file', async () => { + await expect(fs.cp('/foo.txt', '/bar.txt')).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('refuses to copy files to an existing file', async () => { + const path = '/foo.txt' + await fs.writeBytes(Uint8Array.from([0, 1, 3, 4]), path) + + await expect(fs.cp(path, path)).to.eventually.be.rejected + .with.property('code', 'ERR_ALREADY_EXISTS') + + // should succeed with force option + await expect(fs.cp(path, path, { + force: true + })).to.eventually.be.undefined() + }) + + it('copies a file to new location', async () => { + const source = '/foo.txt' + const destination = '/bar' + + const data = Uint8Array.from([0, 1, 3, 4]) + await fs.writeBytes(data, source) + await fs.cp(source, destination) + + const bytes = await toBuffer(fs.cat(destination)) + + expect(bytes).to.deep.equal(data) + }) + + it('copies directories', async () => { + const source = '/foo' + const destination = '/bar' + + await fs.mkdir(source) + await fs.cp(source, destination) + + await expect(fs.stat(destination)).to.eventually.include({ + type: 'directory' + }) + }) + + it('copies a sharded directory to a normal directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const path = '/sharded-dir' + await fs.cp(shardedDirCid, path) + + // should still be a regular directory + await expect(fs.stat('/')).to.eventually.have.nested.property('unixfs.type', 'directory') + + const subDirStats = await fs.stat(path) + expect(subDirStats).to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + expect(subDirStats.cid).to.eql(shardedDirCid) + }) + + it('copies a normal directory to a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-dir' + const normalDirPath = '/normal-dir' + const normalSubDirPath = `${shardedDirPath}/normal-dir` + + await fs.mkdir(normalDirPath) + await fs.cp(shardedDirCid, shardedDirPath) + await fs.cp(normalDirPath, normalSubDirPath) + + // should still be a sharded directory + await expect(fs.stat(shardedDirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const subDirStats = await fs.stat(normalSubDirPath) + expect(subDirStats).to.have.nested.property('unixfs.type', 'directory') + expect(subDirStats.cid.toString()).to.equal('bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354') + }) + + it('copies a file from a normal directory to a sharded directory', async () => { + const filePath = `/file-${Math.random()}.txt` + await fs.writeBytes(Uint8Array.from([0, 1, 3, 4]), filePath, { + rawLeaves: false + }) + + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-dir' + await fs.cp(shardedDirCid, shardedDirPath) + + const fileInDirectoryPath = `${shardedDirPath}${filePath}` + await fs.cp(filePath, fileInDirectoryPath) + + // should still be a sharded directory + await expect(fs.stat(shardedDirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const fileInDirStats = await fs.stat(fileInDirectoryPath) + expect(fileInDirStats).to.have.nested.property('unixfs.type', 'file') + expect(fileInDirStats.cid.toString()).to.equal('bafybeibyuhrlz5wvmzhn5twibjtpofeek4v6afw3uvk6jkewrdkgivvcea') + }) + + it('refuses to copy files to an existing file in a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-dir' + await fs.cp(shardedDirCid, shardedDirPath) + const file = await first(fs.ls(shardedDirPath)) + + if (file == null) { + throw new Error('No files listed') + } + + await expect(fs.cp(file.cid, `${shardedDirPath}/${file.name}`)).to.eventually.be.rejected + .with.property('code', 'ERR_ALREADY_EXISTS') + + // should succeed with force option + await expect(fs.cp(file.cid, `${shardedDirPath}/${file.name}`, { + force: true + })).to.eventually.be.undefined() + }) + + it('copies a file to a sharded directory that creates a subshard', async () => { + const { + containingDirCid, + fileName, + importerCid + } = await createSubshardedDirectory(blockstore) + + const shardedDirPath = '/sharded-dir' + await fs.cp(importerCid, shardedDirPath) + + // adding a file to the importer CID should result in the shard with a subshard + const fileCid = CID.parse('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + await fs.cp(fileCid, `${shardedDirPath}/${fileName}`) + + const stat = await fs.stat(shardedDirPath) + + expect(stat.cid.toString()).to.equal(containingDirCid.toString(), 'adding a file to the imported dir did not result in the same CID') + }) +}) diff --git a/packages/mfs/test/fixtures/create-sharded-directory.ts b/packages/mfs/test/fixtures/create-sharded-directory.ts new file mode 100644 index 00000000..3c8fd627 --- /dev/null +++ b/packages/mfs/test/fixtures/create-sharded-directory.ts @@ -0,0 +1,27 @@ +import { expect } from 'aegir/chai' +import { importer } from 'ipfs-unixfs-importer' +import last from 'it-last' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +export async function createShardedDirectory (blockstore: Blockstore, files = 1001): Promise { + const result = await last(importer((function * () { + for (let i = 0; i < files; i++) { + yield { + path: `./file-${i}`, + content: Uint8Array.from([0, 1, 2, 3, 4]) + } + } + }()), blockstore, { + shardSplitThresholdBytes: 1, + wrapWithDirectory: true + })) + + if (result == null) { + throw new Error('No result received from ipfs.addAll') + } + + expect(result).to.have.nested.property('unixfs.type', 'hamt-sharded-directory', 'tried to create a shared directory but the result was not a shard') + + return result.cid +} diff --git a/packages/mfs/test/fixtures/create-subsharded-directory.ts b/packages/mfs/test/fixtures/create-subsharded-directory.ts new file mode 100644 index 00000000..119f8ac9 --- /dev/null +++ b/packages/mfs/test/fixtures/create-subsharded-directory.ts @@ -0,0 +1,92 @@ +import { unixfs } from '@helia/unixfs' +import * as dagPb from '@ipld/dag-pb' +import { importer } from 'ipfs-unixfs-importer' +import last from 'it-last' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +export async function createSubshardedDirectory (blockstore: Blockstore, depth: number = 1, files: number = 5000): Promise<{ + importerCid: CID + containingDirCid: CID + fileName: string +}> { + const fs = unixfs({ blockstore }) + const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) + let containingDirCid = await fs.addDirectory() + let fileName: string | undefined + let count = 0 + + for (let i = 0; i < files; i++) { + fileName = `file-${i}-${new Array(512).fill('0').join('')}.txt` + + containingDirCid = await fs.cp(fileCid, containingDirCid, fileName, { + shardSplitThresholdBytes: 1 + }) + + if (await searchCIDForSubshards(containingDirCid, blockstore, depth)) { + count = i + + break + } + } + + if (fileName == null) { + throw new Error('could not find file that would create a subshard') + } + + // create a shard with the importer that is the same as the directory after we delete the file that causes a sub-shard to be created + const importResult = await last(importer( + new Array(count).fill(0).map((_, i) => { + return { + path: `file-${i}-${new Array(512).fill('0').join('')}.txt`, + content: Uint8Array.from([0, 1, 2, 3, 4]) + } + }), blockstore, { + wrapWithDirectory: true, + shardSplitThresholdBytes: 1 + })) + + if (importResult == null) { + throw new Error('Nothing imported') + } + + const { cid: importerCid } = importResult + + return { + importerCid, + containingDirCid, + fileName + } +} + +async function searchCIDForSubshards (cid: CID, blockstore: Blockstore, depth: number = 1): Promise { + const block = await blockstore.get(cid) + const node = dagPb.decode(block) + + // search links for subshard + for (const link of node.Links) { + if (link.Name?.length === 2) { + const block = await blockstore.get(link.Hash) + const node = dagPb.decode(block) + const firstLink = node.Links[1] + + if (firstLink == null) { + throw new Error('Subshard had no child links') + } + + if (firstLink.Name == null) { + throw new Error('Subshard child had no name') + } + + if (depth === 1) { + return true + } + + if (await searchCIDForSubshards(link.Hash, blockstore, depth - 1)) { + return true + } + } + } + + return false +} diff --git a/packages/mfs/test/fixtures/files.ts b/packages/mfs/test/fixtures/files.ts new file mode 100644 index 00000000..cce002d2 --- /dev/null +++ b/packages/mfs/test/fixtures/files.ts @@ -0,0 +1,5 @@ +const ONE_MEG = 1024 * 1024 + +export const largeFile = Uint8Array.from(new Array(ONE_MEG * 5).fill(0).map(() => Math.random() * 100)) + +export const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100)) diff --git a/packages/mfs/test/fixtures/print-tree.ts b/packages/mfs/test/fixtures/print-tree.ts new file mode 100644 index 00000000..e41ac6ab --- /dev/null +++ b/packages/mfs/test/fixtures/print-tree.ts @@ -0,0 +1,35 @@ +import * as dagPb from '@ipld/dag-pb' +import { UnixFS } from 'ipfs-unixfs' +import * as raw from 'multiformats/codecs/raw' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' + +type GetStore = Pick + +export async function printTree (cid: CID, blockstore: GetStore, name: string = '', indent: string = ''): Promise { + const block = await blockstore.get(cid) + + if (cid.code === dagPb.code) { + const node = dagPb.decode(block) + + if (node.Data == null) { + return + } + + const unixfs = UnixFS.unmarshal(node.Data) + + console.info(indent, cid.toString(), name, unixfs.type) // eslint-disable-line no-console + + for (const link of node.Links) { + let name = link.Name ?? '' + + if (name.length > 12) { + name = name.substring(0, 12) + '...' + } + + await printTree(link.Hash, blockstore, name, `${indent} `) + } + } else if (cid.code === raw.code) { + console.info(indent, cid.toString(), name, 'dag-raw') // eslint-disable-line no-console + } +} diff --git a/packages/mfs/test/ls.spec.ts b/packages/mfs/test/ls.spec.ts new file mode 100644 index 00000000..45604d10 --- /dev/null +++ b/packages/mfs/test/ls.spec.ts @@ -0,0 +1,179 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import all from 'it-all' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('ls', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('should list mfs root by default', async () => { + await expect(all(fs.ls())).to.eventually.have.lengthOf(0) + }) + + it('lists files in the root directory', async () => { + const fileName = 'foo.txt' + const filePath = `/${fileName}` + const data = Uint8Array.from([0, 1, 2, 3]) + await fs.writeBytes(data, filePath) + const fileStat = await fs.stat(filePath) + const files = await all(fs.ls('/')) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileStat.cid, + name: fileName, + size: BigInt(data.byteLength), + type: 'raw' + }]) + }) + + it('lists files in a directory', async () => { + const dirName = 'bar' + const dirPath = `/${dirName}` + const fileName = 'foo.txt' + const filePath = `${dirPath}/${fileName}` + const data = Uint8Array.from([0, 1, 2, 3]) + await fs.writeBytes(data, filePath, { + force: true + }) + const fileStat = await fs.stat(filePath) + const files = await all(fs.ls(dirPath)) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileStat.cid, + name: fileName, + size: BigInt(data.byteLength), + type: 'raw' + }]) + }) + + it('lists a file', async () => { + const dirName = 'bar' + const dirPath = `/${dirName}` + const fileName = 'foo.txt' + const filePath = `${dirPath}/${fileName}` + const data = Uint8Array.from([0, 1, 2, 3]) + await fs.writeBytes(data, filePath, { + rawLeaves: false, + force: true + }) + const fileStat = await fs.stat(filePath) + const files = await all(fs.ls(filePath)) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileStat.cid, + size: BigInt(data.byteLength), + type: 'file' + }]) + }) + + it('lists a file in a directory', async () => { + const dirName = 'bar' + const dirPath = `/${dirName}` + const fileName = 'foo.txt' + const filePath = `${dirPath}/${fileName}` + const data = Uint8Array.from([0, 1, 2, 3]) + await fs.writeBytes(data, filePath, { + rawLeaves: false, + force: true + }) + const fileStat = await fs.stat(filePath) + const files = await all(fs.ls(dirPath, { + path: fileName + })) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileStat.cid, + size: BigInt(data.byteLength), + type: 'file' + }]) + }) + + it('lists a raw node', async () => { + const dirName = 'bar' + const dirPath = `/${dirName}` + const fileName = 'foo.txt' + const filePath = `${dirPath}/${fileName}` + const data = Uint8Array.from([0, 1, 2, 3]) + await fs.writeBytes(data, filePath, { + force: true, + rawLeaves: true + }) + const fileStat = await fs.stat(filePath) + const files = await all(fs.ls(dirPath)) + + expect(files).to.have.lengthOf(1).and.to.containSubset([{ + cid: fileStat.cid, + size: BigInt(data.byteLength), + type: 'raw' + }]) + }) + + it('lists sharded directory contents', async () => { + const shardedDirPath = '/sharded-dir' + const fileCount = 1001 + const shardedDirCid = await createShardedDirectory(blockstore, fileCount) + await fs.cp(shardedDirCid, shardedDirPath) + const files = await all(fs.ls(shardedDirPath)) + + expect(files.length).to.equal(fileCount) + + files.forEach(file => { + // should be a file + expect(file.type).to.equal('raw') + }) + }) + + it('lists a file inside a sharded directory directly', async () => { + const shardedDirPath = '/sharded-dir' + const fileCount = 1001 + const shardedDirCid = await createShardedDirectory(blockstore, fileCount) + await fs.cp(shardedDirCid, shardedDirPath) + const files = await all(fs.ls(shardedDirPath)) + const fileName = files[0].name + + // should be able to ls new file directly + const directFiles = await all(fs.ls(shardedDirPath, { + path: fileName + })) + + expect(directFiles.length).to.equal(1) + expect(directFiles.filter(file => file.name === fileName)).to.be.ok() + }) + + it('lists the contents of a directory inside a sharded directory', async () => { + const shardedDirPath = '/sharded-dir' + const shardedDirCid = await createShardedDirectory(blockstore) + await fs.cp(shardedDirCid, shardedDirPath) + + const dirName = `subdir-${Math.random()}` + const fileName = `small-file-${Math.random()}.txt` + + const filePath = `${shardedDirPath}/${dirName}/${fileName}` + + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), filePath, { + force: true + }) + + const files = await all(fs.ls(shardedDirPath, { + path: dirName + })) + + expect(files.length).to.equal(1) + expect(files.filter(file => file.name === fileName)).to.be.ok() + }) +}) diff --git a/packages/mfs/test/mkdir.spec.ts b/packages/mfs/test/mkdir.spec.ts new file mode 100644 index 00000000..a837794d --- /dev/null +++ b/packages/mfs/test/mkdir.spec.ts @@ -0,0 +1,113 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import all from 'it-all' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' +import type { Mtime } from 'ipfs-unixfs' + +describe('mkdir', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + async function testMode (mode: number | undefined, expectedMode: number): Promise { + const path = '/sub-directory' + await fs.mkdir(path, { + mode + }) + + await expect(fs.stat(path)).to.eventually.have.property('mode', expectedMode) + } + + async function testMtime (mtime: Mtime, expectedMtime: Mtime): Promise { + const path = '/sub-directory' + await fs.mkdir(path, { + mtime + }) + + await expect(fs.stat(path)).to.eventually.have.deep.property('mtime', expectedMtime) + } + + it('requires a path', async () => { + // @ts-expect-error not enough arguments + await expect(fs.mkdir()).to.eventually.be.rejected() + }) + + it('creates a directory', async () => { + const dirName = 'foo' + const dirPath = `/${dirName}` + await fs.mkdir(dirPath) + + const stats = await fs.stat(dirPath) + expect(stats.type).to.equal('directory') + + const files = await all(fs.ls()) + + expect(files.length).to.equal(1) + expect(files).to.have.nested.property('[0].name', dirName) + }) + + it('refuses to create a directory that already exists', async () => { + const dirName = 'foo' + const dirPath = `/${dirName}` + await fs.mkdir(dirPath) + + await expect(fs.mkdir(dirPath)).to.eventually.be.rejected() + .with.property('code', 'ERR_ALREADY_EXISTS') + }) + + it('creates a nested directory with a different CID version to the parent', async () => { + const path = '/qux' + const subDirectory = `${path}/sub-dir` + + await fs.mkdir(path, { + cidVersion: 1 + }) + await fs.mkdir(subDirectory, { + cidVersion: 0 + }) + + await expect(fs.stat(path)).to.eventually.have.nested.property('cid.version', 1) + await expect(fs.stat(subDirectory)).to.eventually.have.nested.property('cid.version', 0) + }) + + it('should make directory and have default mode', async function () { + await testMode(undefined, parseInt('0755', 8)) + }) + + it('should make directory and specify mtime as { nsecs, secs }', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + + await testMtime(mtime, mtime) + }) + + it('makes a directory inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-directory' + await fs.cp(shardedDirCid, shardedDirPath) + const dirName = `${shardedDirPath}/subdir-${Math.random()}` + + await fs.mkdir(dirName) + + // should still be a sharded directory + await expect(fs.stat(shardedDirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + // subdir should be a regular directory + await expect(fs.stat(dirName)).to.eventually.have.nested.property('unixfs.type', 'directory') + }) +}) diff --git a/packages/mfs/test/rm.spec.ts b/packages/mfs/test/rm.spec.ts new file mode 100644 index 00000000..d9f961c9 --- /dev/null +++ b/packages/mfs/test/rm.spec.ts @@ -0,0 +1,243 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { importer } from 'ipfs-unixfs-importer' +import last from 'it-last' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js' +import { smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('rm', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('refuses to remove files without arguments', async () => { + // @ts-expect-error invalid args + await expect(fs.rm()).to.eventually.be.rejected() + }) + + it('removes a file', async () => { + const fileName = 'foo.txt' + const filePath = `/${fileName}` + await fs.writeBytes(smallFile, filePath) + await expect(fs.stat(filePath)).to.eventually.be.ok() + + await fs.rm(filePath) + + await expect(fs.stat(filePath)).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('removes a directory', async () => { + const dirName = 'foo' + const dirPath = `/${dirName}` + await fs.mkdir(dirPath) + + await fs.rm(dirPath) + + await expect(fs.stat(dirPath)).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + }) + + it('removes a sharded directory inside a normal directory', async () => { + const dirName = `dir-${Math.random()}` + const dirPath = `/${dirName}` + await fs.mkdir(dirPath) + + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = `${dirPath}/sharded-directory` + + await fs.cp(shardedDirCid, shardedDirPath) + + await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'directory') + await expect(fs.stat(shardedDirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + await fs.rm(shardedDirPath) + + await expect(fs.stat(shardedDirPath)).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + + await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'directory') + }) + + it('removes a sharded directory inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-directory' + await fs.cp(shardedDirCid, shardedDirPath) + + const otherShardedDirCid = await createShardedDirectory(blockstore) + const shardedSubDir = `subdir-${Math.random()}` + const shardedSubDirPath = `${shardedDirPath}/${shardedSubDir}` + await fs.cp(otherShardedDirCid, shardedSubDirPath) + + await expect(fs.stat(shardedDirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + await expect(fs.stat(shardedSubDirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + await fs.rm(shardedSubDirPath) + + await expect(fs.stat(shardedSubDirPath)).to.eventually.be.rejected + .with.property('code', 'ERR_DOES_NOT_EXIST') + + const shardedDirStats = await fs.stat(shardedDirPath) + expect(shardedDirStats.cid.toString()).to.equal(shardedDirCid.toString(), 'adding and removing a file from a sharded directory did not result in the original sharded CID') + }) + + it('results in the same hash as a sharded directory created by the importer when removing a file and removing the file means the root node is below the shard threshold', async function () { + const shardSplitThresholdBytes = 55 + + // create a shard with the importer + const importResult = await last(importer([{ + path: 'file-1.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }], blockstore, { + wrapWithDirectory: true, + shardSplitThresholdBytes + })) + + if (importResult == null) { + throw new Error('Nothing imported') + } + + const { cid: importerCid } = importResult + await fs.cp(importerCid, '/importer-dir') + await expect(fs.stat('/importer-dir')).to.eventually.have.nested.property('unixfs.type', 'directory') + + const dirName = 'sharded-dir' + const dirPath = `/${dirName}` + await fs.mkdir(dirPath) + + // create the same shard with unixfs command + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-1.txt`, { + shardSplitThresholdBytes + }) + + await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'directory') + + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-2.txt`, { + shardSplitThresholdBytes + }) + + await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + await fs.rm(`${dirPath}/file-2.txt`, { + shardSplitThresholdBytes + }) + + const dirStats = await fs.stat(dirPath) + expect(dirStats).to.have.nested.property('unixfs.type', 'directory') + expect(dirStats.cid.toString()).to.equal(importerCid.toString()) + }) + + it('results in the same hash as a sharded directory created by the importer when removing a file', async function () { + const shardSplitThresholdBytes = 1 + + // create a shard with the importer + const importResult = await last(importer([{ + path: 'file-1.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }], blockstore, { + wrapWithDirectory: true, + shardSplitThresholdBytes + })) + + if (importResult == null) { + throw new Error('Nothing imported') + } + + const { cid: importerCid } = importResult + await fs.cp(importerCid, '/importer-dir') + await expect(fs.stat('/importer-dir')).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const dirName = 'sharded-dir' + const dirPath = `/${dirName}` + await fs.mkdir(dirPath) + + // create the same shard with unixfs command + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-1.txt`, { + shardSplitThresholdBytes + }) + + await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), `${dirPath}/file-2.txt`, { + shardSplitThresholdBytes + }) + + await expect(fs.stat(dirPath)).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + await fs.rm(`${dirPath}/file-2.txt`, { + shardSplitThresholdBytes + }) + + const dirStats = await fs.stat(dirPath) + expect(dirStats).to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + expect(dirStats.cid.toString()).to.equal(importerCid.toString()) + }) + + it('results in the same hash as a sharded directory created by the importer when removing a subshard', async function () { + const { + containingDirCid, + fileName, + importerCid + } = await createSubshardedDirectory(blockstore) + + await fs.cp(importerCid, '/importer-dir') + await expect(fs.stat('/importer-dir')).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const dirName = 'sharded-dir' + const dirPath = `/${dirName}` + await fs.cp(containingDirCid, dirPath) + + // remove the file that caused the subshard to be created and the CID should be the same as the importer + await fs.rm(`${dirPath}/${fileName}`, { + shardSplitThresholdBytes: 1 + }) + + // should still be a shard + const dirStats = await fs.stat(dirPath) + expect(dirStats) + .to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + expect(dirStats.cid.toString()).to.equal(importerCid.toString(), 'removing a file from the imported dir did not result in the same CID') + }) + + it('results in the same hash as a sharded directory created by the importer when removing a subshard of a subshard', async function () { + const { + containingDirCid, + fileName, + importerCid + } = await createSubshardedDirectory(blockstore, 2) + + await fs.cp(importerCid, '/importer-dir') + await expect(fs.stat('/importer-dir')).to.eventually.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + const dirName = 'sharded-dir' + const dirPath = `/${dirName}` + await fs.cp(containingDirCid, dirPath) + + // remove the file that caused the subshard to be created and the CID should be the same as the importer + await fs.rm(`${dirPath}/${fileName}`, { + shardSplitThresholdBytes: 1 + }) + + // should still be a shard + const dirStats = await fs.stat(dirPath) + expect(dirStats) + .to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + + expect(dirStats.cid.toString()).to.equal(importerCid.toString(), 'removing a file from the imported dir did not result in the same CID') + }) +}) diff --git a/packages/mfs/test/stat.spec.ts b/packages/mfs/test/stat.spec.ts new file mode 100644 index 00000000..ed7f6eef --- /dev/null +++ b/packages/mfs/test/stat.spec.ts @@ -0,0 +1,204 @@ +/* eslint-env mocha */ + +import * as dagPb from '@ipld/dag-pb' +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import { largeFile, smallFile } from './fixtures/files.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('stat', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('stats an empty directory', async () => { + await expect(fs.stat('/')).to.eventually.include({ + fileSize: 0n, + dagSize: 2n, + blocks: 1, + type: 'directory' + }) + }) + + it('computes how much of the DAG is local', async () => { + const filePath = '/foo.txt' + await fs.writeBytes(largeFile, filePath) + + const stats = await fs.stat(filePath) + + const block = await blockstore.get(stats.cid) + const node = dagPb.decode(block) + + expect(node.Links).to.have.lengthOf(5) + + expect(stats).to.include({ + fileSize: 5242880n, + blocks: 6, + localDagSize: 5243139n + }) + + // remove one of the blocks so we now have an incomplete DAG + await blockstore.delete(node.Links[0].Hash) + + // block count and local file/dag sizes should be smaller + await expect(fs.stat(filePath)).to.eventually.include({ + fileSize: 5242880n, + blocks: 5, + localFileSize: 4194304n, + localDagSize: 4194563n + }) + }) + + it('stats a raw node', async () => { + const filePath = '/foo.txt' + await fs.writeBytes(smallFile, filePath) + + await expect(fs.stat(filePath)).to.eventually.include({ + fileSize: BigInt(smallFile.length), + dagSize: 13n, + blocks: 1, + type: 'raw' + }) + }) + + it('stats a small file', async () => { + const filePath = '/foo.txt' + await fs.writeBytes(smallFile, filePath, { + cidVersion: 0, + rawLeaves: false + }) + + await expect(fs.stat(filePath)).to.eventually.include({ + fileSize: BigInt(smallFile.length), + dagSize: 19n, + blocks: 1, + type: 'file' + }) + }) + + it('stats a large file', async () => { + const filePath = '/foo.txt' + await fs.writeBytes(largeFile, filePath) + + await expect(fs.stat(filePath)).to.eventually.include({ + fileSize: BigInt(largeFile.length), + dagSize: 5242907n, + blocks: 6, + type: 'file' + }) + }) + + it('should stat file with mode', async () => { + const mode = 0o644 + const filePath = '/foo.txt' + await fs.writeBytes(smallFile, filePath, { + mode + }) + + await expect(fs.stat(filePath)).to.eventually.include({ + mode + }) + }) + + it('should stat file with mtime', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + const filePath = '/foo.txt' + await fs.writeBytes(smallFile, filePath, { + mtime + }) + + await expect(fs.stat(filePath)).to.eventually.deep.include({ + mtime + }) + }) + + it('should stat a directory', async function () { + await expect(fs.stat('/')).to.eventually.include({ + type: 'directory', + blocks: 1, + fileSize: 0n + }) + }) + + it('should stat dir with mode', async function () { + const mode = 0o755 + const path = '/test-dir' + await fs.mkdir(path, { + mode + }) + + await expect(fs.stat(path)).to.eventually.include({ + mode + }) + }) + + it('should stat dir with mtime', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + + const path = '/test-dir' + await fs.mkdir(path, { + mtime + }) + + await expect(fs.stat(path)).to.eventually.deep.include({ + mtime + }) + }) + + it('stats a sharded directory', async function () { + const mtime = { + secs: 5n, + nsecs: 0 + } + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-directory' + await fs.cp(shardedDirCid, shardedDirPath) + + await fs.touch(shardedDirPath, { + mtime + }) + + const stat = await fs.stat(shardedDirPath) + expect(stat).to.have.property('type', 'directory') + expect(stat).to.have.nested.property('unixfs.type', 'hamt-sharded-directory') + expect(stat).to.include({ + mode: 0o755 + }) + expect(stat).to.deep.include({ + mtime + }) + }) + + it('stats a file inside a sharded directory', async () => { + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-directory' + await fs.cp(shardedDirCid, shardedDirPath) + + const filePath = `${shardedDirPath}/file-inside-sharded-dir.txt` + + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), filePath, { + rawLeaves: false + }) + + const stats = await fs.stat(filePath) + expect(stats.type).to.equal('file') + expect(stats.fileSize).to.equal(4n) + }) +}) diff --git a/packages/mfs/test/touch.spec.ts b/packages/mfs/test/touch.spec.ts new file mode 100644 index 00000000..0869c422 --- /dev/null +++ b/packages/mfs/test/touch.spec.ts @@ -0,0 +1,148 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import delay from 'delay' +import { type MFS, mfs } from '../src/index.js' +import { createShardedDirectory } from './fixtures/create-sharded-directory.js' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +describe('touch', () => { + let blockstore: Blockstore + let datastore: Datastore + let fs: MFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + datastore = new MemoryDatastore() + + fs = mfs({ blockstore, datastore }) + }) + + it('should have default mtime', async () => { + const path = '/foo.txt' + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), path) + + await expect(fs.stat(path)).to.eventually.have.property('mtime') + .that.is.undefined() + + await fs.touch(path) + + await expect(fs.stat(path)).to.eventually.have.property('mtime') + .that.is.not.undefined().and.does.not.deep.equal({ + secs: 0, + nsecs: 0 + }) + }) + + it('should update file mtime', async function () { + this.slow(5 * 1000) + const path = '/foo.txt' + const mtime = new Date() + const seconds = BigInt(Math.floor(mtime.getTime() / 1000)) + + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), path, { + mtime: { + secs: seconds + } + }) + + await delay(2000) + await fs.touch(path) + + await expect(fs.stat(path)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + }) + + it('should update directory mtime', async function () { + this.slow(5 * 1000) + const path = '/foo' + const mtime = new Date() + const seconds = BigInt(Math.floor(mtime.getTime() / 1000)) + + await fs.mkdir(path, { + mtime: { + secs: seconds + } + }) + await delay(2000) + await fs.touch(path) + + await expect(fs.stat(path)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + }) + + it('should update mtime recursively', async function () { + this.slow(5 * 1000) + const path = '/foo.txt' + const mtime = new Date() + const seconds = Math.floor(mtime.getTime() / 1000) + + await fs.writeBytes(Uint8Array.from([0, 1, 2, 3, 4]), path) + + await delay(2000) + await fs.touch(path, { + recursive: true + }) + + await expect(fs.stat(path)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + + await expect(fs.stat(path)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + }) + + it('should update the mtime for a hamt-sharded-directory', async function () { + this.slow(5 * 1000) + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-dir' + await fs.cp(shardedDirCid, shardedDirPath) + await fs.touch(shardedDirPath) + const originalMtime = (await fs.stat(shardedDirPath)).mtime + + if (originalMtime == null) { + throw new Error('No originalMtime found') + } + + await delay(2000) + await fs.touch(shardedDirPath) + const updatedMtime = (await fs.stat(shardedDirPath)).mtime + + if (updatedMtime == null) { + throw new Error('No updatedMtime found') + } + + // no bigint support + expect(updatedMtime.secs).to.satisfy((s: bigint) => s > originalMtime.secs) + }) + + it('should update mtime recursively for a hamt-sharded-directory', async function () { + this.slow(5 * 1000) + const mtime = new Date() + const seconds = Math.floor(mtime.getTime() / 1000) + const shardedDirCid = await createShardedDirectory(blockstore) + const shardedDirPath = '/sharded-dir' + await fs.cp(shardedDirCid, shardedDirPath) + + await delay(2000) + await fs.touch(shardedDirPath, { + recursive: true + }) + + await expect(fs.stat(shardedDirPath)).to.eventually.have.nested.property('mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + + for await (const file of fs.ls(shardedDirPath)) { + expect(file).to.have.nested.property('unixfs.mtime.secs') + // no bigint support + .that.satisfies((s: bigint) => s > seconds) + } + }) +}) diff --git a/packages/mfs/tsconfig.json b/packages/mfs/tsconfig.json new file mode 100644 index 00000000..13a35996 --- /dev/null +++ b/packages/mfs/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/mfs/typedoc.json b/packages/mfs/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/mfs/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} From 0b407a3a7e1da8418fad72e6b3631528a912a493 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 14 Aug 2023 17:07:57 +0100 Subject: [PATCH 128/253] chore: fix up missing deps --- packages/interop/package.json | 6 +----- packages/interop/tsconfig.json | 5 +++++ packages/mfs/package.json | 8 +------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index e59404a1..9ae1978d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -51,7 +51,7 @@ "@chainsafe/libp2p-noise": "^13.0.0", "@chainsafe/libp2p-yamux": "^5.0.0", "@helia/interface": "next", - "@helia/unixfs": "^1.0.0", + "@helia/mfs": "0.0.0", "@libp2p/tcp": "^8.0.2", "@libp2p/websockets": "^7.0.2", "aegir": "^40.0.9", @@ -59,14 +59,10 @@ "datastore-core": "^9.2.0", "go-ipfs": "^0.22.0", "helia": "next", - "ipfs-core-types": "^0.14.0", - "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", - "it-to-buffer": "^4.0.1", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.46.3", "merge-options": "^3.0.4", - "multiformats": "^12.0.1", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json index 13a35996..5869c30d 100644 --- a/packages/interop/tsconfig.json +++ b/packages/interop/tsconfig.json @@ -6,5 +6,10 @@ "include": [ "src", "test" + ], + "references": [ + { + "path": "../mfs" + } ] } diff --git a/packages/mfs/package.json b/packages/mfs/package.json index adcc5d5d..8ea3bdc0 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -139,19 +139,13 @@ "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", "@libp2p/logger": "^3.0.1", - "@multiformats/murmur3": "^2.1.2", - "hamt-sharding": "^3.0.2", "interface-blockstore": "^5.0.0", "interface-datastore": "^8.2.2", "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^13.1.0", "ipfs-unixfs-importer": "^15.1.0", "it-last": "^3.0.1", - "it-pipe": "^3.0.1", - "merge-options": "^3.0.4", - "multiformats": "^12.0.1", - "progress-events": "^1.0.0", - "sparse-array": "^1.3.2" + "multiformats": "^12.0.1" }, "devDependencies": { "aegir": "^40.0.9", From 0e7ca5e3422472712bb83044a483a1a1326ea618 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 14 Aug 2023 17:15:26 +0100 Subject: [PATCH 129/253] chore: use release versions --- packages/interop/package.json | 4 ++-- packages/mfs/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 9ae1978d..9c73ce0e 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -50,7 +50,7 @@ "devDependencies": { "@chainsafe/libp2p-noise": "^13.0.0", "@chainsafe/libp2p-yamux": "^5.0.0", - "@helia/interface": "next", + "@helia/interface": "^1.2.2", "@helia/mfs": "0.0.0", "@libp2p/tcp": "^8.0.2", "@libp2p/websockets": "^7.0.2", @@ -58,7 +58,7 @@ "blockstore-core": "^4.0.1", "datastore-core": "^9.2.0", "go-ipfs": "^0.22.0", - "helia": "next", + "helia": "1.3.12", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.46.3", diff --git a/packages/mfs/package.json b/packages/mfs/package.json index 8ea3bdc0..5f3157ed 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -134,7 +134,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "next", + "@helia/interface": "^1.2.2", "@helia/unixfs": "^1.2.2", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", From 3aceb468c1fd290a287d5763e6c15e30de8e43b8 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 14 Aug 2023 17:16:38 +0100 Subject: [PATCH 130/253] chore: update readme --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 351df654..b24df40e 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,10 @@ ## Table of contents -- - [Structure](#structure) -- [@helia/unixfs ](#heliaunixfs----omit-in-toc---) - - [API Docs](#api-docs) - - [License](#license) - - [Contribute](#contribute) +- [Structure](#structure) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) ## Structure From cfabafde27793eccaf721362f956eabf7a5a833c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 14 Aug 2023 17:40:27 +0100 Subject: [PATCH 131/253] chore: remove boilerplate code --- packages/interop/package.json | 8 ---- .../test/fixtures/create-helia.browser.ts | 46 ------------------- .../interop/test/fixtures/create-helia.ts | 37 +-------------- 3 files changed, 2 insertions(+), 89 deletions(-) delete mode 100644 packages/interop/test/fixtures/create-helia.browser.ts diff --git a/packages/interop/package.json b/packages/interop/package.json index 9c73ce0e..680860d0 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,25 +48,17 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^13.0.0", - "@chainsafe/libp2p-yamux": "^5.0.0", "@helia/interface": "^1.2.2", "@helia/mfs": "0.0.0", - "@libp2p/tcp": "^8.0.2", - "@libp2p/websockets": "^7.0.2", "aegir": "^40.0.9", - "blockstore-core": "^4.0.1", - "datastore-core": "^9.2.0", "go-ipfs": "^0.22.0", "helia": "1.3.12", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.46.3", "merge-options": "^3.0.4", "wherearewe": "^2.0.1" }, "browser": { - "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, "private": true diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts deleted file mode 100644 index 867de492..00000000 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { webSockets } from '@libp2p/websockets' -import { all } from '@libp2p/websockets/filters' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' -import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' -import type { Helia } from '@helia/interface' - -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - // dial-only in the browser until webrtc browser-to-browser arrives - const libp2p = await createLibp2p({ - transports: [ - webSockets({ - filter: all - }) - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - services: { - identify: identifyService() - }, - datastore, - connectionGater: { - denyDialMultiaddr: async () => false - }, - ...config - }) - - const helia = await createHelia({ - libp2p, - blockstore, - datastore - }) - - return helia -} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index f7a25c3f..8743d7b6 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,39 +1,6 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { tcp } from '@libp2p/tcp' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - const libp2p = await createLibp2p({ - transports: [ - tcp() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identifyService() - }, - ...config - }) - - const helia = await createHelia({ - libp2p, - blockstore, - datastore - }) - - return helia +export async function createHeliaNode (): Promise { + return createHelia() } From 205e3632c60a787fc12f70efe6d74cdd40cd5acc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 14 Aug 2023 16:50:29 +0000 Subject: [PATCH 132/253] chore(release): 1.0.0 [skip ci] ## @helia/mfs-v1.0.0 (2023-08-14) ### Features * initial import ([a70f4eb](https://github.com/ipfs/helia-mfs/commit/a70f4eb982e377eeeeb6fd4a53f7baf40c09641b)) ### Trivial Changes * fix up missing deps ([0b407a3](https://github.com/ipfs/helia-mfs/commit/0b407a3a7e1da8418fad72e6b3631528a912a493)) * use release versions ([0e7ca5e](https://github.com/ipfs/helia-mfs/commit/0e7ca5e3422472712bb83044a483a1a1326ea618)) --- packages/mfs/CHANGELOG.md | 12 ++++++++++++ packages/mfs/package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/mfs/CHANGELOG.md diff --git a/packages/mfs/CHANGELOG.md b/packages/mfs/CHANGELOG.md new file mode 100644 index 00000000..a063bd9f --- /dev/null +++ b/packages/mfs/CHANGELOG.md @@ -0,0 +1,12 @@ +## @helia/mfs-v1.0.0 (2023-08-14) + + +### Features + +* initial import ([a70f4eb](https://github.com/ipfs/helia-mfs/commit/a70f4eb982e377eeeeb6fd4a53f7baf40c09641b)) + + +### Trivial Changes + +* fix up missing deps ([0b407a3](https://github.com/ipfs/helia-mfs/commit/0b407a3a7e1da8418fad72e6b3631528a912a493)) +* use release versions ([0e7ca5e](https://github.com/ipfs/helia-mfs/commit/0e7ca5e3422472712bb83044a483a1a1326ea618)) diff --git a/packages/mfs/package.json b/packages/mfs/package.json index 5f3157ed..e1b6805c 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/mfs", - "version": "0.0.0", + "version": "1.0.0", "description": "A mutable filesystem powered by Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/mfs#readme", From 8c775f5985569dc61a97a1a7a9ff4b8d57ad8443 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 14 Aug 2023 16:50:39 +0000 Subject: [PATCH 133/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 680860d0..414e0171 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -49,7 +49,7 @@ }, "devDependencies": { "@helia/interface": "^1.2.2", - "@helia/mfs": "0.0.0", + "@helia/mfs": "^1.0.0", "aegir": "^40.0.9", "go-ipfs": "^0.22.0", "helia": "1.3.12", From 3ba3e6d5712810d7a30b494f3cc363f6176022b3 Mon Sep 17 00:00:00 2001 From: GitHub Date: Sun, 20 Aug 2023 08:25:12 +0000 Subject: [PATCH 134/253] chore: Update .github/workflows/stale.yml [skip ci] --- .github/workflows/stale.yml | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,25 +2,12 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 From 174987b2817cfe99cbabb9835dd6a2d99c1c35a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 09:12:16 +0100 Subject: [PATCH 135/253] deps(dev): bump aegir from 39.0.13 to 40.0.8 (#65) * deps(dev): bump aegir from 39.0.13 to 40.0.8 Bumps [aegir](https://github.com/ipfs/aegir) from 39.0.13 to 40.0.8. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v39.0.13...v40.0.8) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * chore: fix linting * chore: update config * chore: fix docs --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 10 +++------- packages/interop/package.json | 11 ++--------- packages/interop/test/fixtures/wait-for.ts | 1 - packages/ipns/package.json | 9 +-------- packages/ipns/src/index.ts | 2 +- packages/ipns/typedoc.json | 6 ++++++ typedoc.json | 4 ++++ 7 files changed, 17 insertions(+), 26 deletions(-) create mode 100644 packages/ipns/typedoc.json create mode 100644 typedoc.json diff --git a/package.json b/package.json index 059fda1a..e68070dd 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ "keywords": [ "ipfs" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "private": true, "scripts": { "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", @@ -33,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", "docs:no-publish": "npm run docs -- --publish false", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "npm run docs:no-publish && aegir run release && npm run docs -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^39.0.8" + "aegir": "^40.0.8" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index edc05fda..92cead5d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -65,7 +61,7 @@ "@libp2p/peer-id-factory": "^2.0.1", "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.8", + "aegir": "^40.0.8", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", "go-ipfs": "^0.20.0", @@ -86,8 +82,5 @@ "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, - "private": true, - "typedoc": { - "entryPoint": "./src/index.ts" - } + "private": true } diff --git a/packages/interop/test/fixtures/wait-for.ts b/packages/interop/test/fixtures/wait-for.ts index 3c1451f1..bdcd1671 100644 --- a/packages/interop/test/fixtures/wait-for.ts +++ b/packages/interop/test/fixtures/wait-for.ts @@ -1,4 +1,3 @@ - export interface WaitForOptions { timeout: number delay?: number diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 06ab1446..af4d12d0 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "typesVersions": { @@ -177,15 +173,12 @@ }, "devDependencies": { "@libp2p/peer-id-factory": "^2.0.1", - "aegir": "^39.0.8", + "aegir": "^40.0.8", "datastore-core": "^9.0.3", "sinon": "^15.0.1", "sinon-ts": "^1.0.0" }, "browser": { "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" - }, - "typedoc": { - "entryPoint": "./src/index.ts" } } diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 9d89b630..1686b194 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -12,7 +12,7 @@ * import { createHelia } from 'helia' * import { ipns, ipnsValidator, ipnsSelector } from '@helia/ipns' * import { dht, pubsub } from '@helia/ipns/routing' - * import { unixfs } from '@helia/unixfs + * import { unixfs } from '@helia/unixfs' * * const libp2p = await createLibp2p({ * dht: kadDHT({ diff --git a/packages/ipns/typedoc.json b/packages/ipns/typedoc.json new file mode 100644 index 00000000..0c509d8d --- /dev/null +++ b/packages/ipns/typedoc.json @@ -0,0 +1,6 @@ +{ + "entryPoints": [ + "./src/index.ts", + "./src/routing/index.ts" + ] +} diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 00000000..7abd348b --- /dev/null +++ b/typedoc.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "name": "@helia/ipns" +} From 6f93e51e9b6f603f7c1d396705dc5b190108fe79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 09:13:00 +0100 Subject: [PATCH 136/253] deps: bump multiformats from 11.0.2 to 12.0.1 (#57) Bumps [multiformats](https://github.com/multiformats/js-multiformats) from 11.0.2 to 12.0.1. - [Release notes](https://github.com/multiformats/js-multiformats/releases) - [Changelog](https://github.com/multiformats/js-multiformats/blob/master/CHANGELOG.md) - [Commits](https://github.com/multiformats/js-multiformats/compare/v11.0.2...v12.0.1) --- updated-dependencies: - dependency-name: multiformats dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- packages/ipns/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 92cead5d..3c0fe9f9 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -74,7 +74,7 @@ "kubo-rpc-client": "^3.0.0", "libp2p": "^0.45.3", "merge-options": "^3.0.4", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "uint8arrays": "^4.0.3", "wherearewe": "^2.0.1" }, diff --git a/packages/ipns/package.json b/packages/ipns/package.json index af4d12d0..258d7c14 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -166,7 +166,7 @@ "interface-datastore": "^8.0.0", "ipns": "^6.0.0", "is-ipfs": "^8.0.1", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "p-queue": "^7.3.0", "progress-events": "^1.0.0", "uint8arrays": "^4.0.3" From 973bb5b6c8db0fedd70e4058f97bc339018a8193 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 09:49:36 +0100 Subject: [PATCH 137/253] deps(dev): bump aegir from 39.0.13 to 40.0.11 (#29) Bumps [aegir](https://github.com/ipfs/aegir) from 39.0.13 to 40.0.11. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v39.0.13...v40.0.11) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- .github/workflows/js-test-and-release.yml | 1 + package.json | 10 ++-- packages/dag-cbor/package.json | 9 +--- packages/dag-cbor/src/index.ts | 14 +++--- packages/dag-cbor/typedoc.json | 5 ++ packages/interop/package.json | 17 +------ .../test/fixtures/create-helia.browser.ts | 50 ++++++------------- .../interop/test/fixtures/create-helia.ts | 35 +------------ packages/interop/test/fixtures/create-kubo.ts | 1 - 9 files changed, 35 insertions(+), 107 deletions(-) create mode 100644 packages/dag-cbor/typedoc.json diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 51686414..a15c3791 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -17,6 +17,7 @@ jobs: - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present lint - run: npm run --if-present dep-check + - run: npm run --if-present doc-check test-node: needs: check diff --git a/package.json b/package.json index 7f59adb7..610641af 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ "keywords": [ "ipfs" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "private": true, "scripts": { "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", @@ -33,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", - "docs:no-publish": "npm run docs -- --publish false", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^39.0.3" + "aegir": "^40.0.11" }, "type": "module", "workspaces": [ diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 6d417ebd..1a18d414 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -147,10 +143,7 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^39.0.3", + "aegir": "^40.0.11", "blockstore-core": "^4.0.1" - }, - "typedoc": { - "entryPoint": "./src/index.ts" } } diff --git a/packages/dag-cbor/src/index.ts b/packages/dag-cbor/src/index.ts index 18f148f9..5d9e2a6c 100644 --- a/packages/dag-cbor/src/index.ts +++ b/packages/dag-cbor/src/index.ts @@ -3,7 +3,7 @@ * * `@helia/dag-cbor` makes working with DAG-JSON {@link https://github.com/ipfs/helia Helia} simple & straightforward. * - * See the {@link Strings Strings interface} for all available operations. + * See the {@link DAGCBOR} interface for all available operations. * * @example * @@ -56,10 +56,10 @@ export interface DAGCBOR { * @example * * ```typescript - * import { json } from '@helia/dag-cbor' + * import { dagCbor } from '@helia/dag-cbor' * - * const j = json(helia) - * const cid = await str.add({ hello: 'world' }) + * const d = dagCbor(helia) + * const cid = await d.add({ hello: 'world' }) * * console.info(cid) * // CID(bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae) @@ -74,12 +74,12 @@ export interface DAGCBOR { * @example * * ```typescript - * import { json } from '@helia/dag-cbor' + * import { dagCbor } from '@helia/dag-cbor' * import { CID } from 'multiformats/cid' * - * const j = json(helia) + * const d = dagCbor(helia) * const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') - * const obj = await j.get(cid) + * const obj = await d.get(cid) * * console.info(obj) * // { hello: 'world' } diff --git a/packages/dag-cbor/typedoc.json b/packages/dag-cbor/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/dag-cbor/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/interop/package.json b/packages/interop/package.json index 50e05cca..30640c34 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -52,22 +48,16 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/dag-cbor": "~0.0.0", "@helia/interface": "^1.0.0", "@ipld/dag-cbor": "^9.0.0", - "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.3", - "blockstore-core": "^4.0.1", - "datastore-core": "^9.0.3", + "aegir": "^40.0.11", "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", "merge-options": "^3.0.4", "wherearewe": "^2.0.1" }, @@ -75,8 +65,5 @@ "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, - "private": true, - "typedoc": { - "entryPoint": "./src/index.ts" - } + "private": true } diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 867de492..906ca86e 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,45 +1,23 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - // dial-only in the browser until webrtc browser-to-browser arrives - const libp2p = await createLibp2p({ - transports: [ - webSockets({ - filter: all - }) - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - services: { - identify: identifyService() - }, - datastore, - connectionGater: { - denyDialMultiaddr: async () => false - }, - ...config - }) - +export async function createHeliaNode (): Promise { const helia = await createHelia({ - libp2p, - blockstore, - datastore + libp2p: { + addresses: { + listen: [] + }, + transports: [ + webSockets({ + filter: all + }) + ], + connectionGater: { + denyDialMultiaddr: async () => false + } + } }) return helia diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index f7a25c3f..903dfc16 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,39 +1,8 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { tcp } from '@libp2p/tcp' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - const libp2p = await createLibp2p({ - transports: [ - tcp() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identifyService() - }, - ...config - }) - - const helia = await createHelia({ - libp2p, - blockstore, - datastore - }) +export async function createHeliaNode (): Promise { + const helia = await createHelia() return helia } diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 52ed76e1..ab7e24e1 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ - // @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' From 5434035bcc098294a0f40a61247c8693c1e2393f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 08:56:47 +0000 Subject: [PATCH 138/253] chore(release): 1.0.0 [skip ci] ## @helia/dag-cbor-v1.0.0 (2023-08-27) ### Features * initial commit ([ed4c319](https://github.com/ipfs/helia-dag-cbor/commit/ed4c319a67c18a3dd65e18f18aa12e82080b3fdc)) ### Dependencies * **dev:** bump aegir from 39.0.13 to 40.0.11 ([#29](https://github.com/ipfs/helia-dag-cbor/issues/29)) ([973bb5b](https://github.com/ipfs/helia-dag-cbor/commit/973bb5b6c8db0fedd70e4058f97bc339018a8193)) --- packages/dag-cbor/CHANGELOG.md | 11 +++++++++++ packages/dag-cbor/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/dag-cbor/CHANGELOG.md b/packages/dag-cbor/CHANGELOG.md index e69de29b..e6e7a2dc 100644 --- a/packages/dag-cbor/CHANGELOG.md +++ b/packages/dag-cbor/CHANGELOG.md @@ -0,0 +1,11 @@ +## @helia/dag-cbor-v1.0.0 (2023-08-27) + + +### Features + +* initial commit ([ed4c319](https://github.com/ipfs/helia-dag-cbor/commit/ed4c319a67c18a3dd65e18f18aa12e82080b3fdc)) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.11 ([#29](https://github.com/ipfs/helia-dag-cbor/issues/29)) ([973bb5b](https://github.com/ipfs/helia-dag-cbor/commit/973bb5b6c8db0fedd70e4058f97bc339018a8193)) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 1a18d414..6b9c8332 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-cbor", - "version": "0.0.0", + "version": "1.0.0", "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/dag-cbor#readme", From 088527aa4ef66fef4f0c4611f93af9d9aa0eedd4 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Sun, 27 Aug 2023 08:56:57 +0000 Subject: [PATCH 139/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 30640c34..2b8708ba 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,7 +48,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/dag-cbor": "~0.0.0", + "@helia/dag-cbor": "^1.0.0", "@helia/interface": "^1.0.0", "@ipld/dag-cbor": "^9.0.0", "@libp2p/websockets": "^6.0.1", From d126e6a3c845f25a4910c18fa476304d8534be91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:06:09 +0100 Subject: [PATCH 140/253] deps(dev): bump aegir from 39.0.13 to 40.0.11 (#28) * deps(dev): bump aegir from 39.0.13 to 40.0.11 Bumps [aegir](https://github.com/ipfs/aegir) from 39.0.13 to 40.0.11. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v39.0.13...v40.0.11) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- .github/workflows/js-test-and-release.yml | 1 + package.json | 10 ++-- packages/dag-json/package.json | 10 +--- packages/dag-json/src/index.ts | 2 +- packages/dag-json/typedoc.json | 5 ++ packages/interop/package.json | 17 +------ .../test/fixtures/create-helia.browser.ts | 50 ++++++------------- .../interop/test/fixtures/create-helia.ts | 35 +------------ packages/interop/test/fixtures/create-kubo.ts | 1 - 9 files changed, 29 insertions(+), 102 deletions(-) create mode 100644 packages/dag-json/typedoc.json diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 51686414..a15c3791 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -17,6 +17,7 @@ jobs: - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present lint - run: npm run --if-present dep-check + - run: npm run --if-present doc-check test-node: needs: check diff --git a/package.json b/package.json index 180614e2..105ff48e 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ "keywords": [ "ipfs" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "private": true, "scripts": { "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", @@ -33,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", - "docs:no-publish": "npm run docs -- --publish false", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^39.0.3" + "aegir": "^40.0.11" }, "type": "module", "workspaces": [ diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 0a9cab52..5da3442f 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -128,7 +124,6 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", - "docs": "aegir docs", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", @@ -147,10 +142,7 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^39.0.3", + "aegir": "^40.0.11", "blockstore-core": "^4.0.1" - }, - "typedoc": { - "entryPoint": "./src/index.ts" } } diff --git a/packages/dag-json/src/index.ts b/packages/dag-json/src/index.ts index 860a3094..d123449b 100644 --- a/packages/dag-json/src/index.ts +++ b/packages/dag-json/src/index.ts @@ -3,7 +3,7 @@ * * `@helia/dag-json` makes working with DAG-JSON {@link https://github.com/ipfs/helia Helia} simple & straightforward. * - * See the {@link Strings Strings interface} for all available operations. + * See the {@link DAGJSON} interface for all available operations. * * @example * diff --git a/packages/dag-json/typedoc.json b/packages/dag-json/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/dag-json/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/interop/package.json b/packages/interop/package.json index c6b01fb4..4e93b1c5 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -52,21 +48,15 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/dag-json": "^1.0.0", "@helia/interface": "^1.0.0", - "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.3", - "blockstore-core": "^4.0.1", - "datastore-core": "^9.0.3", + "aegir": "^40.0.11", "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.2", "wherearewe": "^2.0.1" @@ -75,8 +65,5 @@ "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, - "private": true, - "typedoc": { - "entryPoint": "./src/index.ts" - } + "private": true } diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 867de492..906ca86e 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,45 +1,23 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - // dial-only in the browser until webrtc browser-to-browser arrives - const libp2p = await createLibp2p({ - transports: [ - webSockets({ - filter: all - }) - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - services: { - identify: identifyService() - }, - datastore, - connectionGater: { - denyDialMultiaddr: async () => false - }, - ...config - }) - +export async function createHeliaNode (): Promise { const helia = await createHelia({ - libp2p, - blockstore, - datastore + libp2p: { + addresses: { + listen: [] + }, + transports: [ + webSockets({ + filter: all + }) + ], + connectionGater: { + denyDialMultiaddr: async () => false + } + } }) return helia diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index f7a25c3f..903dfc16 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,39 +1,8 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { tcp } from '@libp2p/tcp' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - const libp2p = await createLibp2p({ - transports: [ - tcp() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identifyService() - }, - ...config - }) - - const helia = await createHelia({ - libp2p, - blockstore, - datastore - }) +export async function createHeliaNode (): Promise { + const helia = await createHelia() return helia } diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 52ed76e1..ab7e24e1 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ - // @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' From 2c89d9f3b61e9975e98f58535bc708bf4fc3927f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:13:19 +0100 Subject: [PATCH 141/253] deps(dev): bump go-ipfs from 0.20.0 to 0.22.0 (#24) * deps(dev): bump go-ipfs from 0.20.0 to 0.22.0 Bumps [go-ipfs](https://github.com/ipfs/npm-go-ipfs) from 0.20.0 to 0.22.0. - [Commits](https://github.com/ipfs/npm-go-ipfs/compare/v0.20.0...v0.22.0) --- updated-dependencies: - dependency-name: go-ipfs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 2 +- packages/interop/test/fixtures/create-kubo.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 2b8708ba..92afb95f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,7 +53,7 @@ "@ipld/dag-cbor": "^9.0.0", "@libp2p/websockets": "^6.0.1", "aegir": "^40.0.11", - "go-ipfs": "^0.20.0", + "go-ipfs": "^0.22.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index ab7e24e1..c032e86d 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ -// @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' From 4354316e6870440bf04ecb14bf323649b4582c05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:13:36 +0100 Subject: [PATCH 142/253] deps(dev): bump @libp2p/websockets from 6.0.3 to 7.0.5 (#35) Bumps [@libp2p/websockets](https://github.com/libp2p/js-libp2p) from 6.0.3 to 7.0.5. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/.release-please.json) - [Commits](https://github.com/libp2p/js-libp2p/compare/peer-record-v6.0.3...websockets-v7.0.5) --- updated-dependencies: - dependency-name: "@libp2p/websockets" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 92afb95f..88b6fb02 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -51,7 +51,7 @@ "@helia/dag-cbor": "^1.0.0", "@helia/interface": "^1.0.0", "@ipld/dag-cbor": "^9.0.0", - "@libp2p/websockets": "^6.0.1", + "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", "helia": "^1.0.0", From 7a842d3cc4cd97e02e5a196aa512cfe36be4c388 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:13:48 +0100 Subject: [PATCH 143/253] deps: bump multiformats from 11.0.2 to 12.0.1 (#8) Bumps [multiformats](https://github.com/multiformats/js-multiformats) from 11.0.2 to 12.0.1. - [Release notes](https://github.com/multiformats/js-multiformats/releases) - [Changelog](https://github.com/multiformats/js-multiformats/blob/master/CHANGELOG.md) - [Commits](https://github.com/multiformats/js-multiformats/compare/v11.0.2...v12.0.1) --- updated-dependencies: - dependency-name: multiformats dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/dag-cbor/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 6b9c8332..c076ce6d 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -139,7 +139,7 @@ "@ipld/dag-cbor": "^9.0.0", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "progress-events": "^1.0.0" }, "devDependencies": { From ea26a0bd14137eb1de6ab282cdcecd55578064ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:14:14 +0100 Subject: [PATCH 144/253] deps(dev): bump aegir from 39.0.13 to 40.0.11 (#30) * deps(dev): bump aegir from 39.0.13 to 40.0.11 Bumps [aegir](https://github.com/ipfs/aegir) from 39.0.13 to 40.0.11. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v39.0.13...v40.0.11) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 6 +-- packages/car/package.json | 6 +-- packages/car/src/index.ts | 2 +- packages/car/typedoc.json | 6 +++ packages/interop/package.json | 13 +---- .../test/fixtures/create-helia.browser.ts | 50 ++++++------------- .../interop/test/fixtures/create-helia.ts | 35 +------------ packages/interop/test/fixtures/create-kubo.ts | 1 - 8 files changed, 29 insertions(+), 90 deletions(-) create mode 100644 packages/car/typedoc.json diff --git a/package.json b/package.json index 92535a03..a5d20a28 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", - "docs:no-publish": "npm run docs -- --publish false", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^39.0.8" + "aegir": "^40.0.11" }, "type": "module", "workspaces": [ diff --git a/packages/car/package.json b/packages/car/package.json index 7d38bf43..12c03322 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -144,7 +144,6 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", - "docs": "aegir docs", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", @@ -169,13 +168,10 @@ }, "devDependencies": { "@helia/unixfs": "^1.2.2", - "aegir": "^39.0.8", + "aegir": "^40.0.11", "blockstore-core": "^4.0.1", "ipfs-unixfs-importer": "^15.1.0", "it-drain": "^3.0.1", "it-to-buffer": "^4.0.2" - }, - "typedoc": { - "entryPoint": "./src/index.ts" } } diff --git a/packages/car/src/index.ts b/packages/car/src/index.ts index 8640a4b4..1759daa2 100644 --- a/packages/car/src/index.ts +++ b/packages/car/src/index.ts @@ -3,7 +3,7 @@ * * `@helia/car` provides `import` and `export` methods to read/write Car files to {@link https://github.com/ipfs/helia Helia}'s blockstore. * - * See the {@link Car Car interface} for all available operations. + * See the {@link Car} interface for all available operations. * * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more esoteric DAG walkers can be passed as an init option. * diff --git a/packages/car/typedoc.json b/packages/car/typedoc.json new file mode 100644 index 00000000..d63e8ce5 --- /dev/null +++ b/packages/car/typedoc.json @@ -0,0 +1,6 @@ +{ + "entryPoints": [ + "./src/index.ts", + "./src/errors.ts" + ] +} diff --git a/packages/interop/package.json b/packages/interop/package.json index 49fd5942..68ad01cd 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,24 +48,18 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.2", "@helia/car": "^1.0.0", "@helia/interface": "^1.0.0", "@helia/unixfs": "^1.2.2", "@ipld/car": "^5.1.1", - "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.8", - "blockstore-core": "^4.0.1", - "datastore-core": "^9.0.3", + "aegir": "^40.0.11", "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^4.0.2", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.1", "p-defer": "^4.0.0", @@ -75,8 +69,5 @@ "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, - "private": true, - "typedoc": { - "entryPoint": "./src/index.ts" - } + "private": true } diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 867de492..906ca86e 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,45 +1,23 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - // dial-only in the browser until webrtc browser-to-browser arrives - const libp2p = await createLibp2p({ - transports: [ - webSockets({ - filter: all - }) - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - services: { - identify: identifyService() - }, - datastore, - connectionGater: { - denyDialMultiaddr: async () => false - }, - ...config - }) - +export async function createHeliaNode (): Promise { const helia = await createHelia({ - libp2p, - blockstore, - datastore + libp2p: { + addresses: { + listen: [] + }, + transports: [ + webSockets({ + filter: all + }) + ], + connectionGater: { + denyDialMultiaddr: async () => false + } + } }) return helia diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index f7a25c3f..903dfc16 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,39 +1,8 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { tcp } from '@libp2p/tcp' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - const libp2p = await createLibp2p({ - transports: [ - tcp() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identifyService() - }, - ...config - }) - - const helia = await createHelia({ - libp2p, - blockstore, - datastore - }) +export async function createHeliaNode (): Promise { + const helia = await createHelia() return helia } diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 52ed76e1..ab7e24e1 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ - // @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' From 1b2e696f4eab544607e09a7baa1e92d8f8e48f5a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 09:16:30 +0000 Subject: [PATCH 145/253] chore(release): 1.0.1 [skip ci] ## [@helia/dag-json-v1.0.1](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.0...@helia/dag-json-v1.0.1) (2023-08-27) ### Dependencies * **dev:** bump aegir from 39.0.13 to 40.0.11 ([#28](https://github.com/ipfs/helia-dag-json/issues/28)) ([d126e6a](https://github.com/ipfs/helia-dag-json/commit/d126e6a3c845f25a4910c18fa476304d8534be91)) --- packages/dag-json/CHANGELOG.md | 7 +++++++ packages/dag-json/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dag-json/CHANGELOG.md b/packages/dag-json/CHANGELOG.md index 1a13d6eb..bb923ec1 100644 --- a/packages/dag-json/CHANGELOG.md +++ b/packages/dag-json/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/dag-json-v1.0.1](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.0...@helia/dag-json-v1.0.1) (2023-08-27) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.11 ([#28](https://github.com/ipfs/helia-dag-json/issues/28)) ([d126e6a](https://github.com/ipfs/helia-dag-json/commit/d126e6a3c845f25a4910c18fa476304d8534be91)) + ## @helia/dag-json-v1.0.0 (2023-05-03) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 5da3442f..f3b18eb8 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-json", - "version": "1.0.0", + "version": "1.0.1", "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", From 104a1dd17e4e4e01a0b48de06cceceb40ff0025c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:20:03 +0100 Subject: [PATCH 146/253] deps(dev): bump go-ipfs from 0.20.0 to 0.22.0 (#24) Bumps [go-ipfs](https://github.com/ipfs/npm-go-ipfs) from 0.20.0 to 0.22.0. - [Commits](https://github.com/ipfs/npm-go-ipfs/compare/v0.20.0...v0.22.0) --- updated-dependencies: - dependency-name: go-ipfs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 2 +- packages/interop/test/fixtures/create-kubo.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 4e93b1c5..595d9272 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "@helia/interface": "^1.0.0", "@libp2p/websockets": "^6.0.1", "aegir": "^40.0.11", - "go-ipfs": "^0.20.0", + "go-ipfs": "^0.22.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index ab7e24e1..c032e86d 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ -// @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' From 2836bb85b75d32cbe4b0dc7bd3ef85912a26396d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:20:16 +0100 Subject: [PATCH 147/253] deps(dev): bump @libp2p/websockets from 6.0.3 to 7.0.5 (#35) Bumps [@libp2p/websockets](https://github.com/libp2p/js-libp2p) from 6.0.3 to 7.0.5. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/.release-please.json) - [Commits](https://github.com/libp2p/js-libp2p/compare/peer-record-v6.0.3...websockets-v7.0.5) --- updated-dependencies: - dependency-name: "@libp2p/websockets" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 595d9272..100f7734 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -50,7 +50,7 @@ "devDependencies": { "@helia/dag-json": "^1.0.0", "@helia/interface": "^1.0.0", - "@libp2p/websockets": "^6.0.1", + "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", "helia": "^1.0.0", From c89b8f12d700f0e23dc574cc32f7726d9c9558de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:21:33 +0100 Subject: [PATCH 148/253] deps: bump multiformats from 11.0.2 to 12.0.1 (#8) Bumps [multiformats](https://github.com/multiformats/js-multiformats) from 11.0.2 to 12.0.1. - [Release notes](https://github.com/multiformats/js-multiformats/releases) - [Changelog](https://github.com/multiformats/js-multiformats/blob/master/CHANGELOG.md) - [Commits](https://github.com/multiformats/js-multiformats/compare/v11.0.2...v12.0.1) --- updated-dependencies: - dependency-name: multiformats dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/dag-json/package.json | 2 +- packages/interop/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index f3b18eb8..b7e1081e 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -138,7 +138,7 @@ "@ipld/dag-json": "^10.0.1", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "progress-events": "^1.0.0" }, "devDependencies": { diff --git a/packages/interop/package.json b/packages/interop/package.json index 100f7734..90e7a067 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,7 +58,7 @@ "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", - "multiformats": "^11.0.2", + "multiformats": "^12.0.1", "wherearewe": "^2.0.1" }, "browser": { From 546afefdf3abd4ab39070b739d31d5a118ba0de3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 09:22:40 +0000 Subject: [PATCH 149/253] chore(release): 1.0.2 [skip ci] ## [@helia/car-v1.0.2](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.1...@helia/car-v1.0.2) (2023-08-27) ### Dependencies * **dev:** bump aegir from 39.0.13 to 40.0.11 ([#30](https://github.com/ipfs/helia-car/issues/30)) ([ea26a0b](https://github.com/ipfs/helia-car/commit/ea26a0bd14137eb1de6ab282cdcecd55578064ab)) --- packages/car/CHANGELOG.md | 7 +++++++ packages/car/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/car/CHANGELOG.md b/packages/car/CHANGELOG.md index c550554a..245ec11d 100644 --- a/packages/car/CHANGELOG.md +++ b/packages/car/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/car-v1.0.2](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.1...@helia/car-v1.0.2) (2023-08-27) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.11 ([#30](https://github.com/ipfs/helia-car/issues/30)) ([ea26a0b](https://github.com/ipfs/helia-car/commit/ea26a0bd14137eb1de6ab282cdcecd55578064ab)) + ## [@helia/car-v1.0.1](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.0...@helia/car-v1.0.1) (2023-08-14) diff --git a/packages/car/package.json b/packages/car/package.json index 12c03322..c0d93ce9 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -1,6 +1,6 @@ { "name": "@helia/car", - "version": "1.0.1", + "version": "1.0.2", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", From f559d644b3c3d113e2036160b0c66c20b0a8c57f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 09:32:26 +0000 Subject: [PATCH 150/253] chore(release): 1.0.1 [skip ci] ## [@helia/dag-cbor-v1.0.1](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.0...@helia/dag-cbor-v1.0.1) (2023-08-27) ### Dependencies * bump multiformats from 11.0.2 to 12.0.1 ([#8](https://github.com/ipfs/helia-dag-cbor/issues/8)) ([7a842d3](https://github.com/ipfs/helia-dag-cbor/commit/7a842d3cc4cd97e02e5a196aa512cfe36be4c388)) --- packages/dag-cbor/CHANGELOG.md | 7 +++++++ packages/dag-cbor/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dag-cbor/CHANGELOG.md b/packages/dag-cbor/CHANGELOG.md index e6e7a2dc..bb92e628 100644 --- a/packages/dag-cbor/CHANGELOG.md +++ b/packages/dag-cbor/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/dag-cbor-v1.0.1](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.0...@helia/dag-cbor-v1.0.1) (2023-08-27) + + +### Dependencies + +* bump multiformats from 11.0.2 to 12.0.1 ([#8](https://github.com/ipfs/helia-dag-cbor/issues/8)) ([7a842d3](https://github.com/ipfs/helia-dag-cbor/commit/7a842d3cc4cd97e02e5a196aa512cfe36be4c388)) + ## @helia/dag-cbor-v1.0.0 (2023-08-27) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index c076ce6d..27a8d63e 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-cbor", - "version": "1.0.0", + "version": "1.0.1", "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/dag-cbor#readme", From 602947cf5b75b89cc5053aaae1ae88eb263fe649 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 09:41:06 +0000 Subject: [PATCH 151/253] chore(release): 1.0.2 [skip ci] ## [@helia/dag-json-v1.0.2](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.1...@helia/dag-json-v1.0.2) (2023-08-27) ### Dependencies * bump multiformats from 11.0.2 to 12.0.1 ([#8](https://github.com/ipfs/helia-dag-json/issues/8)) ([c89b8f1](https://github.com/ipfs/helia-dag-json/commit/c89b8f12d700f0e23dc574cc32f7726d9c9558de)) --- packages/dag-json/CHANGELOG.md | 7 +++++++ packages/dag-json/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dag-json/CHANGELOG.md b/packages/dag-json/CHANGELOG.md index bb923ec1..3b56dee4 100644 --- a/packages/dag-json/CHANGELOG.md +++ b/packages/dag-json/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/dag-json-v1.0.2](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.1...@helia/dag-json-v1.0.2) (2023-08-27) + + +### Dependencies + +* bump multiformats from 11.0.2 to 12.0.1 ([#8](https://github.com/ipfs/helia-dag-json/issues/8)) ([c89b8f1](https://github.com/ipfs/helia-dag-json/commit/c89b8f12d700f0e23dc574cc32f7726d9c9558de)) + ## [@helia/dag-json-v1.0.1](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.0...@helia/dag-json-v1.0.1) (2023-08-27) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index b7e1081e..3925672e 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-json", - "version": "1.0.1", + "version": "1.0.2", "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", From aa6ebcf9f58eebf842113985adee4710b009562d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:41:57 +0100 Subject: [PATCH 152/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#30) Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/dag-cbor/package.json | 2 +- packages/interop/package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 27a8d63e..519d46b8 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -135,7 +135,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@ipld/dag-cbor": "^9.0.0", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", diff --git a/packages/interop/package.json b/packages/interop/package.json index 88b6fb02..e2b49566 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -49,12 +49,12 @@ }, "devDependencies": { "@helia/dag-cbor": "^1.0.0", - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@ipld/dag-cbor": "^9.0.0", "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", - "helia": "^1.0.0", + "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", From 07ed91191d7b3d4de11bde5293fce17f7145fcaf Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 09:49:15 +0000 Subject: [PATCH 153/253] chore(release): 1.0.2 [skip ci] ## [@helia/dag-cbor-v1.0.2](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.1...@helia/dag-cbor-v1.0.2) (2023-08-27) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#30](https://github.com/ipfs/helia-dag-cbor/issues/30)) ([aa6ebcf](https://github.com/ipfs/helia-dag-cbor/commit/aa6ebcf9f58eebf842113985adee4710b009562d)) --- packages/dag-cbor/CHANGELOG.md | 7 +++++++ packages/dag-cbor/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dag-cbor/CHANGELOG.md b/packages/dag-cbor/CHANGELOG.md index bb92e628..76b702fd 100644 --- a/packages/dag-cbor/CHANGELOG.md +++ b/packages/dag-cbor/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/dag-cbor-v1.0.2](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.1...@helia/dag-cbor-v1.0.2) (2023-08-27) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#30](https://github.com/ipfs/helia-dag-cbor/issues/30)) ([aa6ebcf](https://github.com/ipfs/helia-dag-cbor/commit/aa6ebcf9f58eebf842113985adee4710b009562d)) + ## [@helia/dag-cbor-v1.0.1](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.0...@helia/dag-cbor-v1.0.1) (2023-08-27) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 519d46b8..84c94071 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-cbor", - "version": "1.0.1", + "version": "1.0.2", "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/dag-cbor#readme", From c8a2e7ff11df35b6c7e4e3d336890e5d9f5f51fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:58:41 +0100 Subject: [PATCH 154/253] deps(dev): bump go-ipfs from 0.20.0 to 0.22.0 (#22) Bumps [go-ipfs](https://github.com/ipfs/npm-go-ipfs) from 0.20.0 to 0.22.0. - [Commits](https://github.com/ipfs/npm-go-ipfs/compare/v0.20.0...v0.22.0) --- updated-dependencies: - dependency-name: go-ipfs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 2 +- packages/interop/test/fixtures/create-kubo.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 68ad01cd..54e9a4e6 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -54,7 +54,7 @@ "@ipld/car": "^5.1.1", "@libp2p/websockets": "^6.0.1", "aegir": "^40.0.11", - "go-ipfs": "^0.20.0", + "go-ipfs": "^0.22.0", "helia": "^1.0.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index ab7e24e1..c032e86d 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ -// @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' From eb836ef15f6bc754fbab4fdbe47c76f5492a56d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:16:34 +0100 Subject: [PATCH 155/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#32) * deps: bump @helia/interface from 1.2.2 to 2.0.0 Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * deps(dev): bump helia from 1.3.12 to 2.0.1 (#31) Bumps [helia](https://github.com/ipfs/helia) from 1.3.12 to 2.0.1. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/helia-v1.3.12...helia-v2.0.1) --- updated-dependencies: - dependency-name: helia dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/dag-json/package.json | 2 +- packages/interop/package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 3925672e..eef0e31a 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -134,7 +134,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@ipld/dag-json": "^10.0.1", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", diff --git a/packages/interop/package.json b/packages/interop/package.json index 90e7a067..695f1e96 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -49,11 +49,11 @@ }, "devDependencies": { "@helia/dag-json": "^1.0.0", - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", - "helia": "^1.0.0", + "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", From 5f4169e9ddb16a7d026db395ad3e9c1a2f764bb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:16:58 +0100 Subject: [PATCH 156/253] deps(dev): bump @libp2p/websockets from 6.0.3 to 7.0.5 (#36) Bumps [@libp2p/websockets](https://github.com/libp2p/js-libp2p) from 6.0.3 to 7.0.5. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/.release-please.json) - [Commits](https://github.com/libp2p/js-libp2p/compare/peer-record-v6.0.3...websockets-v7.0.5) --- updated-dependencies: - dependency-name: "@libp2p/websockets" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 54e9a4e6..fbece72f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "@helia/interface": "^1.0.0", "@helia/unixfs": "^1.2.2", "@ipld/car": "^5.1.1", - "@libp2p/websockets": "^6.0.1", + "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", "helia": "^1.0.0", From 68656a81b7cd1238641a41573915635905e4a6ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:17:07 +0100 Subject: [PATCH 157/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#32) * deps: bump @helia/interface from 1.2.2 to 2.0.0 Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * chore: update helia --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/car/package.json | 2 +- packages/interop/package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/car/package.json b/packages/car/package.json index c0d93ce9..f23e646d 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -154,7 +154,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@ipld/car": "^5.1.1", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", diff --git a/packages/interop/package.json b/packages/interop/package.json index fbece72f..a052a996 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -49,13 +49,13 @@ }, "devDependencies": { "@helia/car": "^1.0.0", - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@helia/unixfs": "^1.2.2", "@ipld/car": "^5.1.1", "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", - "helia": "^1.0.0", + "helia": "^2.0.1", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^4.0.2", From 10994ea9abdff8906ae8c3f7d0ff5f50b50d9e60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:17:15 +0100 Subject: [PATCH 158/253] deps: bump cborg from 1.10.2 to 2.0.5 (#35) Bumps [cborg](https://github.com/rvagg/cborg) from 1.10.2 to 2.0.5. - [Release notes](https://github.com/rvagg/cborg/releases) - [Changelog](https://github.com/rvagg/cborg/blob/master/CHANGELOG.md) - [Commits](https://github.com/rvagg/cborg/compare/v1.10.2...v2.0.5) --- updated-dependencies: - dependency-name: cborg dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/car/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/car/package.json b/packages/car/package.json index f23e646d..0127d74f 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -158,7 +158,7 @@ "@ipld/car": "^5.1.1", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", - "cborg": "^1.10.2", + "cborg": "^2.0.5", "interface-blockstore": "^5.0.0", "it-map": "^3.0.3", "multiformats": "^11.0.1", From 50bed0f32b3c07111de804b0e6471e36d8e66626 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:17:24 +0100 Subject: [PATCH 159/253] deps: bump multiformats from 11.0.2 to 12.0.1 (#4) Bumps [multiformats](https://github.com/multiformats/js-multiformats) from 11.0.2 to 12.0.1. - [Release notes](https://github.com/multiformats/js-multiformats/releases) - [Changelog](https://github.com/multiformats/js-multiformats/blob/master/CHANGELOG.md) - [Commits](https://github.com/multiformats/js-multiformats/compare/v11.0.2...v12.0.1) --- updated-dependencies: - dependency-name: multiformats dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/car/package.json | 2 +- packages/interop/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/car/package.json b/packages/car/package.json index 0127d74f..846666e3 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -161,7 +161,7 @@ "cborg": "^2.0.5", "interface-blockstore": "^5.0.0", "it-map": "^3.0.3", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "p-defer": "^4.0.0", "p-queue": "^7.3.4", "progress-events": "^1.0.0" diff --git a/packages/interop/package.json b/packages/interop/package.json index a052a996..5df0045a 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -61,7 +61,7 @@ "it-to-buffer": "^4.0.2", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "p-defer": "^4.0.0", "wherearewe": "^2.0.1" }, From 37b6ba14e085073b966fced3c3777519601d0a81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:20:55 +0100 Subject: [PATCH 160/253] deps(dev): bump aegir from 39.0.13 to 40.0.11 (#26) Bumps [aegir](https://github.com/ipfs/aegir) from 39.0.13 to 40.0.11. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v39.0.13...v40.0.11) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- .github/workflows/js-test-and-release.yml | 1 + package.json | 10 ++-- packages/interop/package.json | 17 +------ .../test/fixtures/create-helia.browser.ts | 50 ++++++------------- .../interop/test/fixtures/create-helia.ts | 35 +------------ packages/interop/test/fixtures/create-kubo.ts | 1 - packages/json/package.json | 10 +--- packages/json/src/index.ts | 2 +- packages/json/typedoc.json | 5 ++ 9 files changed, 29 insertions(+), 102 deletions(-) create mode 100644 packages/json/typedoc.json diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 51686414..a15c3791 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -17,6 +17,7 @@ jobs: - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present lint - run: npm run --if-present dep-check + - run: npm run --if-present doc-check test-node: needs: check diff --git a/package.json b/package.json index 135b1be4..f9cf02ce 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ "keywords": [ "ipfs" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "private": true, "scripts": { "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", @@ -33,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", - "docs:no-publish": "npm run docs -- --publish false", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^39.0.3" + "aegir": "^40.0.11" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 406403dc..9743f60d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -52,21 +48,15 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.1", "@helia/interface": "^1.0.0", "@helia/json": "^1.0.0", - "@libp2p/tcp": "^7.0.1", "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.3", - "blockstore-core": "^4.0.1", - "datastore-core": "^9.0.3", + "aegir": "^40.0.11", "go-ipfs": "^0.20.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", "merge-options": "^3.0.4", "multiformats": "^11.0.2", "wherearewe": "^2.0.1" @@ -75,8 +65,5 @@ "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, - "private": true, - "typedoc": { - "entryPoint": "./src/index.ts" - } + "private": true } diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 867de492..906ca86e 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,45 +1,23 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - // dial-only in the browser until webrtc browser-to-browser arrives - const libp2p = await createLibp2p({ - transports: [ - webSockets({ - filter: all - }) - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - services: { - identify: identifyService() - }, - datastore, - connectionGater: { - denyDialMultiaddr: async () => false - }, - ...config - }) - +export async function createHeliaNode (): Promise { const helia = await createHelia({ - libp2p, - blockstore, - datastore + libp2p: { + addresses: { + listen: [] + }, + transports: [ + webSockets({ + filter: all + }) + ], + connectionGater: { + denyDialMultiaddr: async () => false + } + } }) return helia diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index f7a25c3f..903dfc16 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,39 +1,8 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { tcp } from '@libp2p/tcp' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() - - const libp2p = await createLibp2p({ - transports: [ - tcp() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identifyService() - }, - ...config - }) - - const helia = await createHelia({ - libp2p, - blockstore, - datastore - }) +export async function createHeliaNode (): Promise { + const helia = await createHelia() return helia } diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 52ed76e1..ab7e24e1 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ - // @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' diff --git a/packages/json/package.json b/packages/json/package.json index c6919255..001ba10a 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -128,7 +124,6 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", - "docs": "aegir docs", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", @@ -146,10 +141,7 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^39.0.3", + "aegir": "^40.0.11", "blockstore-core": "^4.0.1" - }, - "typedoc": { - "entryPoint": "./src/index.ts" } } diff --git a/packages/json/src/index.ts b/packages/json/src/index.ts index 1d39213f..25ae0459 100644 --- a/packages/json/src/index.ts +++ b/packages/json/src/index.ts @@ -3,7 +3,7 @@ * * `@helia/json` makes working with JSON in {@link https://github.com/ipfs/helia Helia} simple & straightforward. * - * See the {@link JSON JSON interface} for all available operations. + * See the {@link JSON} interface for all available operations. * * @example * diff --git a/packages/json/typedoc.json b/packages/json/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/json/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} From 3d87c6880e426cf98c5e9877861f4ac96d52576f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 11:36:41 +0000 Subject: [PATCH 161/253] chore(release): 1.0.3 [skip ci] ## [@helia/car-v1.0.3](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.2...@helia/car-v1.0.3) (2023-08-27) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#32](https://github.com/ipfs/helia-car/issues/32)) ([68656a8](https://github.com/ipfs/helia-car/commit/68656a81b7cd1238641a41573915635905e4a6ed)) * bump cborg from 1.10.2 to 2.0.5 ([#35](https://github.com/ipfs/helia-car/issues/35)) ([10994ea](https://github.com/ipfs/helia-car/commit/10994ea9abdff8906ae8c3f7d0ff5f50b50d9e60)) * bump multiformats from 11.0.2 to 12.0.1 ([#4](https://github.com/ipfs/helia-car/issues/4)) ([50bed0f](https://github.com/ipfs/helia-car/commit/50bed0f32b3c07111de804b0e6471e36d8e66626)) --- packages/car/CHANGELOG.md | 9 +++++++++ packages/car/package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/car/CHANGELOG.md b/packages/car/CHANGELOG.md index 245ec11d..578e2995 100644 --- a/packages/car/CHANGELOG.md +++ b/packages/car/CHANGELOG.md @@ -1,3 +1,12 @@ +## [@helia/car-v1.0.3](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.2...@helia/car-v1.0.3) (2023-08-27) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#32](https://github.com/ipfs/helia-car/issues/32)) ([68656a8](https://github.com/ipfs/helia-car/commit/68656a81b7cd1238641a41573915635905e4a6ed)) +* bump cborg from 1.10.2 to 2.0.5 ([#35](https://github.com/ipfs/helia-car/issues/35)) ([10994ea](https://github.com/ipfs/helia-car/commit/10994ea9abdff8906ae8c3f7d0ff5f50b50d9e60)) +* bump multiformats from 11.0.2 to 12.0.1 ([#4](https://github.com/ipfs/helia-car/issues/4)) ([50bed0f](https://github.com/ipfs/helia-car/commit/50bed0f32b3c07111de804b0e6471e36d8e66626)) + ## [@helia/car-v1.0.2](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.1...@helia/car-v1.0.2) (2023-08-27) diff --git a/packages/car/package.json b/packages/car/package.json index 846666e3..aae1d6d2 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -1,6 +1,6 @@ { "name": "@helia/car", - "version": "1.0.2", + "version": "1.0.3", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", From cb8bc2095e8d0440c4d5c8e76611a3aa75fda5d3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 11:40:19 +0000 Subject: [PATCH 162/253] chore(release): 1.0.1 [skip ci] ## [@helia/json-v1.0.1](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.0...@helia/json-v1.0.1) (2023-08-27) ### Dependencies * **dev:** bump aegir from 39.0.13 to 40.0.11 ([#26](https://github.com/ipfs/helia-json/issues/26)) ([37b6ba1](https://github.com/ipfs/helia-json/commit/37b6ba14e085073b966fced3c3777519601d0a81)) --- packages/json/CHANGELOG.md | 7 +++++++ packages/json/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/json/CHANGELOG.md b/packages/json/CHANGELOG.md index b4debf14..fe0306ae 100644 --- a/packages/json/CHANGELOG.md +++ b/packages/json/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/json-v1.0.1](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.0...@helia/json-v1.0.1) (2023-08-27) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.11 ([#26](https://github.com/ipfs/helia-json/issues/26)) ([37b6ba1](https://github.com/ipfs/helia-json/commit/37b6ba14e085073b966fced3c3777519601d0a81)) + ## @helia/json-v1.0.0 (2023-05-03) diff --git a/packages/json/package.json b/packages/json/package.json index 001ba10a..2d382342 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/json", - "version": "1.0.0", + "version": "1.0.1", "description": "Add/get IPLD blocks containing json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/json#readme", From cff694f6bc88b25e71d3243b045c65932bfa9874 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:51:20 +0100 Subject: [PATCH 163/253] deps(dev): bump go-ipfs from 0.20.0 to 0.22.0 (#24) Bumps [go-ipfs](https://github.com/ipfs/npm-go-ipfs) from 0.20.0 to 0.22.0. - [Commits](https://github.com/ipfs/npm-go-ipfs/compare/v0.20.0...v0.22.0) --- updated-dependencies: - dependency-name: go-ipfs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 2 +- packages/interop/test/fixtures/create-kubo.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 9743f60d..cbc4707a 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "@helia/json": "^1.0.0", "@libp2p/websockets": "^6.0.1", "aegir": "^40.0.11", - "go-ipfs": "^0.20.0", + "go-ipfs": "^0.22.0", "helia": "^1.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index ab7e24e1..c032e86d 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ -// @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' From de048348666a7961cda517ce26f8aedfa987a3bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:51:31 +0100 Subject: [PATCH 164/253] deps(dev): bump @libp2p/websockets from 6.0.3 to 7.0.5 (#35) Bumps [@libp2p/websockets](https://github.com/libp2p/js-libp2p) from 6.0.3 to 7.0.5. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/.release-please.json) - [Commits](https://github.com/libp2p/js-libp2p/compare/peer-record-v6.0.3...websockets-v7.0.5) --- updated-dependencies: - dependency-name: "@libp2p/websockets" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index cbc4707a..70ea9a3c 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -50,7 +50,7 @@ "devDependencies": { "@helia/interface": "^1.0.0", "@helia/json": "^1.0.0", - "@libp2p/websockets": "^6.0.1", + "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", "helia": "^1.0.0", From d48f2c58338af0d096a1f855ab85a621fce1cc01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 13:09:37 +0100 Subject: [PATCH 165/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#34) Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * deps(dev): bump helia from 1.3.12 to 2.0.1 (#32) Bumps [helia](https://github.com/ipfs/helia) from 1.3.12 to 2.0.1. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/helia-v1.3.12...helia-v2.0.1) --- updated-dependencies: - dependency-name: helia dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/interop/package.json | 4 ++-- packages/json/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 70ea9a3c..80ad4841 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,12 +48,12 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@helia/json": "^1.0.0", "@libp2p/websockets": "^7.0.5", "aegir": "^40.0.11", "go-ipfs": "^0.22.0", - "helia": "^1.0.0", + "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", diff --git a/packages/json/package.json b/packages/json/package.json index 2d382342..e6899a7b 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -134,7 +134,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", "multiformats": "^11.0.1", From c2a2ee38cc8fa76c8a6d0c92c44023c148148a7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 13:09:55 +0100 Subject: [PATCH 166/253] deps: bump multiformats from 11.0.2 to 12.0.1 (#8) Bumps [multiformats](https://github.com/multiformats/js-multiformats) from 11.0.2 to 12.0.1. - [Release notes](https://github.com/multiformats/js-multiformats/releases) - [Changelog](https://github.com/multiformats/js-multiformats/blob/master/CHANGELOG.md) - [Commits](https://github.com/multiformats/js-multiformats/compare/v11.0.2...v12.0.1) --- updated-dependencies: - dependency-name: multiformats dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- packages/json/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 80ad4841..4da8f6fd 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,7 +58,7 @@ "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", - "multiformats": "^11.0.2", + "multiformats": "^12.0.1", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/json/package.json b/packages/json/package.json index e6899a7b..d7367efb 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -137,7 +137,7 @@ "@helia/interface": "^2.0.0", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", - "multiformats": "^11.0.1", + "multiformats": "^12.0.1", "progress-events": "^1.0.0" }, "devDependencies": { From 3bff2668b3a0d0a9ec0a31b90feba74e38aa93b9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 27 Aug 2023 12:27:36 +0000 Subject: [PATCH 167/253] chore(release): 1.0.2 [skip ci] ## [@helia/json-v1.0.2](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.1...@helia/json-v1.0.2) (2023-08-27) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#34](https://github.com/ipfs/helia-json/issues/34)) ([d48f2c5](https://github.com/ipfs/helia-json/commit/d48f2c58338af0d096a1f855ab85a621fce1cc01)) * bump multiformats from 11.0.2 to 12.0.1 ([#8](https://github.com/ipfs/helia-json/issues/8)) ([c2a2ee3](https://github.com/ipfs/helia-json/commit/c2a2ee38cc8fa76c8a6d0c92c44023c148148a7e)) --- packages/json/CHANGELOG.md | 8 ++++++++ packages/json/package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/json/CHANGELOG.md b/packages/json/CHANGELOG.md index fe0306ae..2ca813a6 100644 --- a/packages/json/CHANGELOG.md +++ b/packages/json/CHANGELOG.md @@ -1,3 +1,11 @@ +## [@helia/json-v1.0.2](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.1...@helia/json-v1.0.2) (2023-08-27) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#34](https://github.com/ipfs/helia-json/issues/34)) ([d48f2c5](https://github.com/ipfs/helia-json/commit/d48f2c58338af0d096a1f855ab85a621fce1cc01)) +* bump multiformats from 11.0.2 to 12.0.1 ([#8](https://github.com/ipfs/helia-json/issues/8)) ([c2a2ee3](https://github.com/ipfs/helia-json/commit/c2a2ee38cc8fa76c8a6d0c92c44023c148148a7e)) + ## [@helia/json-v1.0.1](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.0...@helia/json-v1.0.1) (2023-08-27) diff --git a/packages/json/package.json b/packages/json/package.json index d7367efb..cf4339aa 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/json", - "version": "1.0.1", + "version": "1.0.2", "description": "Add/get IPLD blocks containing json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/json#readme", From 8d1c9b286811b354be48afd858187e3748fac9c3 Mon Sep 17 00:00:00 2001 From: web3-bot Date: Mon, 28 Aug 2023 12:25:00 +0000 Subject: [PATCH 168/253] chore: delete templates [skip ci] --- .github/workflows/automerge.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .github/workflows/automerge.yml diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index d57c2a02..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Automerge -on: [ pull_request ] - -jobs: - automerge: - uses: protocol/.github/.github/workflows/automerge.yml@master - with: - job: 'automerge' From 1b0b81cbcb4891a3bc31d847718fc57fcdacfe68 Mon Sep 17 00:00:00 2001 From: web3-bot Date: Mon, 28 Aug 2023 12:30:37 +0000 Subject: [PATCH 169/253] chore: add or force update .github/workflows/js-test-and-release.yml --- .github/workflows/js-test-and-release.yml | 189 ++-------------------- 1 file changed, 16 insertions(+), 173 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index a15c3791..35d87d10 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,182 +1,25 @@ name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - run: npm run --if-present doc-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + packages: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} From 351fae7a129e642a6f312c9a61609273dec190bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:24:41 +0100 Subject: [PATCH 170/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#2) * deps: bump @helia/interface from 1.2.2 to 2.0.0 Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * deps(dev): bump helia from 1.3.12 to 2.0.1 (#1) Bumps [helia](https://github.com/ipfs/helia) from 1.3.12 to 2.0.1. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/helia-v1.3.12...helia-v2.0.1) --- updated-dependencies: - dependency-name: helia dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/interop/package.json | 4 ++-- packages/mfs/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 414e0171..da51e9d4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,11 +48,11 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/interface": "^1.2.2", + "@helia/interface": "^2.0.0", "@helia/mfs": "^1.0.0", "aegir": "^40.0.9", "go-ipfs": "^0.22.0", - "helia": "1.3.12", + "helia": "2.0.1", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", diff --git a/packages/mfs/package.json b/packages/mfs/package.json index e1b6805c..73de32ce 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -134,7 +134,7 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.2.2", + "@helia/interface": "^2.0.0", "@helia/unixfs": "^1.2.2", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", From 88b23b0db4ac9da2a9e94291f2db7b10f436ce00 Mon Sep 17 00:00:00 2001 From: Bonjour Internet Date: Fri, 8 Sep 2023 17:30:50 +0200 Subject: [PATCH 171/253] docs: update docs to use MFS style API (#4) The MFS API does not return a CID from every operation so update the docs to reflect that. --------- Co-authored-by: achingbrain --- packages/mfs/src/index.ts | 91 ++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 48 deletions(-) diff --git a/packages/mfs/src/index.ts b/packages/mfs/src/index.ts index 1fa81b9b..bbc9bde1 100644 --- a/packages/mfs/src/index.ts +++ b/packages/mfs/src/index.ts @@ -88,20 +88,18 @@ export interface RmOptions extends UnixFsRmOptions { */ export interface MFS { /** - * Add a single `Uint8Array` to your Helia node as a file. + * Add a single `Uint8Array` to your MFS as a file. * * @example * * ```typescript - * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) - * - * console.info(cid) + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') * ``` */ writeBytes: (bytes: Uint8Array, path: string, options?: Partial) => Promise /** - * Add a stream of `Uint8Array` to your Helia node as a file. + * Add a stream of `Uint8Array` to your MFS as a file. * * @example * @@ -109,20 +107,18 @@ export interface MFS { * import fs from 'node:fs' * * const stream = fs.createReadStream('./foo.txt') - * const cid = await fs.addByteStream(stream) - * - * console.info(cid) + * await fs.writeByteStream(stream, '/foo.txt') * ``` */ writeByteStream: (bytes: ByteStream, path: string, options?: Partial) => Promise /** - * Retrieve the contents of a file from your Helia node. + * Retrieve the contents of a file from your MFS. * * @example * * ```typescript - * for await (const buf of fs.cat(cid)) { + * for await (const buf of fs.cat('/foo.txt')) { * console.info(buf) * } * ``` @@ -130,112 +126,111 @@ export interface MFS { cat: (path: string, options?: Partial) => AsyncIterable /** - * Change the permissions on a file or directory in a DAG + * Change the permissions on a file or directory in your MFS * * @example * * ```typescript - * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) - * const beforeStats = await fs.stat(beforeCid) + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') + * const beforeStats = await fs.stat('/foo.txt') * - * const afterCid = await fs.chmod(cid, 0x755) - * const afterStats = await fs.stat(afterCid) + * await fs.chmod('/foo.txt', 0x755) + * const afterStats = await fs.stat('/foo.txt') * - * console.info(beforeCid, beforeStats) - * console.info(afterCid, afterStats) + * console.info(beforeStats) + * console.info(afterStats) * ``` */ chmod: (path: string, mode: number, options?: Partial) => Promise /** - * Add a file or directory to a target directory. + * Add a file or directory to a target directory in your MFS. * * @example * * ```typescript - * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) - * const directoryCid = await fs.addDirectory() + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') + * await fs.mkdir('/bar') + * + * await fs.cp('/foo.txt', '/bar') + * ``` + * + * Copy a file from one place to another in your MFS. + * + * @example * - * const updatedCid = await fs.cp(fileCid, directoryCid, 'foo.txt') + * ```typescript + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') * - * console.info(updatedCid) + * await fs.cp('/foo.txt', '/bar.txt') * ``` */ cp: (source: CID | string, destination: string, options?: Partial) => Promise /** - * List directory contents. + * List directory contents from your MFS. * * @example * * ```typescript - * for await (const entry of fs.ls(directoryCid)) { - * console.info(etnry) + * for await (const entry of fs.ls('/bar')) { + * console.info(entry) * } * ``` */ ls: (path?: string, options?: Partial) => AsyncIterable /** - * Make a new directory under an existing directory. + * Make a new directory in your MFS. * * @example * * ```typescript - * const directoryCid = await fs.addDirectory() - * - * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') - * - * console.info(updatedCid) + * await fs.mkdir('/new-dir') * ``` */ mkdir: (path: string, options?: Partial) => Promise /** - * Remove a file or directory from an existing directory. + * Remove a file or directory from your MFS. * * @example * * ```typescript - * const directoryCid = await fs.addDirectory() - * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') - * - * const finalCid = await fs.rm(updatedCid, 'new-dir') - * - * console.info(finalCid) + * await fs.mkdir('/new-dir') + * await fs.rm('/new-dir') * ``` */ rm: (path: string, options?: Partial) => Promise /** - * Return statistics about a UnixFS DAG. + * Return statistics about a UnixFS DAG in your MFS. * * @example * * ```typescript - * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) - * - * const stats = await fs.stat(fileCid) + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') * + * const stats = await fs.stat('/foo.txt') * console.info(stats) * ``` */ stat: (path: string, options?: Partial) => Promise /** - * Update the mtime of a UnixFS DAG + * Update the mtime of a UnixFS DAG in your MFS. * * @example * * ```typescript - * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) - * const beforeStats = await fs.stat(beforeCid) + * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') + * const beforeStats = await fs.stat('/foo.txt') * - * const afterCid = await fs.touch(beforeCid) + * await fs.touch('/foo.txt') * const afterStats = await fs.stat(afterCid) * - * console.info(beforeCid, beforeStats) - * console.info(afterCid, afterStats) + * console.info(beforeStats) + * console.info(afterStats) * ``` */ touch: (path: string, options?: Partial) => Promise From 077021a34dd77014c7cdee38336fc9a682a47f73 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 8 Sep 2023 15:39:04 +0000 Subject: [PATCH 172/253] chore(release): 1.0.1 [skip ci] ## [@helia/mfs-v1.0.1](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.0...@helia/mfs-v1.0.1) (2023-09-08) ### Documentation * update docs to use MFS style API ([#4](https://github.com/ipfs/helia-mfs/issues/4)) ([88b23b0](https://github.com/ipfs/helia-mfs/commit/88b23b0db4ac9da2a9e94291f2db7b10f436ce00)) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#2](https://github.com/ipfs/helia-mfs/issues/2)) ([351fae7](https://github.com/ipfs/helia-mfs/commit/351fae7a129e642a6f312c9a61609273dec190bf)) --- packages/mfs/CHANGELOG.md | 12 ++++++++++++ packages/mfs/package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/mfs/CHANGELOG.md b/packages/mfs/CHANGELOG.md index a063bd9f..9cd89e2f 100644 --- a/packages/mfs/CHANGELOG.md +++ b/packages/mfs/CHANGELOG.md @@ -1,3 +1,15 @@ +## [@helia/mfs-v1.0.1](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.0...@helia/mfs-v1.0.1) (2023-09-08) + + +### Documentation + +* update docs to use MFS style API ([#4](https://github.com/ipfs/helia-mfs/issues/4)) ([88b23b0](https://github.com/ipfs/helia-mfs/commit/88b23b0db4ac9da2a9e94291f2db7b10f436ce00)) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#2](https://github.com/ipfs/helia-mfs/issues/2)) ([351fae7](https://github.com/ipfs/helia-mfs/commit/351fae7a129e642a6f312c9a61609273dec190bf)) + ## @helia/mfs-v1.0.0 (2023-08-14) diff --git a/packages/mfs/package.json b/packages/mfs/package.json index 73de32ce..8dd7bc25 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/mfs", - "version": "1.0.0", + "version": "1.0.1", "description": "A mutable filesystem powered by Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/mfs#readme", From a704fdcbe8507ea97065f40525f0bbc251b57a4d Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 11 Sep 2023 09:40:21 +0100 Subject: [PATCH 173/253] chore: update project config (#99) --- packages/interop/typedoc.json | 5 +++++ packages/ipns/package.json | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 packages/interop/typedoc.json diff --git a/packages/interop/typedoc.json b/packages/interop/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/interop/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 258d7c14..74f2a27c 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -144,7 +144,6 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", - "docs": "aegir docs", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", From 7ea13fcd390849e37a15077e8bb81388d0d0f565 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 11 Sep 2023 09:55:18 +0100 Subject: [PATCH 174/253] chore: fix docs command --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index e68070dd..20cf7f17 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,8 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", - "docs:no-publish": "npm run docs -- --publish false", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", "release": "npm run docs:no-publish && aegir run release && npm run docs -- --exclude packages/interop" }, From 58a5a11a2b95ba14230f5a20bdc205414f6a67a4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Sep 2023 09:02:13 +0000 Subject: [PATCH 175/253] chore(release): 1.1.4 [skip ci] ## [@helia/ipns-v1.1.4](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.3...@helia/ipns-v1.1.4) (2023-09-11) ### Trivial Changes * update project config ([#99](https://github.com/ipfs/helia-ipns/issues/99)) ([a704fdc](https://github.com/ipfs/helia-ipns/commit/a704fdcbe8507ea97065f40525f0bbc251b57a4d)) ### Dependencies * bump multiformats from 11.0.2 to 12.0.1 ([#57](https://github.com/ipfs/helia-ipns/issues/57)) ([6f93e51](https://github.com/ipfs/helia-ipns/commit/6f93e51e9b6f603f7c1d396705dc5b190108fe79)) * **dev:** bump aegir from 39.0.13 to 40.0.8 ([#65](https://github.com/ipfs/helia-ipns/issues/65)) ([174987b](https://github.com/ipfs/helia-ipns/commit/174987b2817cfe99cbabb9835dd6a2d99c1c35a9)) --- packages/ipns/CHANGELOG.md | 13 +++++++++++++ packages/ipns/package.json | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 79f228ef..bb855eee 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,16 @@ +## [@helia/ipns-v1.1.4](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.3...@helia/ipns-v1.1.4) (2023-09-11) + + +### Trivial Changes + +* update project config ([#99](https://github.com/ipfs/helia-ipns/issues/99)) ([a704fdc](https://github.com/ipfs/helia-ipns/commit/a704fdcbe8507ea97065f40525f0bbc251b57a4d)) + + +### Dependencies + +* bump multiformats from 11.0.2 to 12.0.1 ([#57](https://github.com/ipfs/helia-ipns/issues/57)) ([6f93e51](https://github.com/ipfs/helia-ipns/commit/6f93e51e9b6f603f7c1d396705dc5b190108fe79)) +* **dev:** bump aegir from 39.0.13 to 40.0.8 ([#65](https://github.com/ipfs/helia-ipns/issues/65)) ([174987b](https://github.com/ipfs/helia-ipns/commit/174987b2817cfe99cbabb9835dd6a2d99c1c35a9)) + ## [@helia/ipns-v1.1.3](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.2...@helia/ipns-v1.1.3) (2023-05-24) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 74f2a27c..7b22b4a9 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.3", + "version": "1.1.4", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 15fb86380bff97b54120009fb20a0dc5bfa4cc92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:10:07 +0100 Subject: [PATCH 176/253] deps(dev): bump go-ipfs from 0.20.0 to 0.22.0 (#81) Bumps [go-ipfs](https://github.com/ipfs/npm-go-ipfs) from 0.20.0 to 0.22.0. - [Commits](https://github.com/ipfs/npm-go-ipfs/compare/v0.20.0...v0.22.0) --- updated-dependencies: - dependency-name: go-ipfs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 3c0fe9f9..716bfcf0 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -64,7 +64,7 @@ "aegir": "^40.0.8", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.20.0", + "go-ipfs": "^0.22.0", "helia": "^1.0.0", "ipfsd-ctl": "^13.0.0", "ipns": "^6.0.0", From b2886b9598a66a31c69ee0c3c7e13748614be37e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:10:30 +0100 Subject: [PATCH 177/253] deps: bump @libp2p/logger from 2.1.1 to 3.0.2 (#87) Bumps [@libp2p/logger](https://github.com/libp2p/js-libp2p) from 2.1.1 to 3.0.2. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/.release-please.json) - [Commits](https://github.com/libp2p/js-libp2p/commits/logger-v3.0.2) --- updated-dependencies: - dependency-name: "@libp2p/logger" dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/ipns/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 7b22b4a9..b6aa1c9c 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -158,7 +158,7 @@ "@libp2p/interface-peer-id": "^2.0.1", "@libp2p/interface-pubsub": "^4.0.1", "@libp2p/interfaces": "^3.3.1", - "@libp2p/logger": "^2.0.6", + "@libp2p/logger": "^3.0.2", "@libp2p/peer-id": "^2.0.1", "@libp2p/record": "^3.0.0", "hashlru": "^2.3.0", From 9cf48aa8e5ee76552ab8b8a50f5d6439eb3439e3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Sep 2023 09:18:20 +0000 Subject: [PATCH 178/253] chore(release): 1.1.5 [skip ci] ## [@helia/ipns-v1.1.5](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.4...@helia/ipns-v1.1.5) (2023-09-11) ### Dependencies * bump @libp2p/logger from 2.1.1 to 3.0.2 ([#87](https://github.com/ipfs/helia-ipns/issues/87)) ([b2886b9](https://github.com/ipfs/helia-ipns/commit/b2886b9598a66a31c69ee0c3c7e13748614be37e)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index bb855eee..f8ca7e46 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.5](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.4...@helia/ipns-v1.1.5) (2023-09-11) + + +### Dependencies + +* bump @libp2p/logger from 2.1.1 to 3.0.2 ([#87](https://github.com/ipfs/helia-ipns/issues/87)) ([b2886b9](https://github.com/ipfs/helia-ipns/commit/b2886b9598a66a31c69ee0c3c7e13748614be37e)) + ## [@helia/ipns-v1.1.4](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.3...@helia/ipns-v1.1.4) (2023-09-11) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index b6aa1c9c..ade6b1aa 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.4", + "version": "1.1.5", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From efe02e5b38992189edb40cd34d79e76dca4c34a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 11:31:09 +0100 Subject: [PATCH 179/253] deps(dev): bump libp2p from 0.45.9 to 0.46.6 (#92) Bumps [libp2p](https://github.com/libp2p/js-libp2p) from 0.45.9 to 0.46.6. - [Release notes](https://github.com/libp2p/js-libp2p/releases) - [Changelog](https://github.com/libp2p/js-libp2p/blob/master/.release-please.json) - [Commits](https://github.com/libp2p/js-libp2p/compare/libp2p-v0.45.9...libp2p-v0.46.6) --- updated-dependencies: - dependency-name: libp2p dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- packages/interop/package.json | 26 +++++++++---------- packages/interop/test/dht.spec.ts | 6 ++--- .../test/fixtures/create-helia.browser.ts | 2 +- .../interop/test/fixtures/create-peer-ids.ts | 2 +- packages/interop/test/fixtures/key-types.ts | 2 +- packages/interop/test/pubsub.spec.ts | 2 +- packages/ipns/package.json | 9 +++---- packages/ipns/src/index.ts | 6 ++--- packages/ipns/src/routing/dht.ts | 2 +- packages/ipns/src/routing/index.ts | 2 +- packages/ipns/src/routing/local-store.ts | 2 +- packages/ipns/src/routing/pubsub.ts | 6 ++--- .../src/utils/resolve-dns-link.browser.ts | 2 +- packages/ipns/src/utils/resolve-dns-link.ts | 2 +- 14 files changed, 33 insertions(+), 38 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 716bfcf0..5f762e03 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,31 +48,29 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-gossipsub": "^8.0.0", - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.2", - "@helia/interface": "^1.0.0", + "@chainsafe/libp2p-gossipsub": "^10.1.0", + "@chainsafe/libp2p-noise": "^13.0.0", + "@chainsafe/libp2p-yamux": "^5.0.0", + "@helia/interface": "^2.0.0", "@helia/ipns": "^1.0.0", - "@libp2p/interface-libp2p": "^3.2.0", - "@libp2p/interface-peer-id": "^2.0.1", - "@libp2p/interface-pubsub": "^4.0.1", - "@libp2p/kad-dht": "^9.3.4", - "@libp2p/peer-id": "^2.0.1", - "@libp2p/peer-id-factory": "^2.0.1", - "@libp2p/tcp": "^7.0.1", - "@libp2p/websockets": "^6.0.1", + "@libp2p/interface": "^0.1.2", + "@libp2p/kad-dht": "^10.0.4", + "@libp2p/peer-id": "^3.0.2", + "@libp2p/peer-id-factory": "^3.0.3", + "@libp2p/tcp": "^8.0.4", + "@libp2p/websockets": "^7.0.4", "aegir": "^40.0.8", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", "go-ipfs": "^0.22.0", - "helia": "^1.0.0", + "helia": "^2.0.1", "ipfsd-ctl": "^13.0.0", "ipns": "^6.0.0", "it-all": "^3.0.2", "it-last": "^3.0.1", "it-map": "^3.0.3", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", + "libp2p": "^0.46.6", "merge-options": "^3.0.4", "multiformats": "^12.0.1", "uint8arrays": "^4.0.3", diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index 5dbff01c..6c07ea3c 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -2,7 +2,7 @@ import { ipns } from '@helia/ipns' import { dht } from '@helia/ipns/routing' -import { type KadDHT, kadDHT } from '@libp2p/kad-dht' +import { kadDHT, type DualKadDHT } from '@libp2p/kad-dht' import { createEd25519PeerId, createRSAPeerId, createSecp256k1PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { ipnsSelector } from 'ipns/selector' @@ -23,13 +23,13 @@ import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' -import type { Libp2p } from '@libp2p/interface-libp2p' +import type { Libp2p } from '@libp2p/interface' import type { Controller } from 'ipfsd-ctl' import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' keyTypes.forEach(type => { describe(`dht routing with ${type} keys`, () => { - let helia: Helia> + let helia: Helia> let kubo: Controller let name: IPNS diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index f24c3518..a9583011 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -7,7 +7,7 @@ import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' import type { Helia } from '@helia/interface' -import type { Libp2p } from '@libp2p/interface-libp2p' +import type { Libp2p } from '@libp2p/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { const blockstore = new MemoryBlockstore() diff --git a/packages/interop/test/fixtures/create-peer-ids.ts b/packages/interop/test/fixtures/create-peer-ids.ts index 36403d85..01faee2f 100644 --- a/packages/interop/test/fixtures/create-peer-ids.ts +++ b/packages/interop/test/fixtures/create-peer-ids.ts @@ -3,7 +3,7 @@ import map from 'it-map' import { sha256 } from 'multiformats/hashes/sha2' import { compare as uint8ArrayCompare } from 'uint8arrays/compare' import { xor as uint8ArrayXor } from 'uint8arrays/xor' -import type { PeerId } from '@libp2p/interface-peer-id' +import type { PeerId } from '@libp2p/interface/peer-id' /** * Sort peers by distance to the KadID of the passed buffer diff --git a/packages/interop/test/fixtures/key-types.ts b/packages/interop/test/fixtures/key-types.ts index 78b60e33..4cd79919 100644 --- a/packages/interop/test/fixtures/key-types.ts +++ b/packages/interop/test/fixtures/key-types.ts @@ -1,4 +1,4 @@ -import type { PeerIdType } from '@libp2p/interface-peer-id' +import type { PeerIdType } from '@libp2p/interface/peer-id' export const keyTypes: PeerIdType[] = [ 'Ed25519', diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index 9180d35d..f9826d12 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -23,7 +23,7 @@ import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' -import type { PubSub } from '@libp2p/interface-pubsub' +import type { PubSub } from '@libp2p/interface/pubsub' import type { Controller } from 'ipfsd-ctl' import type { Libp2p } from 'libp2p' diff --git a/packages/ipns/package.json b/packages/ipns/package.json index ade6b1aa..d7a4a54f 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -154,12 +154,9 @@ "release": "aegir release" }, "dependencies": { - "@libp2p/interface-content-routing": "^2.1.0", - "@libp2p/interface-peer-id": "^2.0.1", - "@libp2p/interface-pubsub": "^4.0.1", - "@libp2p/interfaces": "^3.3.1", + "@libp2p/interface": "^0.1.2", "@libp2p/logger": "^3.0.2", - "@libp2p/peer-id": "^2.0.1", + "@libp2p/peer-id": "^3.0.2", "@libp2p/record": "^3.0.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", @@ -171,7 +168,7 @@ "uint8arrays": "^4.0.3" }, "devDependencies": { - "@libp2p/peer-id-factory": "^2.0.1", + "@libp2p/peer-id-factory": "^3.0.3", "aegir": "^40.0.8", "datastore-core": "^9.0.3", "sinon": "^15.0.1", diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 1686b194..b486a21e 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -62,8 +62,8 @@ * ``` */ -import { isPeerId, type PeerId } from '@libp2p/interface-peer-id' -import { CodeError } from '@libp2p/interfaces/errors' +import { CodeError } from '@libp2p/interface/errors' +import { isPeerId, type PeerId } from '@libp2p/interface/peer-id' import { logger } from '@libp2p/logger' import { peerIdFromString } from '@libp2p/peer-id' import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' @@ -76,7 +76,7 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { localStore, type LocalStore } from './routing/local-store.js' import { resolveDnslink } from './utils/resolve-dns-link.js' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' import type { IPNSEntry } from 'ipns' import type { ProgressEvent, ProgressOptions } from 'progress-events' diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/dht.ts index 2b970cb3..ea13b79f 100644 --- a/packages/ipns/src/routing/dht.ts +++ b/packages/ipns/src/routing/dht.ts @@ -1,7 +1,7 @@ import { CustomProgressEvent, type ProgressEvent } from 'progress-events' import type { GetOptions, PutOptions } from './index.js' import type { IPNSRouting } from '../index.js' -import type { ContentRouting } from '@libp2p/interface-content-routing' +import type { ContentRouting } from '@libp2p/interface/content-routing' export interface DHTRoutingComponents { libp2p: { diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts index bc0659b6..d7021d50 100644 --- a/packages/ipns/src/routing/index.ts +++ b/packages/ipns/src/routing/index.ts @@ -1,7 +1,7 @@ import type { DHTProgressEvents } from './dht.js' import type { DatastoreProgressEvents } from './local-store.js' import type { PubSubProgressEvents } from './pubsub.js' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' export interface PutOptions extends AbortOptions, ProgressOptions { diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index eef3f41e..9b80a460 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -3,7 +3,7 @@ import { type Datastore, Key } from 'interface-datastore' import { CustomProgressEvent, type ProgressEvent } from 'progress-events' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import type { GetOptions, IPNSRouting, PutOptions } from '../routing' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' function dhtRoutingKey (key: Uint8Array): Key { return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts index 83a57e43..9b08141c 100644 --- a/packages/ipns/src/routing/pubsub.ts +++ b/packages/ipns/src/routing/pubsub.ts @@ -1,4 +1,4 @@ -import { CodeError } from '@libp2p/interfaces/errors' +import { CodeError } from '@libp2p/interface/errors' import { logger } from '@libp2p/logger' import { peerIdToRoutingKey } from 'ipns' import { ipnsSelector } from 'ipns/selector' @@ -9,8 +9,8 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { localStore, type LocalStore } from './local-store.js' import type { GetOptions, IPNSRouting, PutOptions } from './index.js' -import type { PeerId } from '@libp2p/interface-peer-id' -import type { Message, PublishResult, PubSub } from '@libp2p/interface-pubsub' +import type { PeerId } from '@libp2p/interface/peer-id' +import type { Message, PublishResult, PubSub } from '@libp2p/interface/pubsub' import type { Datastore } from 'interface-datastore' const log = logger('helia:ipns:routing:pubsub') diff --git a/packages/ipns/src/utils/resolve-dns-link.browser.ts b/packages/ipns/src/utils/resolve-dns-link.browser.ts index 4035ca6a..3a6d7074 100644 --- a/packages/ipns/src/utils/resolve-dns-link.browser.ts +++ b/packages/ipns/src/utils/resolve-dns-link.browser.ts @@ -2,7 +2,7 @@ import PQueue from 'p-queue' import { TLRU } from './tlru.js' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' // Avoid sending multiple queries for the same hostname by caching results const cache = new TLRU<{ Path: string, Message: string }>(1000) diff --git a/packages/ipns/src/utils/resolve-dns-link.ts b/packages/ipns/src/utils/resolve-dns-link.ts index d77cf2d7..1975cf9f 100644 --- a/packages/ipns/src/utils/resolve-dns-link.ts +++ b/packages/ipns/src/utils/resolve-dns-link.ts @@ -1,7 +1,7 @@ import dns from 'dns' import { promisify } from 'util' import * as isIPFS from 'is-ipfs' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' const MAX_RECURSIVE_DEPTH = 32 From 098a305241024ed3903b686892ded8abfca55f5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 14:21:16 +0100 Subject: [PATCH 180/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#87) Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 6 ++--- packages/interop/package.json | 25 ++++++++----------- packages/interop/test/fixtures/create-kubo.ts | 1 - packages/unixfs/package.json | 21 ++++++---------- packages/unixfs/src/commands/stat.ts | 2 +- .../unixfs/src/commands/utils/add-link.ts | 2 +- .../unixfs/src/commands/utils/constants.ts | 1 - .../unixfs/src/commands/utils/hamt-utils.ts | 2 +- .../commands/utils/is-over-shard-threshold.ts | 2 +- .../unixfs/src/commands/utils/remove-link.ts | 3 +-- packages/unixfs/src/commands/utils/resolve.ts | 2 +- packages/unixfs/src/index.ts | 4 +-- packages/unixfs/src/utils/glob-source.ts | 6 ++--- .../unixfs/test/utils/glob-source.spec.ts | 6 ++--- packages/unixfs/typedoc.json | 6 +++++ typedoc.json | 4 +++ 16 files changed, 46 insertions(+), 47 deletions(-) create mode 100644 packages/unixfs/typedoc.json create mode 100644 typedoc.json diff --git a/package.json b/package.json index 880bc567..13f12b41 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", - "docs:no-publish": "npm run docs -- --publish false", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", "release": "npm run docs:no-publish && aegir run release && npm run docs" }, "devDependencies": { - "aegir": "^39.0.8" + "aegir": "^40.0.13" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 585d781d..4ccd86b5 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -48,33 +48,30 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.2", - "@helia/interface": "^1.0.0", + "@chainsafe/libp2p-noise": "^13.0.1", + "@chainsafe/libp2p-yamux": "^5.0.0", + "@helia/interface": "^2.0.0", "@helia/unixfs": "^1.0.0", - "@libp2p/tcp": "^7.0.1", - "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.8", + "@libp2p/tcp": "^8.0.6", + "@libp2p/websockets": "^7.0.6", + "aegir": "^40.0.13", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.20.0", - "helia": "^1.0.0", + "go-ipfs": "^0.22.0", + "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^4.0.1", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", + "libp2p": "^0.46.10", "merge-options": "^3.0.4", - "multiformats": "^11.0.1", + "multiformats": "^12.1.1", "wherearewe": "^2.0.1" }, "browser": { "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", "go-ipfs": false }, - "private": true, - "typedoc": { - "entryPoint": "./src/index.ts" - } + "private": true } diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index f4612231..eea65725 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,4 +1,3 @@ - /* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */ // @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped import * as goIpfs from 'go-ipfs' diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 80f4d7f0..27212f31 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -144,7 +144,6 @@ "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", - "docs": "aegir docs", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", @@ -155,10 +154,10 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@ipld/dag-pb": "^4.0.0", - "@libp2p/interfaces": "^3.3.1", - "@libp2p/logger": "^2.0.6", + "@libp2p/interface": "^0.1.2", + "@libp2p/logger": "^3.0.2", "@multiformats/murmur3": "^2.1.2", "hamt-sharding": "^3.0.2", "interface-blockstore": "^5.0.0", @@ -169,12 +168,12 @@ "it-last": "^3.0.1", "it-pipe": "^3.0.1", "merge-options": "^3.0.4", - "multiformats": "^11.0.1", + "multiformats": "^12.1.1", "progress-events": "^1.0.0", "sparse-array": "^1.3.2" }, "devDependencies": { - "aegir": "^39.0.8", + "aegir": "^40.0.13", "blockstore-core": "^4.0.1", "delay": "^6.0.0", "iso-url": "^1.2.1", @@ -185,14 +184,10 @@ "uint8arrays": "^4.0.3", "wherearewe": "^2.0.1" }, - "typedoc": { - "entryPoint": "./src/index.ts" - }, "browser": { "./dist/src/utils/glob-source.js": false, - "node:fs": false, - "node:fs/promises": false, - "node:path": false, - "node:url": false + "fs": false, + "path": false, + "url": false } } diff --git a/packages/unixfs/src/commands/stat.ts b/packages/unixfs/src/commands/stat.ts index e7a84f86..23cded33 100644 --- a/packages/unixfs/src/commands/stat.ts +++ b/packages/unixfs/src/commands/stat.ts @@ -8,7 +8,7 @@ import { InvalidPBNodeError, NotUnixFSError, UnknownError } from '../errors.js' import { resolve } from './utils/resolve.js' import type { StatOptions, UnixFSStats } from '../index.js' import type { Blocks } from '@helia/interface/blocks' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { Mtime } from 'ipfs-unixfs' import type { CID } from 'multiformats/cid' diff --git a/packages/unixfs/src/commands/utils/add-link.ts b/packages/unixfs/src/commands/utils/add-link.ts index 64329142..2a43b55c 100644 --- a/packages/unixfs/src/commands/utils/add-link.ts +++ b/packages/unixfs/src/commands/utils/add-link.ts @@ -19,7 +19,7 @@ import { isOverShardThreshold } from './is-over-shard-threshold.js' import type { Directory } from './cid-to-directory.js' import type { Blocks } from '@helia/interface/blocks' import type { PBNode, PBLink } from '@ipld/dag-pb/interface' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { ImportResult } from 'ipfs-unixfs-importer' const log = logger('helia:unixfs:components:utils:add-link') diff --git a/packages/unixfs/src/commands/utils/constants.ts b/packages/unixfs/src/commands/utils/constants.ts index 919f10d9..3b399bc5 100644 --- a/packages/unixfs/src/commands/utils/constants.ts +++ b/packages/unixfs/src/commands/utils/constants.ts @@ -1,2 +1 @@ - export const SHARD_SPLIT_THRESHOLD_BYTES = 262144 diff --git a/packages/unixfs/src/commands/utils/hamt-utils.ts b/packages/unixfs/src/commands/utils/hamt-utils.ts index aed968f2..7dd4fb5a 100644 --- a/packages/unixfs/src/commands/utils/hamt-utils.ts +++ b/packages/unixfs/src/commands/utils/hamt-utils.ts @@ -15,7 +15,7 @@ import { import { persist } from './persist.js' import type { PersistOptions } from './persist.js' import type { Blocks } from '@helia/interface/blocks' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { Mtime } from 'ipfs-unixfs' import type { ImportResult } from 'ipfs-unixfs-importer' import type { CID, Version } from 'multiformats/cid' diff --git a/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts index 312b8ec9..aa0edcbb 100644 --- a/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts +++ b/packages/unixfs/src/commands/utils/is-over-shard-threshold.ts @@ -3,7 +3,7 @@ import { UnixFS } from 'ipfs-unixfs' import { CID_V0, CID_V1 } from './dir-sharded.js' import type { Blocks } from '@helia/interface/blocks' import type { PBNode } from '@ipld/dag-pb' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' /** * Estimate node size only based on DAGLink name and CID byte lengths diff --git a/packages/unixfs/src/commands/utils/remove-link.ts b/packages/unixfs/src/commands/utils/remove-link.ts index 23542486..52ad517b 100644 --- a/packages/unixfs/src/commands/utils/remove-link.ts +++ b/packages/unixfs/src/commands/utils/remove-link.ts @@ -1,4 +1,3 @@ - import * as dagPB from '@ipld/dag-pb' import { logger } from '@libp2p/logger' import { UnixFS } from 'ipfs-unixfs' @@ -14,7 +13,7 @@ import { persist } from './persist.js' import type { Directory } from './cid-to-directory.js' import type { Blocks } from '@helia/interface/blocks' import type { PBNode } from '@ipld/dag-pb' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { CID, Version } from 'multiformats/cid' const log = logger('helia:unixfs:utils:remove-link') diff --git a/packages/unixfs/src/commands/utils/resolve.ts b/packages/unixfs/src/commands/utils/resolve.ts index 9f8cecf2..ed3ddd64 100644 --- a/packages/unixfs/src/commands/utils/resolve.ts +++ b/packages/unixfs/src/commands/utils/resolve.ts @@ -5,7 +5,7 @@ import { addLink } from './add-link.js' import { cidToDirectory } from './cid-to-directory.js' import { cidToPBLink } from './cid-to-pblink.js' import type { Blocks } from '@helia/interface/blocks' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { CID } from 'multiformats/cid' const log = logger('helia:unixfs:components:utils:resolve') diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index d6501b4f..84f96ff6 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -53,7 +53,7 @@ import { rm } from './commands/rm.js' import { stat } from './commands/stat.js' import { touch } from './commands/touch.js' import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks' -import type { AbortOptions } from '@libp2p/interfaces' +import type { AbortOptions } from '@libp2p/interface' import type { Mtime, UnixFS as IPFSUnixFS } from 'ipfs-unixfs' import type { ExporterProgressEvents, UnixFSEntry } from 'ipfs-unixfs-exporter' import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImporterProgressEvents, ImportResult } from 'ipfs-unixfs-importer' @@ -384,7 +384,7 @@ export interface UnixFS { * @example * * ```typescript - * import fs from 'node:fs' + * import fs from 'fs' * * const stream = fs.createReadStream('./foo.txt') * const cid = await fs.addByteStream(stream) diff --git a/packages/unixfs/src/utils/glob-source.ts b/packages/unixfs/src/utils/glob-source.ts index a834cc3a..ea19500e 100644 --- a/packages/unixfs/src/utils/glob-source.ts +++ b/packages/unixfs/src/utils/glob-source.ts @@ -1,6 +1,6 @@ -import fs from 'node:fs' -import fsp from 'node:fs/promises' -import Path from 'node:path' +import fs from 'fs' +import fsp from 'fs/promises' +import Path from 'path' import glob from 'it-glob' import { InvalidParametersError } from '../errors.js' import type { MtimeLike } from 'ipfs-unixfs' diff --git a/packages/unixfs/test/utils/glob-source.spec.ts b/packages/unixfs/test/utils/glob-source.spec.ts index cdf00fab..ebb09c9c 100644 --- a/packages/unixfs/test/utils/glob-source.spec.ts +++ b/packages/unixfs/test/utils/glob-source.spec.ts @@ -1,8 +1,8 @@ /* eslint-env mocha */ -import fs from 'node:fs' -import Path from 'node:path' -import { fileURLToPath } from 'node:url' +import fs from 'fs' +import Path from 'path' +import { fileURLToPath } from 'url' import { expect } from 'aegir/chai' import all from 'it-all' import { isNode } from 'wherearewe' diff --git a/packages/unixfs/typedoc.json b/packages/unixfs/typedoc.json new file mode 100644 index 00000000..d63e8ce5 --- /dev/null +++ b/packages/unixfs/typedoc.json @@ -0,0 +1,6 @@ +{ + "entryPoints": [ + "./src/index.ts", + "./src/errors.ts" + ] +} diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 00000000..d5ea3fbc --- /dev/null +++ b/typedoc.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "name": "Helia UnixFS" +} From 4e5b1bf43b6160279d42590e71a560f0b10c94af Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 14 Sep 2023 13:28:06 +0000 Subject: [PATCH 181/253] chore(release): 1.4.2 [skip ci] ## [@helia/unixfs-v1.4.2](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.1...@helia/unixfs-v1.4.2) (2023-09-14) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#87](https://github.com/ipfs/helia-unixfs/issues/87)) ([098a305](https://github.com/ipfs/helia-unixfs/commit/098a305241024ed3903b686892ded8abfca55f5f)) --- packages/unixfs/CHANGELOG.md | 7 +++++++ packages/unixfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 8ec7d3d5..4aca9ab0 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/unixfs-v1.4.2](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.1...@helia/unixfs-v1.4.2) (2023-09-14) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#87](https://github.com/ipfs/helia-unixfs/issues/87)) ([098a305](https://github.com/ipfs/helia-unixfs/commit/098a305241024ed3903b686892ded8abfca55f5f)) + ## [@helia/unixfs-v1.4.1](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.0...@helia/unixfs-v1.4.1) (2023-07-25) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 27212f31..b827cdcb 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.4.1", + "version": "1.4.2", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 64143225dd9ab042aa875adbe0131e11b9604eb1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 15 Sep 2023 16:31:06 +0000 Subject: [PATCH 182/253] chore(release): 1.1.6 [skip ci] ## [@helia/ipns-v1.1.6](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.5...@helia/ipns-v1.1.6) (2023-09-15) ### Dependencies * **dev:** bump libp2p from 0.45.9 to 0.46.6 ([#92](https://github.com/ipfs/helia-ipns/issues/92)) ([efe02e5](https://github.com/ipfs/helia-ipns/commit/efe02e5b38992189edb40cd34d79e76dca4c34a3)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index f8ca7e46..7407c0cc 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.6](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.5...@helia/ipns-v1.1.6) (2023-09-15) + + +### Dependencies + +* **dev:** bump libp2p from 0.45.9 to 0.46.6 ([#92](https://github.com/ipfs/helia-ipns/issues/92)) ([efe02e5](https://github.com/ipfs/helia-ipns/commit/efe02e5b38992189edb40cd34d79e76dca4c34a3)) + ## [@helia/ipns-v1.1.5](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.4...@helia/ipns-v1.1.5) (2023-09-11) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index d7a4a54f..865516c1 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.5", + "version": "1.1.6", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 231ebbd4cda2196d7914a81aa1b0d79473c3a325 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 17:38:02 +0100 Subject: [PATCH 183/253] deps(dev): bump sinon from 15.2.0 to 16.0.0 (#105) Bumps [sinon](https://github.com/sinonjs/sinon) from 15.2.0 to 16.0.0. - [Release notes](https://github.com/sinonjs/sinon/releases) - [Changelog](https://github.com/sinonjs/sinon/blob/main/docs/changelog.md) - [Commits](https://github.com/sinonjs/sinon/compare/v15.2.0...v16.0.0) --- updated-dependencies: - dependency-name: sinon dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/ipns/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 865516c1..ad741b83 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -171,7 +171,7 @@ "@libp2p/peer-id-factory": "^3.0.3", "aegir": "^40.0.8", "datastore-core": "^9.0.3", - "sinon": "^15.0.1", + "sinon": "^16.0.0", "sinon-ts": "^1.0.0" }, "browser": { From aaf402d27c05f5e6b65d1beaf3e99d36835d6892 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 15 Sep 2023 17:09:33 +0000 Subject: [PATCH 184/253] chore(release): 1.1.7 [skip ci] ## [@helia/ipns-v1.1.7](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.6...@helia/ipns-v1.1.7) (2023-09-15) ### Dependencies * **dev:** bump sinon from 15.2.0 to 16.0.0 ([#105](https://github.com/ipfs/helia-ipns/issues/105)) ([231ebbd](https://github.com/ipfs/helia-ipns/commit/231ebbd4cda2196d7914a81aa1b0d79473c3a325)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 7407c0cc..00dcfe27 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v1.1.7](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.6...@helia/ipns-v1.1.7) (2023-09-15) + + +### Dependencies + +* **dev:** bump sinon from 15.2.0 to 16.0.0 ([#105](https://github.com/ipfs/helia-ipns/issues/105)) ([231ebbd](https://github.com/ipfs/helia-ipns/commit/231ebbd4cda2196d7914a81aa1b0d79473c3a325)) + ## [@helia/ipns-v1.1.6](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.5...@helia/ipns-v1.1.6) (2023-09-15) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index ad741b83..7006bcac 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.6", + "version": "1.1.7", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 83a1d147e8ba758efd7d2574ea486218bd1f3df2 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 15 Sep 2023 18:21:32 +0100 Subject: [PATCH 185/253] deps!: update ipns to v7.x.x (#106) Updates ipns to v7.x.x. See the [release notes](https://github.com/ipfs/js-ipns/releases/tag/v7.0.0) for full details. BREAKING CHANGE: the `IPNSRecord` type returned from the `publish` method has changed --- packages/interop/package.json | 2 +- .../interop/test/fixtures/create-helia.ts | 6 +- packages/interop/test/pubsub.spec.ts | 2 +- packages/ipns/package.json | 2 +- packages/ipns/src/index.ts | 62 ++++++------------- packages/ipns/test/resolve.spec.ts | 18 +++--- 6 files changed, 33 insertions(+), 59 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 5f762e03..72cea9c0 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -65,7 +65,7 @@ "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfsd-ctl": "^13.0.0", - "ipns": "^6.0.0", + "ipns": "^7.0.1", "it-all": "^3.0.2", "it-last": "^3.0.1", "it-map": "^3.0.3", diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index 9ffe4c7a..a3826ef6 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -4,10 +4,12 @@ import { tcp } from '@libp2p/tcp' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' -import { createLibp2p, type Libp2p, type Libp2pOptions } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' +import type { IdentifyService } from 'libp2p/identify' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index f9826d12..22fff4a3 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -23,9 +23,9 @@ import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' +import type { Libp2p } from '@libp2p/interface' import type { PubSub } from '@libp2p/interface/pubsub' import type { Controller } from 'ipfsd-ctl' -import type { Libp2p } from 'libp2p' const LIBP2P_KEY_CODEC = 0x72 diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 7006bcac..0a335e22 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -160,7 +160,7 @@ "@libp2p/record": "^3.0.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", - "ipns": "^6.0.0", + "ipns": "^7.0.1", "is-ipfs": "^8.0.1", "multiformats": "^12.0.1", "p-queue": "^7.3.0", diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index b486a21e..ecbf0eaa 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -6,30 +6,11 @@ * @example * * ```typescript - * import { gossipsub } from '@chainsafe/libp2p-gossipsub' - * import { kadDHT } from '@libp2p/kad-dht' - * import { createLibp2p } from 'libp2p' * import { createHelia } from 'helia' - * import { ipns, ipnsValidator, ipnsSelector } from '@helia/ipns' * import { dht, pubsub } from '@helia/ipns/routing' * import { unixfs } from '@helia/unixfs' * - * const libp2p = await createLibp2p({ - * dht: kadDHT({ - * validators: { - * ipns: ipnsValidator - * }, - * selectors: { - * ipns: ipnsSelector - * } - * }), - * pubsub: gossipsub() - * }) - * - * const helia = await createHelia({ - * libp2p, - * //.. other options - * }) + * const helia = await createHelia() * const name = ipns(helia, [ * dht(helia), * pubsub(helia) @@ -63,7 +44,6 @@ */ import { CodeError } from '@libp2p/interface/errors' -import { isPeerId, type PeerId } from '@libp2p/interface/peer-id' import { logger } from '@libp2p/logger' import { peerIdFromString } from '@libp2p/peer-id' import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' @@ -71,14 +51,13 @@ import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' import { CID } from 'multiformats/cid' import { CustomProgressEvent } from 'progress-events' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { localStore, type LocalStore } from './routing/local-store.js' import { resolveDnslink } from './utils/resolve-dns-link.js' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' import type { AbortOptions } from '@libp2p/interface' +import type { PeerId } from '@libp2p/interface/peer-id' import type { Datastore } from 'interface-datastore' -import type { IPNSEntry } from 'ipns' +import type { IPNSRecord } from 'ipns' import type { ProgressEvent, ProgressOptions } from 'progress-events' const log = logger('helia:ipns') @@ -91,18 +70,18 @@ const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR export type PublishProgressEvents = ProgressEvent<'ipns:publish:start'> | - ProgressEvent<'ipns:publish:success', IPNSEntry> | + ProgressEvent<'ipns:publish:success', IPNSRecord> | ProgressEvent<'ipns:publish:error', Error> export type ResolveProgressEvents = ProgressEvent<'ipns:resolve:start', unknown> | - ProgressEvent<'ipns:resolve:success', IPNSEntry> | + ProgressEvent<'ipns:resolve:success', IPNSRecord> | ProgressEvent<'ipns:resolve:error', Error> export type RepublishProgressEvents = ProgressEvent<'ipns:republish:start', unknown> | - ProgressEvent<'ipns:republish:success', IPNSEntry> | - ProgressEvent<'ipns:republish:error', { record: IPNSEntry, err: Error }> + ProgressEvent<'ipns:republish:success', IPNSRecord> | + ProgressEvent<'ipns:republish:error', { record: IPNSRecord, err: Error }> export interface PublishOptions extends AbortOptions, ProgressOptions { /** @@ -114,6 +93,12 @@ export interface PublishOptions extends AbortOptions, ProgressOptions { @@ -143,7 +128,7 @@ export interface IPNS { * * If the valid is a PeerId, a recursive IPNS record will be created. */ - publish: (key: PeerId, value: CID | PeerId, options?: PublishOptions) => Promise + publish: (key: PeerId, value: CID | PeerId, options?: PublishOptions) => Promise /** * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS record @@ -178,7 +163,7 @@ class DefaultIPNS implements IPNS { this.localStore = localStore(components.datastore) } - async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { + async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { try { let sequenceNumber = 1n const routingKey = peerIdToRoutingKey(key) @@ -190,18 +175,8 @@ class DefaultIPNS implements IPNS { sequenceNumber = existingRecord.sequence + 1n } - let str - - if (isPeerId(value)) { - str = `/ipns/${value.toString()}` - } else { - str = `/ipfs/${value.toString()}` - } - - const bytes = uint8ArrayFromString(str) - // create record - const record = await create(key, bytes, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS) + const record = await create(key, value, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS, options) const marshaledRecord = marshal(record) await this.localStore.put(routingKey, marshaledRecord, options) @@ -221,9 +196,8 @@ class DefaultIPNS implements IPNS { async resolve (key: PeerId, options: ResolveOptions = {}): Promise { const routingKey = peerIdToRoutingKey(key) const record = await this.#findIpnsRecord(routingKey, options) - const str = uint8ArrayToString(record.value) - return this.#resolve(str, options) + return this.#resolve(record.value, options) } async resolveDns (domain: string, options: ResolveDNSOptions = {}): Promise { @@ -285,7 +259,7 @@ class DefaultIPNS implements IPNS { throw new Error('Invalid value') } - async #findIpnsRecord (routingKey: Uint8Array, options: ResolveOptions = {}): Promise { + async #findIpnsRecord (routingKey: Uint8Array, options: ResolveOptions = {}): Promise { let routers = [ this.localStore, ...this.routers diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index f3b23e13..7815ef85 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -9,7 +9,6 @@ import { create, marshal, peerIdToRoutingKey } from 'ipns' import { CID } from 'multiformats/cid' import Sinon from 'sinon' import { type StubbedInstance, stubInterface } from 'sinon-ts' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { ipns } from '../src/index.js' import type { IPNS, IPNSRouting } from '../src/index.js' @@ -39,7 +38,7 @@ describe('resolve', () => { throw new Error('Did not resolve entry') } - expect(resolvedValue.toString()).to.equal(cid.toString()) + expect(resolvedValue.toString()).to.equal(cid.toV1().toString()) }) it('should resolve a record offline', async () => { @@ -58,7 +57,7 @@ describe('resolve', () => { throw new Error('Did not resolve entry') } - expect(resolvedValue.toString()).to.equal(cid.toString()) + expect(resolvedValue.toString()).to.equal(cid.toV1().toString()) }) it('should resolve a recursive record', async () => { @@ -73,7 +72,7 @@ describe('resolve', () => { throw new Error('Did not resolve entry') } - expect(resolvedValue.toString()).to.equal(cid.toString()) + expect(resolvedValue.toString()).to.equal(cid.toV1().toString()) }) it('should resolve /ipns/tableflip.io', async function () { @@ -112,14 +111,13 @@ describe('resolve', () => { expect(datastore.has(dhtKey)).to.be.false('already had record') - const bytes = uint8ArrayFromString(`/ipfs/${cid.toString()}`) - const record = await create(peerId, bytes, 0n, 60000) + const record = await create(peerId, cid, 0n, 60000) const marshalledRecord = marshal(record) routing.get.withArgs(routingKey).resolves(marshalledRecord) const result = await name.resolve(peerId) - expect(result.toString()).to.equal(cid.toString(), 'incorrect record resolved') + expect(result.toString()).to.equal(cid.toV1().toString(), 'incorrect record resolved') expect(datastore.has(dhtKey)).to.be.true('did not cache record locally') }) @@ -129,8 +127,8 @@ describe('resolve', () => { const routingKey = peerIdToRoutingKey(peerId) const dhtKey = new Key('/dht/record/' + uint8ArrayToString(routingKey, 'base32'), false) - const marshalledRecordA = marshal(await create(peerId, uint8ArrayFromString(`/ipfs/${cid.toString()}`), 0n, 60000)) - const marshalledRecordB = marshal(await create(peerId, uint8ArrayFromString(`/ipfs/${cid.toString()}`), 10n, 60000)) + const marshalledRecordA = marshal(await create(peerId, cid, 0n, 60000)) + const marshalledRecordB = marshal(await create(peerId, cid, 10n, 60000)) // records should not match expect(marshalledRecordA).to.not.equalBytes(marshalledRecordB) @@ -140,7 +138,7 @@ describe('resolve', () => { routing.get.withArgs(routingKey).resolves(marshalledRecordB) const result = await name.resolve(peerId) - expect(result.toString()).to.equal(cid.toString(), 'incorrect record resolved') + expect(result.toString()).to.equal(cid.toV1().toString(), 'incorrect record resolved') const cached = await datastore.get(dhtKey) const record = Libp2pRecord.deserialize(cached) From f29a523ed0cada1a5760cb780abe90e9bd3cea84 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 22 Sep 2023 12:42:24 +0000 Subject: [PATCH 186/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/ipns-v2.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.7...@helia/ipns-v2.0.0) (2023-09-22) ### ⚠ BREAKING CHANGES * the `IPNSRecord` type returned from the `publish` method has changed ### Dependencies * update ipns to v7.x.x ([#106](https://github.com/ipfs/helia-ipns/issues/106)) ([83a1d14](https://github.com/ipfs/helia-ipns/commit/83a1d147e8ba758efd7d2574ea486218bd1f3df2)) --- packages/ipns/CHANGELOG.md | 11 +++++++++++ packages/ipns/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 00dcfe27..49862170 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/ipns-v2.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.7...@helia/ipns-v2.0.0) (2023-09-22) + + +### ⚠ BREAKING CHANGES + +* the `IPNSRecord` type returned from the `publish` method has changed + +### Dependencies + +* update ipns to v7.x.x ([#106](https://github.com/ipfs/helia-ipns/issues/106)) ([83a1d14](https://github.com/ipfs/helia-ipns/commit/83a1d147e8ba758efd7d2574ea486218bd1f3df2)) + ## [@helia/ipns-v1.1.7](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.6...@helia/ipns-v1.1.7) (2023-09-15) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 0a335e22..cc347576 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "1.1.7", + "version": "2.0.0", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From e20677784bce5a863403b06868511be894f9929f Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Fri, 22 Sep 2023 12:43:01 +0000 Subject: [PATCH 187/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 72cea9c0..936a55d4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "@chainsafe/libp2p-noise": "^13.0.0", "@chainsafe/libp2p-yamux": "^5.0.0", "@helia/interface": "^2.0.0", - "@helia/ipns": "^1.0.0", + "@helia/ipns": "^2.0.0", "@libp2p/interface": "^0.1.2", "@libp2p/kad-dht": "^10.0.4", "@libp2p/peer-id": "^3.0.2", From 7c9bc2e9f99ccbaec1d8c25c900585deb5f6a327 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:04:49 +0100 Subject: [PATCH 188/253] deps: bump @helia/interface from 1.2.2 to 2.0.0 (#39) Bumps [@helia/interface](https://github.com/ipfs/helia) from 1.2.2 to 2.0.0. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/@helia/interface-v1.2.2...helia-v2.0.0) --- updated-dependencies: - dependency-name: "@helia/interface" dependency-type: direct:production update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- .github/workflows/js-test-and-release.yml | 1 + package.json | 13 ++++------- packages/interop/package.json | 23 ++++++++----------- packages/interop/test/fixtures/create-kubo.ts | 2 -- packages/strings/package.json | 11 ++++----- packages/strings/src/index.ts | 4 ++-- packages/strings/typedoc.json | 5 ++++ 7 files changed, 27 insertions(+), 32 deletions(-) create mode 100644 packages/strings/typedoc.json diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 51686414..a15c3791 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -17,6 +17,7 @@ jobs: - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present lint - run: npm run --if-present dep-check + - run: npm run --if-present doc-check test-node: needs: check diff --git a/package.json b/package.json index 9a5f0206..940f26aa 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ "keywords": [ "ipfs" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "private": true, "scripts": { "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", @@ -33,13 +29,14 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", - "docs:no-publish": "npm run docs -- --publish false", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^39.0.3" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 99bdd12f..549fd8e6 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -35,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "package": true, "sourceType": "module" } }, @@ -52,21 +49,21 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^12.0.1", - "@chainsafe/libp2p-yamux": "^4.0.1", - "@helia/interface": "^1.0.0", + "@chainsafe/libp2p-noise": "^13.0.1", + "@chainsafe/libp2p-yamux": "^5.0.0", + "@helia/interface": "^2.0.0", "@helia/strings": "^1.0.0", - "@libp2p/tcp": "^7.0.1", - "@libp2p/websockets": "^6.0.1", - "aegir": "^39.0.3", + "@libp2p/tcp": "^8.0.8", + "@libp2p/websockets": "^7.0.8", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.20.0", - "helia": "^1.0.0", + "go-ipfs": "^0.22.0", + "helia": "^2.0.3", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.45.3", + "libp2p": "^0.46.12", "merge-options": "^3.0.4", "uint8arrays": "^4.0.3", "wherearewe": "^2.0.1" diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 52ed76e1..c032e86d 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,5 +1,3 @@ - -// @ts-expect-error no types import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' import * as kuboRpcClient from 'kubo-rpc-client' diff --git a/packages/strings/package.json b/packages/strings/package.json index 64494b61..781298b6 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -14,10 +14,6 @@ "keywords": [ "IPFS" ], - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -35,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "package": true, "sourceType": "module" } }, @@ -139,15 +136,15 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^1.0.0", + "@helia/interface": "^2.0.0", "@libp2p/interfaces": "^3.3.1", "interface-blockstore": "^5.0.0", - "multiformats": "^11.0.1", + "multiformats": "^12.1.2", "progress-events": "^1.0.0", "uint8arrays": "^4.0.3" }, "devDependencies": { - "aegir": "^39.0.3", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1" }, "typedoc": { diff --git a/packages/strings/src/index.ts b/packages/strings/src/index.ts index df7ae5ef..f99e8987 100644 --- a/packages/strings/src/index.ts +++ b/packages/strings/src/index.ts @@ -66,7 +66,7 @@ export interface Strings { * // CID(bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e) * ``` */ - add: (str: string, options?: Partial) => Promise + add(str: string, options?: Partial): Promise /** * Get a string from your Helia node, either previously added to it or to @@ -86,7 +86,7 @@ export interface Strings { * // hello world * ``` */ - get: (cid: CID, options?: Partial) => Promise + get(cid: CID, options?: Partial): Promise } class DefaultStrings implements Strings { diff --git a/packages/strings/typedoc.json b/packages/strings/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/strings/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} From 2ad9d4032c0b5599259377f51a71a6b67142eaad Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 6 Oct 2023 16:27:22 -0700 Subject: [PATCH 189/253] chore: fix release script (#52) --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 940f26aa..850b254a 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "lint": "aegir run lint", "dep-check": "aegir run dep-check", "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, From 6c68f4ee8ad92ec718d4734f5ec6a9eec245e6d2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 6 Oct 2023 23:35:52 +0000 Subject: [PATCH 190/253] chore(release): 1.0.1 [skip ci] ## [@helia/strings-v1.0.1](https://github.com/ipfs/helia-strings/compare/@helia/strings-v1.0.0...@helia/strings-v1.0.1) (2023-10-06) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#39](https://github.com/ipfs/helia-strings/issues/39)) ([7c9bc2e](https://github.com/ipfs/helia-strings/commit/7c9bc2e9f99ccbaec1d8c25c900585deb5f6a327)) --- packages/strings/CHANGELOG.md | 7 +++++++ packages/strings/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/strings/CHANGELOG.md b/packages/strings/CHANGELOG.md index ab5d5632..8e971df1 100644 --- a/packages/strings/CHANGELOG.md +++ b/packages/strings/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/strings-v1.0.1](https://github.com/ipfs/helia-strings/compare/@helia/strings-v1.0.0...@helia/strings-v1.0.1) (2023-10-06) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#39](https://github.com/ipfs/helia-strings/issues/39)) ([7c9bc2e](https://github.com/ipfs/helia-strings/commit/7c9bc2e9f99ccbaec1d8c25c900585deb5f6a327)) + ## @helia/strings-v1.0.0 (2023-05-03) diff --git a/packages/strings/package.json b/packages/strings/package.json index 781298b6..96ceac99 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -1,6 +1,6 @@ { "name": "@helia/strings", - "version": "1.0.0", + "version": "1.0.1", "description": "Add/get IPLD blocks containing strings with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/strings#readme", From 77e29bcdda33387b8bf15124bc316ef03b434433 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:37:37 +0100 Subject: [PATCH 191/253] deps(dev): bump aegir from 40.0.13 to 41.0.0 (#36) Bumps [aegir](https://github.com/ipfs/aegir) from 40.0.13 to 41.0.0. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v40.0.13...v41.0.0) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 8 ++++---- packages/dag-cbor/package.json | 2 +- packages/dag-cbor/src/index.ts | 4 ++-- packages/interop/package.json | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 610641af..89a489c8 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,13 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.11" + "aegir": "^41.0.0" }, "type": "module", "workspaces": [ diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 84c94071..2f223b4d 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -143,7 +143,7 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^40.0.11", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1" } } diff --git a/packages/dag-cbor/src/index.ts b/packages/dag-cbor/src/index.ts index 5d9e2a6c..7457c64d 100644 --- a/packages/dag-cbor/src/index.ts +++ b/packages/dag-cbor/src/index.ts @@ -65,7 +65,7 @@ export interface DAGCBOR { * // CID(bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae) * ``` */ - add: (str: unknown, options?: Partial) => Promise + add(str: unknown, options?: Partial): Promise /** * Get an object from your Helia node, either previously added to it or to @@ -85,7 +85,7 @@ export interface DAGCBOR { * // { hello: 'world' } * ``` */ - get: (cid: CID, options?: Partial) => Promise + get(cid: CID, options?: Partial): Promise } class DefaultDAGCBOR implements DAGCBOR { diff --git a/packages/interop/package.json b/packages/interop/package.json index e2b49566..4115d050 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,7 +52,7 @@ "@helia/interface": "^2.0.0", "@ipld/dag-cbor": "^9.0.0", "@libp2p/websockets": "^7.0.5", - "aegir": "^40.0.11", + "aegir": "^41.0.0", "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", From 9f57d11e461a3b1fddbc2a92e225d31eee56613c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:37:54 +0100 Subject: [PATCH 192/253] deps(dev): bump aegir from 40.0.13 to 41.0.0 (#36) Bumps [aegir](https://github.com/ipfs/aegir) from 40.0.13 to 41.0.0. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v40.0.13...v41.0.0) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 9 +++++---- packages/dag-json/package.json | 3 ++- packages/dag-json/src/index.ts | 4 ++-- packages/interop/package.json | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 105ff48e..e873b647 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,14 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.11" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index eef0e31a..0477454d 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -142,7 +143,7 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^40.0.11", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1" } } diff --git a/packages/dag-json/src/index.ts b/packages/dag-json/src/index.ts index d123449b..90be4cda 100644 --- a/packages/dag-json/src/index.ts +++ b/packages/dag-json/src/index.ts @@ -65,7 +65,7 @@ export interface DAGJSON { * // CID(baguqeerasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea) * ``` */ - add: (str: unknown, options?: Partial) => Promise + add(str: unknown, options?: Partial): Promise /** * Get an object from your Helia node, either previously added to it or to @@ -85,7 +85,7 @@ export interface DAGJSON { * // { hello: 'world' } * ``` */ - get: (cid: CID, options?: Partial) => Promise + get(cid: CID, options?: Partial): Promise } class DefaultDAGJSON implements DAGJSON { diff --git a/packages/interop/package.json b/packages/interop/package.json index 695f1e96..79e5604f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -51,7 +52,7 @@ "@helia/dag-json": "^1.0.0", "@helia/interface": "^2.0.0", "@libp2p/websockets": "^7.0.5", - "aegir": "^40.0.11", + "aegir": "^41.0.0", "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", From e8fc99f4e372eaf72c2598f5a7a9942143c6d788 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:38:17 +0100 Subject: [PATCH 193/253] deps(dev): bump aegir from 40.0.13 to 41.0.0 (#41) Bumps [aegir](https://github.com/ipfs/aegir) from 40.0.13 to 41.0.0. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v40.0.13...v41.0.0) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 9 +++++---- packages/car/package.json | 5 +++-- packages/car/src/index.ts | 6 +++--- packages/car/src/utils/dag-walkers.ts | 2 +- packages/car/test/fixtures/memory-car.ts | 2 +- packages/interop/package.json | 3 ++- packages/interop/test/fixtures/memory-car.ts | 2 +- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index a5d20a28..27c1fd04 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,14 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.11" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/car/package.json b/packages/car/package.json index aae1d6d2..ec745b42 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -51,6 +51,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -158,7 +159,7 @@ "@ipld/car": "^5.1.1", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", - "cborg": "^2.0.5", + "cborg": "^4.0.3", "interface-blockstore": "^5.0.0", "it-map": "^3.0.3", "multiformats": "^12.0.1", @@ -168,7 +169,7 @@ }, "devDependencies": { "@helia/unixfs": "^1.2.2", - "aegir": "^40.0.11", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "ipfs-unixfs-importer": "^15.1.0", "it-drain": "^3.0.1", diff --git a/packages/car/src/index.ts b/packages/car/src/index.ts index 1759daa2..7813a2ac 100644 --- a/packages/car/src/index.ts +++ b/packages/car/src/index.ts @@ -87,7 +87,7 @@ export interface CarInit { */ export interface DAGWalker { codec: number - walk: (block: Uint8Array) => AsyncGenerator + walk(block: Uint8Array): AsyncGenerator } /** @@ -115,7 +115,7 @@ export interface Car { * await c.import(reader) * ``` */ - import: (reader: Pick, options?: AbortOptions & ProgressOptions) => Promise + import(reader: Pick, options?: AbortOptions & ProgressOptions): Promise /** * Store all blocks that make up one or more DAGs in a car file. @@ -143,7 +143,7 @@ export interface Car { * await eventPromise * ``` */ - export: (root: CID | CID[], writer: Pick, options?: AbortOptions & ProgressOptions) => Promise + export(root: CID | CID[], writer: Pick, options?: AbortOptions & ProgressOptions): Promise } const DEFAULT_DAG_WALKERS = [ diff --git a/packages/car/src/utils/dag-walkers.ts b/packages/car/src/utils/dag-walkers.ts index 62de926f..80cc25b7 100644 --- a/packages/car/src/utils/dag-walkers.ts +++ b/packages/car/src/utils/dag-walkers.ts @@ -13,7 +13,7 @@ import * as raw from 'multiformats/codecs/raw' */ export interface DAGWalker { codec: number - walk: (block: Uint8Array) => AsyncGenerator + walk(block: Uint8Array): AsyncGenerator } /** diff --git a/packages/car/test/fixtures/memory-car.ts b/packages/car/test/fixtures/memory-car.ts index 54debcac..ad210cb3 100644 --- a/packages/car/test/fixtures/memory-car.ts +++ b/packages/car/test/fixtures/memory-car.ts @@ -4,7 +4,7 @@ import defer from 'p-defer' import type { CID } from 'multiformats/cid' export interface MemoryCar extends Pick { - bytes: () => Promise + bytes(): Promise } export function memoryCarWriter (root: CID | CID[]): MemoryCar { diff --git a/packages/interop/package.json b/packages/interop/package.json index 5df0045a..485965a8 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -53,7 +54,7 @@ "@helia/unixfs": "^1.2.2", "@ipld/car": "^5.1.1", "@libp2p/websockets": "^7.0.5", - "aegir": "^40.0.11", + "aegir": "^41.0.0", "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfs-unixfs-importer": "^15.1.0", diff --git a/packages/interop/test/fixtures/memory-car.ts b/packages/interop/test/fixtures/memory-car.ts index 54debcac..ad210cb3 100644 --- a/packages/interop/test/fixtures/memory-car.ts +++ b/packages/interop/test/fixtures/memory-car.ts @@ -4,7 +4,7 @@ import defer from 'p-defer' import type { CID } from 'multiformats/cid' export interface MemoryCar extends Pick { - bytes: () => Promise + bytes(): Promise } export function memoryCarWriter (root: CID | CID[]): MemoryCar { From 33a5c9f43fe79c4a3ee0c6c9ab51f727a24a1160 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 7 Oct 2023 10:45:33 +0100 Subject: [PATCH 194/253] chore: add missing dep --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 89a489c8..05b88923 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ From b19d2e9f408fba02de557dd7d67624d95b8b0e82 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 7 Oct 2023 10:59:28 +0100 Subject: [PATCH 195/253] chore: add missing script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 27c1fd04..c54a5ef0 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "lint": "aegir run lint", "dep-check": "aegir run dep-check", "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, From 343e7ce65e38e80039e66e1d9d73b14031c9d0d9 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 7 Oct 2023 11:01:28 +0100 Subject: [PATCH 196/253] chore: add missing script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index e873b647..4a1e19bd 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "lint": "aegir run lint", "dep-check": "aegir run dep-check", "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, From 2f39695a7d0f0c1884163df6119a13eb16a13cac Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 7 Oct 2023 11:02:02 +0100 Subject: [PATCH 197/253] chore: add missing script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 05b88923..2a1d8c4a 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "lint": "aegir run lint", "dep-check": "aegir run dep-check", "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, From 6911470cb43720798fca571669a166eb3689dad2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 11:04:56 +0100 Subject: [PATCH 198/253] deps(dev): bump helia from 2.0.1 to 2.0.3 (#10) Bumps [helia](https://github.com/ipfs/helia) from 2.0.1 to 2.0.3. - [Release notes](https://github.com/ipfs/helia/releases) - [Changelog](https://github.com/ipfs/helia/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/helia/compare/helia-v2.0.1...helia-v2.0.3) --- updated-dependencies: - dependency-name: helia dependency-type: direct:development update-type: version-update:semver-patch ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 10 ++++++---- packages/interop/package.json | 5 +++-- packages/mfs/package.json | 3 ++- packages/mfs/src/index.ts | 20 ++++++++++---------- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index c050bf2a..ee3e238f 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,15 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.9" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index da51e9d4..e7047557 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -50,9 +51,9 @@ "devDependencies": { "@helia/interface": "^2.0.0", "@helia/mfs": "^1.0.0", - "aegir": "^40.0.9", + "aegir": "^41.0.3", "go-ipfs": "^0.22.0", - "helia": "2.0.1", + "helia": "^2.0.3", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", diff --git a/packages/mfs/package.json b/packages/mfs/package.json index 8dd7bc25..e66fc14b 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -148,7 +149,7 @@ "multiformats": "^12.0.1" }, "devDependencies": { - "aegir": "^40.0.9", + "aegir": "^41.0.3", "blockstore-core": "^4.0.1", "datastore-core": "^9.2.0", "delay": "^6.0.0", diff --git a/packages/mfs/src/index.ts b/packages/mfs/src/index.ts index bbc9bde1..de9e930c 100644 --- a/packages/mfs/src/index.ts +++ b/packages/mfs/src/index.ts @@ -96,7 +96,7 @@ export interface MFS { * await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), '/foo.txt') * ``` */ - writeBytes: (bytes: Uint8Array, path: string, options?: Partial) => Promise + writeBytes(bytes: Uint8Array, path: string, options?: Partial): Promise /** * Add a stream of `Uint8Array` to your MFS as a file. @@ -110,7 +110,7 @@ export interface MFS { * await fs.writeByteStream(stream, '/foo.txt') * ``` */ - writeByteStream: (bytes: ByteStream, path: string, options?: Partial) => Promise + writeByteStream(bytes: ByteStream, path: string, options?: Partial): Promise /** * Retrieve the contents of a file from your MFS. @@ -123,7 +123,7 @@ export interface MFS { * } * ``` */ - cat: (path: string, options?: Partial) => AsyncIterable + cat(path: string, options?: Partial): AsyncIterable /** * Change the permissions on a file or directory in your MFS @@ -141,7 +141,7 @@ export interface MFS { * console.info(afterStats) * ``` */ - chmod: (path: string, mode: number, options?: Partial) => Promise + chmod(path: string, mode: number, options?: Partial): Promise /** * Add a file or directory to a target directory in your MFS. @@ -165,7 +165,7 @@ export interface MFS { * await fs.cp('/foo.txt', '/bar.txt') * ``` */ - cp: (source: CID | string, destination: string, options?: Partial) => Promise + cp(source: CID | string, destination: string, options?: Partial): Promise /** * List directory contents from your MFS. @@ -178,7 +178,7 @@ export interface MFS { * } * ``` */ - ls: (path?: string, options?: Partial) => AsyncIterable + ls(path?: string, options?: Partial): AsyncIterable /** * Make a new directory in your MFS. @@ -189,7 +189,7 @@ export interface MFS { * await fs.mkdir('/new-dir') * ``` */ - mkdir: (path: string, options?: Partial) => Promise + mkdir(path: string, options?: Partial): Promise /** * Remove a file or directory from your MFS. @@ -201,7 +201,7 @@ export interface MFS { * await fs.rm('/new-dir') * ``` */ - rm: (path: string, options?: Partial) => Promise + rm(path: string, options?: Partial): Promise /** * Return statistics about a UnixFS DAG in your MFS. @@ -215,7 +215,7 @@ export interface MFS { * console.info(stats) * ``` */ - stat: (path: string, options?: Partial) => Promise + stat(path: string, options?: Partial): Promise /** * Update the mtime of a UnixFS DAG in your MFS. @@ -233,7 +233,7 @@ export interface MFS { * console.info(afterStats) * ``` */ - touch: (path: string, options?: Partial) => Promise + touch(path: string, options?: Partial): Promise } interface PathEntry { From 5402d30de1437052e9e9b955d9be3c2898515447 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 11:05:23 +0100 Subject: [PATCH 199/253] deps(dev): bump aegir from 40.0.13 to 41.0.0 (#107) Bumps [aegir](https://github.com/ipfs/aegir) from 40.0.13 to 41.0.0. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v40.0.13...v41.0.0) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 10 ++++++---- packages/interop/package.json | 3 ++- packages/interop/typedoc.json | 5 ----- packages/ipns/package.json | 3 ++- packages/ipns/src/index.ts | 8 ++++---- packages/ipns/src/routing/index.ts | 4 ++-- packages/ipns/src/routing/local-store.ts | 2 +- 7 files changed, 17 insertions(+), 18 deletions(-) delete mode 100644 packages/interop/typedoc.json diff --git a/package.json b/package.json index 20cf7f17..e9b39b85 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,15 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs -- --exclude packages/interop" + "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.8" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 936a55d4..8b13403c 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -59,7 +60,7 @@ "@libp2p/peer-id-factory": "^3.0.3", "@libp2p/tcp": "^8.0.4", "@libp2p/websockets": "^7.0.4", - "aegir": "^40.0.8", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", "go-ipfs": "^0.22.0", diff --git a/packages/interop/typedoc.json b/packages/interop/typedoc.json deleted file mode 100644 index f599dc72..00000000 --- a/packages/interop/typedoc.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "entryPoints": [ - "./src/index.ts" - ] -} diff --git a/packages/ipns/package.json b/packages/ipns/package.json index cc347576..42fb6a77 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -51,6 +51,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -169,7 +170,7 @@ }, "devDependencies": { "@libp2p/peer-id-factory": "^3.0.3", - "aegir": "^40.0.8", + "aegir": "^41.0.0", "datastore-core": "^9.0.3", "sinon": "^16.0.0", "sinon-ts": "^1.0.0" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index ecbf0eaa..dedc2277 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -128,23 +128,23 @@ export interface IPNS { * * If the valid is a PeerId, a recursive IPNS record will be created. */ - publish: (key: PeerId, value: CID | PeerId, options?: PublishOptions) => Promise + publish(key: PeerId, value: CID | PeerId, options?: PublishOptions): Promise /** * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS record * corresponding to that public key until a value is found */ - resolve: (key: PeerId, options?: ResolveOptions) => Promise + resolve(key: PeerId, options?: ResolveOptions): Promise /** * Resolve a CID from a dns-link style IPNS record */ - resolveDns: (domain: string, options?: ResolveDNSOptions) => Promise + resolveDns(domain: string, options?: ResolveDNSOptions): Promise /** * Periodically republish all IPNS records found in the datastore */ - republish: (options?: RepublishOptions) => void + republish(options?: RepublishOptions): void } export type { IPNSRouting } from './routing/index.js' diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts index d7021d50..232c2212 100644 --- a/packages/ipns/src/routing/index.ts +++ b/packages/ipns/src/routing/index.ts @@ -13,8 +13,8 @@ export interface GetOptions extends AbortOptions, ProgressOptions { } export interface IPNSRouting { - put: (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions) => Promise - get: (routingKey: Uint8Array, options?: GetOptions) => Promise + put(routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions): Promise + get(routingKey: Uint8Array, options?: GetOptions): Promise } export type IPNSRoutingEvents = diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index 9b80a460..06302129 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -15,7 +15,7 @@ export type DatastoreProgressEvents = ProgressEvent<'ipns:routing:datastore:error', Error> export interface LocalStore extends IPNSRouting { - has: (routingKey: Uint8Array, options?: AbortOptions) => Promise + has(routingKey: Uint8Array, options?: AbortOptions): Promise } /** From ca3f05a74316f53b0e44f5edd6e303b6e8463bf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 11:08:49 +0100 Subject: [PATCH 200/253] deps(dev): bump aegir from 40.0.13 to 41.0.0 (#36) Bumps [aegir](https://github.com/ipfs/aegir) from 40.0.13 to 41.0.0. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v40.0.13...v41.0.0) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 10 ++++++---- packages/interop/package.json | 3 ++- packages/json/package.json | 3 ++- packages/json/src/index.ts | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index f9cf02ce..510cb429 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,15 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.11" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 4da8f6fd..8def232a 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -51,7 +52,7 @@ "@helia/interface": "^2.0.0", "@helia/json": "^1.0.0", "@libp2p/websockets": "^7.0.5", - "aegir": "^40.0.11", + "aegir": "^41.0.0", "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", diff --git a/packages/json/package.json b/packages/json/package.json index cf4339aa..1c479036 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -141,7 +142,7 @@ "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^40.0.11", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1" } } diff --git a/packages/json/src/index.ts b/packages/json/src/index.ts index 25ae0459..053ebe09 100644 --- a/packages/json/src/index.ts +++ b/packages/json/src/index.ts @@ -65,7 +65,7 @@ export interface JSON { * // CID(bagaaierasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea) * ``` */ - add: (str: unknown, options?: Partial) => Promise + add(str: unknown, options?: Partial): Promise /** * Get an object from your Helia node, either previously added to it or to @@ -85,7 +85,7 @@ export interface JSON { * // { hello: 'world' } * ``` */ - get: (cid: CID, options?: Partial) => Promise + get(cid: CID, options?: Partial): Promise } class DefaultJSON implements JSON { From a52eae7ebcd1cb95a0c83c2f2d256f0f84774bf7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 7 Oct 2023 10:09:57 +0000 Subject: [PATCH 201/253] chore(release): 1.0.3 [skip ci] ## [@helia/dag-json-v1.0.3](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.2...@helia/dag-json-v1.0.3) (2023-10-07) ### Dependencies * bump @helia/interface from 1.2.2 to 2.0.0 ([#32](https://github.com/ipfs/helia-dag-json/issues/32)) ([eb836ef](https://github.com/ipfs/helia-dag-json/commit/eb836ef15f6bc754fbab4fdbe47c76f5492a56d9)) * **dev:** bump aegir from 40.0.13 to 41.0.0 ([#36](https://github.com/ipfs/helia-dag-json/issues/36)) ([9f57d11](https://github.com/ipfs/helia-dag-json/commit/9f57d11e461a3b1fddbc2a92e225d31eee56613c)) --- packages/dag-json/CHANGELOG.md | 8 ++++++++ packages/dag-json/package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/dag-json/CHANGELOG.md b/packages/dag-json/CHANGELOG.md index 3b56dee4..03d7ae87 100644 --- a/packages/dag-json/CHANGELOG.md +++ b/packages/dag-json/CHANGELOG.md @@ -1,3 +1,11 @@ +## [@helia/dag-json-v1.0.3](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.2...@helia/dag-json-v1.0.3) (2023-10-07) + + +### Dependencies + +* bump @helia/interface from 1.2.2 to 2.0.0 ([#32](https://github.com/ipfs/helia-dag-json/issues/32)) ([eb836ef](https://github.com/ipfs/helia-dag-json/commit/eb836ef15f6bc754fbab4fdbe47c76f5492a56d9)) +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#36](https://github.com/ipfs/helia-dag-json/issues/36)) ([9f57d11](https://github.com/ipfs/helia-dag-json/commit/9f57d11e461a3b1fddbc2a92e225d31eee56613c)) + ## [@helia/dag-json-v1.0.2](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.1...@helia/dag-json-v1.0.2) (2023-08-27) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 0477454d..7c95ad4c 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-json", - "version": "1.0.2", + "version": "1.0.3", "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", From 15d42a0f8f96d15c1e63b5ada3fa25071933c577 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 7 Oct 2023 10:12:39 +0000 Subject: [PATCH 202/253] chore(release): 1.0.2 [skip ci] ## [@helia/mfs-v1.0.2](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.1...@helia/mfs-v1.0.2) (2023-10-07) ### Dependencies * **dev:** bump helia from 2.0.1 to 2.0.3 ([#10](https://github.com/ipfs/helia-mfs/issues/10)) ([6911470](https://github.com/ipfs/helia-mfs/commit/6911470cb43720798fca571669a166eb3689dad2)) --- packages/mfs/CHANGELOG.md | 7 +++++++ packages/mfs/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/mfs/CHANGELOG.md b/packages/mfs/CHANGELOG.md index 9cd89e2f..b5a21fcc 100644 --- a/packages/mfs/CHANGELOG.md +++ b/packages/mfs/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/mfs-v1.0.2](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.1...@helia/mfs-v1.0.2) (2023-10-07) + + +### Dependencies + +* **dev:** bump helia from 2.0.1 to 2.0.3 ([#10](https://github.com/ipfs/helia-mfs/issues/10)) ([6911470](https://github.com/ipfs/helia-mfs/commit/6911470cb43720798fca571669a166eb3689dad2)) + ## [@helia/mfs-v1.0.1](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.0...@helia/mfs-v1.0.1) (2023-09-08) diff --git a/packages/mfs/package.json b/packages/mfs/package.json index e66fc14b..00bf6715 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/mfs", - "version": "1.0.1", + "version": "1.0.2", "description": "A mutable filesystem powered by Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/mfs#readme", From 02d52f9226d68ac6f98dba0f742cfe49ae070fe5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 7 Oct 2023 10:13:49 +0000 Subject: [PATCH 203/253] chore(release): 2.0.1 [skip ci] ## [@helia/ipns-v2.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.0...@helia/ipns-v2.0.1) (2023-10-07) ### Dependencies * **dev:** bump aegir from 40.0.13 to 41.0.0 ([#107](https://github.com/ipfs/helia-ipns/issues/107)) ([5402d30](https://github.com/ipfs/helia-ipns/commit/5402d30de1437052e9e9b955d9be3c2898515447)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 49862170..f5512596 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v2.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.0...@helia/ipns-v2.0.1) (2023-10-07) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#107](https://github.com/ipfs/helia-ipns/issues/107)) ([5402d30](https://github.com/ipfs/helia-ipns/commit/5402d30de1437052e9e9b955d9be3c2898515447)) + ## [@helia/ipns-v2.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v1.1.7...@helia/ipns-v2.0.0) (2023-09-22) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 42fb6a77..fd4d2b6f 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "2.0.0", + "version": "2.0.1", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From e7b9a9d5c420fca8c77eadd2685fb3614253bbbc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 7 Oct 2023 10:13:58 +0000 Subject: [PATCH 204/253] chore(release): 1.0.3 [skip ci] ## [@helia/dag-cbor-v1.0.3](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.2...@helia/dag-cbor-v1.0.3) (2023-10-07) ### Dependencies * **dev:** bump aegir from 40.0.13 to 41.0.0 ([#36](https://github.com/ipfs/helia-dag-cbor/issues/36)) ([77e29bc](https://github.com/ipfs/helia-dag-cbor/commit/77e29bcdda33387b8bf15124bc316ef03b434433)) --- packages/dag-cbor/CHANGELOG.md | 7 +++++++ packages/dag-cbor/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dag-cbor/CHANGELOG.md b/packages/dag-cbor/CHANGELOG.md index 76b702fd..3a3b9f75 100644 --- a/packages/dag-cbor/CHANGELOG.md +++ b/packages/dag-cbor/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/dag-cbor-v1.0.3](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.2...@helia/dag-cbor-v1.0.3) (2023-10-07) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#36](https://github.com/ipfs/helia-dag-cbor/issues/36)) ([77e29bc](https://github.com/ipfs/helia-dag-cbor/commit/77e29bcdda33387b8bf15124bc316ef03b434433)) + ## [@helia/dag-cbor-v1.0.2](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.1...@helia/dag-cbor-v1.0.2) (2023-08-27) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 2f223b4d..7aab7be2 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-cbor", - "version": "1.0.2", + "version": "1.0.3", "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/dag-cbor#readme", From d0e859796b9d4bfead3505b2b60bef52ca5d18b6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 7 Oct 2023 10:14:50 +0000 Subject: [PATCH 205/253] chore(release): 1.0.4 [skip ci] ## [@helia/car-v1.0.4](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.3...@helia/car-v1.0.4) (2023-10-07) ### Dependencies * **dev:** bump aegir from 40.0.13 to 41.0.0 ([#41](https://github.com/ipfs/helia-car/issues/41)) ([e8fc99f](https://github.com/ipfs/helia-car/commit/e8fc99f4e372eaf72c2598f5a7a9942143c6d788)) --- packages/car/CHANGELOG.md | 7 +++++++ packages/car/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/car/CHANGELOG.md b/packages/car/CHANGELOG.md index 578e2995..b718dda8 100644 --- a/packages/car/CHANGELOG.md +++ b/packages/car/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/car-v1.0.4](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.3...@helia/car-v1.0.4) (2023-10-07) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#41](https://github.com/ipfs/helia-car/issues/41)) ([e8fc99f](https://github.com/ipfs/helia-car/commit/e8fc99f4e372eaf72c2598f5a7a9942143c6d788)) + ## [@helia/car-v1.0.3](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.2...@helia/car-v1.0.3) (2023-08-27) diff --git a/packages/car/package.json b/packages/car/package.json index ec745b42..2ef8852d 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -1,6 +1,6 @@ { "name": "@helia/car", - "version": "1.0.3", + "version": "1.0.4", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", From 903d9e4fa92c9673495f96d5a2dcb63c426293f9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 7 Oct 2023 10:17:40 +0000 Subject: [PATCH 206/253] chore(release): 1.0.3 [skip ci] ## [@helia/json-v1.0.3](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.2...@helia/json-v1.0.3) (2023-10-07) ### Dependencies * **dev:** bump aegir from 40.0.13 to 41.0.0 ([#36](https://github.com/ipfs/helia-json/issues/36)) ([ca3f05a](https://github.com/ipfs/helia-json/commit/ca3f05a74316f53b0e44f5edd6e303b6e8463bf2)) --- packages/json/CHANGELOG.md | 7 +++++++ packages/json/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/json/CHANGELOG.md b/packages/json/CHANGELOG.md index 2ca813a6..0b84cecf 100644 --- a/packages/json/CHANGELOG.md +++ b/packages/json/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/json-v1.0.3](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.2...@helia/json-v1.0.3) (2023-10-07) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#36](https://github.com/ipfs/helia-json/issues/36)) ([ca3f05a](https://github.com/ipfs/helia-json/commit/ca3f05a74316f53b0e44f5edd6e303b6e8463bf2)) + ## [@helia/json-v1.0.2](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.1...@helia/json-v1.0.2) (2023-08-27) diff --git a/packages/json/package.json b/packages/json/package.json index 1c479036..208945c5 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/json", - "version": "1.0.2", + "version": "1.0.3", "description": "Add/get IPLD blocks containing json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/json#readme", From 2421ee2b4440446160e1a665bc5ecfc92d2b64de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 08:52:56 +0300 Subject: [PATCH 207/253] deps(dev): bump aegir from 40.0.13 to 41.0.0 (#105) Bumps [aegir](https://github.com/ipfs/aegir) from 40.0.13 to 41.0.0. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v40.0.13...v41.0.0) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: achingbrain --- package.json | 10 ++++++---- packages/interop/package.json | 3 ++- packages/unixfs/package.json | 3 ++- packages/unixfs/src/index.ts | 26 +++++++++++++------------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 13f12b41..6960c4c8 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,15 @@ "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", - "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop", - "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop", "dep-check": "aegir run dep-check", - "release": "npm run docs:no-publish && aegir run release && npm run docs" + "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir run release", + "docs": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs -- --exclude packages/interop --excludeExternals", + "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^40.0.13" + "aegir": "^41.0.0", + "npm-run-all": "^4.1.5" }, "type": "module", "workspaces": [ diff --git a/packages/interop/package.json b/packages/interop/package.json index 4ccd86b5..7ee17a56 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -31,6 +31,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -54,7 +55,7 @@ "@helia/unixfs": "^1.0.0", "@libp2p/tcp": "^8.0.6", "@libp2p/websockets": "^7.0.6", - "aegir": "^40.0.13", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", "go-ipfs": "^0.22.0", diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index b827cdcb..5c1bcc9b 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -51,6 +51,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -173,7 +174,7 @@ "sparse-array": "^1.3.2" }, "devDependencies": { - "aegir": "^40.0.13", + "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "delay": "^6.0.0", "iso-url": "^1.2.1", diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index 84f96ff6..6019dff9 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -363,7 +363,7 @@ export interface UnixFS { * } * ``` */ - addAll: (source: ImportCandidateStream, options?: Partial) => AsyncIterable + addAll(source: ImportCandidateStream, options?: Partial): AsyncIterable /** * Add a single `Uint8Array` to your Helia node as a file. @@ -376,7 +376,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addBytes: (bytes: Uint8Array, options?: Partial) => Promise + addBytes(bytes: Uint8Array, options?: Partial): Promise /** * Add a stream of `Uint8Array` to your Helia node as a file. @@ -392,7 +392,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addByteStream: (bytes: ByteStream, options?: Partial) => Promise + addByteStream(bytes: ByteStream, options?: Partial): Promise /** * Add a file to your Helia node with optional metadata. @@ -413,7 +413,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addFile: (file: FileCandidate, options?: Partial) => Promise + addFile(file: FileCandidate, options?: Partial): Promise /** * Add a directory to your Helia node. @@ -426,7 +426,7 @@ export interface UnixFS { * console.info(cid) * ``` */ - addDirectory: (dir?: Partial, options?: Partial) => Promise + addDirectory(dir?: Partial, options?: Partial): Promise /** * Retrieve the contents of a file from your Helia node. @@ -439,7 +439,7 @@ export interface UnixFS { * } * ``` */ - cat: (cid: CID, options?: Partial) => AsyncIterable + cat(cid: CID, options?: Partial): AsyncIterable /** * Change the permissions on a file or directory in a DAG @@ -457,7 +457,7 @@ export interface UnixFS { * console.info(afterCid, afterStats) * ``` */ - chmod: (cid: CID, mode: number, options?: Partial) => Promise + chmod(cid: CID, mode: number, options?: Partial): Promise /** * Add a file or directory to a target directory. @@ -473,7 +473,7 @@ export interface UnixFS { * console.info(updatedCid) * ``` */ - cp: (source: CID, target: CID, name: string, options?: Partial) => Promise + cp(source: CID, target: CID, name: string, options?: Partial): Promise /** * List directory contents. @@ -486,7 +486,7 @@ export interface UnixFS { * } * ``` */ - ls: (cid: CID, options?: Partial) => AsyncIterable + ls(cid: CID, options?: Partial): AsyncIterable /** * Make a new directory under an existing directory. @@ -501,7 +501,7 @@ export interface UnixFS { * console.info(updatedCid) * ``` */ - mkdir: (cid: CID, dirname: string, options?: Partial) => Promise + mkdir(cid: CID, dirname: string, options?: Partial): Promise /** * Remove a file or directory from an existing directory. @@ -517,7 +517,7 @@ export interface UnixFS { * console.info(finalCid) * ``` */ - rm: (cid: CID, path: string, options?: Partial) => Promise + rm(cid: CID, path: string, options?: Partial): Promise /** * Return statistics about a UnixFS DAG. @@ -532,7 +532,7 @@ export interface UnixFS { * console.info(stats) * ``` */ - stat: (cid: CID, options?: Partial) => Promise + stat(cid: CID, options?: Partial): Promise /** * Update the mtime of a UnixFS DAG @@ -550,7 +550,7 @@ export interface UnixFS { * console.info(afterCid, afterStats) * ``` */ - touch: (cid: CID, options?: Partial) => Promise + touch(cid: CID, options?: Partial): Promise } class DefaultUnixFS implements UnixFS { From 530aeff8af103c9126411cc1b035ee106f113f1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 14:33:39 +0100 Subject: [PATCH 208/253] deps(dev): bump sinon from 16.1.3 to 17.0.0 (#108) Bumps [sinon](https://github.com/sinonjs/sinon) from 16.1.3 to 17.0.0. - [Release notes](https://github.com/sinonjs/sinon/releases) - [Changelog](https://github.com/sinonjs/sinon/blob/main/docs/changelog.md) - [Commits](https://github.com/sinonjs/sinon/compare/v16.1.3...v17.0.0) --- updated-dependencies: - dependency-name: sinon dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/ipns/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index fd4d2b6f..58c164a2 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -172,7 +172,7 @@ "@libp2p/peer-id-factory": "^3.0.3", "aegir": "^41.0.0", "datastore-core": "^9.0.3", - "sinon": "^16.0.0", + "sinon": "^17.0.0", "sinon-ts": "^1.0.0" }, "browser": { From 31c3f5cec449b867078d74856421431b4015096d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 24 Oct 2023 16:34:57 +0000 Subject: [PATCH 209/253] chore(release): 2.0.2 [skip ci] ## [@helia/ipns-v2.0.2](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.1...@helia/ipns-v2.0.2) (2023-10-24) ### Dependencies * **dev:** bump sinon from 16.1.3 to 17.0.0 ([#108](https://github.com/ipfs/helia-ipns/issues/108)) ([530aeff](https://github.com/ipfs/helia-ipns/commit/530aeff8af103c9126411cc1b035ee106f113f1f)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index f5512596..4c52d406 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v2.0.2](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.1...@helia/ipns-v2.0.2) (2023-10-24) + + +### Dependencies + +* **dev:** bump sinon from 16.1.3 to 17.0.0 ([#108](https://github.com/ipfs/helia-ipns/issues/108)) ([530aeff](https://github.com/ipfs/helia-ipns/commit/530aeff8af103c9126411cc1b035ee106f113f1f)) + ## [@helia/ipns-v2.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.0...@helia/ipns-v2.0.1) (2023-10-07) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 58c164a2..a9b426cd 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "2.0.1", + "version": "2.0.2", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 514b6e1e4192f700a6f0e769d52a4ec5dfe757ec Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 25 Oct 2023 18:11:25 +0100 Subject: [PATCH 210/253] fix: update libp2p interfaces (#109) Remove deprecated interfaces for newer versions. --- packages/ipns/package.json | 2 +- packages/ipns/src/routing/local-store.ts | 6 +++--- packages/ipns/test/resolve.spec.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index a9b426cd..9e9a9f8a 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -156,9 +156,9 @@ }, "dependencies": { "@libp2p/interface": "^0.1.2", + "@libp2p/kad-dht": "^10.0.11", "@libp2p/logger": "^3.0.2", "@libp2p/peer-id": "^3.0.2", - "@libp2p/record": "^3.0.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", "ipns": "^7.0.1", diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index 06302129..bb142e18 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -1,4 +1,4 @@ -import { Libp2pRecord } from '@libp2p/record' +import { Record } from '@libp2p/kad-dht' import { type Datastore, Key } from 'interface-datastore' import { CustomProgressEvent, type ProgressEvent } from 'progress-events' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' @@ -30,7 +30,7 @@ export function localStore (datastore: Datastore): LocalStore { const key = dhtRoutingKey(routingKey) // Marshal to libp2p record as the DHT does - const record = new Libp2pRecord(routingKey, marshalledRecord, new Date()) + const record = new Record(routingKey, marshalledRecord, new Date()) options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put')) await datastore.put(key, record.serialize(), options) @@ -47,7 +47,7 @@ export function localStore (datastore: Datastore): LocalStore { const buf = await datastore.get(key, options) // Unmarshal libp2p record as the DHT does - const record = Libp2pRecord.deserialize(buf) + const record = Record.deserialize(buf) return record.value } catch (err: any) { diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index 7815ef85..e605da32 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -1,7 +1,7 @@ /* eslint-env mocha */ +import { Record } from '@libp2p/kad-dht' import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { Libp2pRecord } from '@libp2p/record' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import { type Datastore, Key } from 'interface-datastore' @@ -141,7 +141,7 @@ describe('resolve', () => { expect(result.toString()).to.equal(cid.toV1().toString(), 'incorrect record resolved') const cached = await datastore.get(dhtKey) - const record = Libp2pRecord.deserialize(cached) + const record = Record.deserialize(cached) // should have cached the updated record expect(record.value).to.equalBytes(marshalledRecordB) From d954e0a74eceeb9f6e1b3aa9a49448b79ca3aca7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 26 Oct 2023 14:52:38 +0000 Subject: [PATCH 211/253] chore(release): 2.0.3 [skip ci] ## [@helia/ipns-v2.0.3](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.2...@helia/ipns-v2.0.3) (2023-10-26) ### Bug Fixes * update libp2p interfaces ([#109](https://github.com/ipfs/helia-ipns/issues/109)) ([514b6e1](https://github.com/ipfs/helia-ipns/commit/514b6e1e4192f700a6f0e769d52a4ec5dfe757ec)) --- packages/ipns/CHANGELOG.md | 7 +++++++ packages/ipns/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 4c52d406..bcd19099 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,10 @@ +## [@helia/ipns-v2.0.3](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.2...@helia/ipns-v2.0.3) (2023-10-26) + + +### Bug Fixes + +* update libp2p interfaces ([#109](https://github.com/ipfs/helia-ipns/issues/109)) ([514b6e1](https://github.com/ipfs/helia-ipns/commit/514b6e1e4192f700a6f0e769d52a4ec5dfe757ec)) + ## [@helia/ipns-v2.0.2](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.1...@helia/ipns-v2.0.2) (2023-10-24) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 9e9a9f8a..cb19b4a5 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "2.0.2", + "version": "2.0.3", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From f13daae3d62579b0b181bc5387c5d7b8e10029a5 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 30 Nov 2023 12:55:24 -0800 Subject: [PATCH 212/253] deps: go-ipfs -> kubo (#53) --- packages/interop/.aegir.js | 2 +- packages/interop/package.json | 4 ++-- packages/interop/test/fixtures/create-kubo.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index 498799ac..35da9fa6 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -12,7 +12,7 @@ export default { host: '127.0.0.1', port: ipfsdPort }, { - ipfsBin: (await import('go-ipfs')).default.path(), + ipfsBin: (await import('kubo')).default.path(), kuboRpcModule: kuboRpcClient, ipfsOptions: { config: { diff --git a/packages/interop/package.json b/packages/interop/package.json index 549fd8e6..99c135c6 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,10 +58,10 @@ "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.22.0", "helia": "^2.0.3", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", + "kubo": "^0.23.0", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.46.12", "merge-options": "^3.0.4", @@ -70,7 +70,7 @@ }, "browser": { "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", - "go-ipfs": false + "kubo": false }, "private": true, "typedoc": { diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index c032e86d..75f2281f 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,5 +1,5 @@ -import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kubo from 'kubo' import * as kuboRpcClient from 'kubo-rpc-client' import mergeOptions from 'merge-options' import { isElectronMain, isNode } from 'wherearewe' @@ -7,7 +7,7 @@ import { isElectronMain, isNode } from 'wherearewe' export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { const opts = mergeOptions({ kuboRpcModule: kuboRpcClient, - ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + ipfsBin: isNode || isElectronMain ? kubo.path() : undefined, test: true, endpoint: process.env.IPFSD_SERVER, ipfsOptions: { From 1184ea695987cee7f922b7954c8bc626bc035dba Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Fri, 1 Dec 2023 08:47:26 +0000 Subject: [PATCH 213/253] chore: update sibling dependencies --- packages/strings/README.md | 50 ++++++++++---------------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/packages/strings/README.md b/packages/strings/README.md index 7ef5902c..bbca118d 100644 --- a/packages/strings/README.md +++ b/packages/strings/README.md @@ -13,47 +13,23 @@ > Add/get IPLD blocks containing strings with your Helia node -## Table of contents +# About -- [Install](#install) - - [Browser ` +console.info(string) +// hello world ``` - -## API Docs - -- - -## License - -Licensed under either of - -- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) -- MIT ([LICENSE-MIT](LICENSE-MIT) / ) - -## Contribute - -Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). - -Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. - -Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). - -Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. - -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) From cd9e903c2ccac61372eaa64a61b4a8f3d79f9d4a Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Sun, 3 Dec 2023 08:17:42 +0000 Subject: [PATCH 214/253] fix: convert date to mtime in glob source (#106) The UnixFS importer requires mtimes to follow the `Mtime` interface so convert files/folders read from glob source. --- packages/interop/.aegir.js | 2 +- packages/interop/package.json | 5 +- .../test/fixtures/create-helia.browser.ts | 6 +- .../interop/test/fixtures/create-helia.ts | 6 +- packages/interop/test/fixtures/create-kubo.ts | 3 +- packages/unixfs/src/utils/glob-source.ts | 3 +- packages/unixfs/src/utils/to-mtime.ts | 57 +++++++++++++++++ .../unixfs/test/utils/glob-source.spec.ts | 24 ++++---- packages/unixfs/test/utils/to-mtime.spec.ts | 61 +++++++++++++++++++ 9 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 packages/unixfs/src/utils/to-mtime.ts create mode 100644 packages/unixfs/test/utils/to-mtime.spec.ts diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index 498799ac..35da9fa6 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -12,7 +12,7 @@ export default { host: '127.0.0.1', port: ipfsdPort }, { - ipfsBin: (await import('go-ipfs')).default.path(), + ipfsBin: (await import('kubo')).default.path(), kuboRpcModule: kuboRpcClient, ipfsOptions: { config: { diff --git a/packages/interop/package.json b/packages/interop/package.json index 7ee17a56..c75c60f4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -44,7 +44,6 @@ "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", "test:firefox": "aegir test -t browser -- --browser firefox", - "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", "test:node": "aegir test -t node --cov", "test:electron-main": "aegir test -t electron-main" }, @@ -58,7 +57,7 @@ "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.22.0", + "kubo": "^0.24.0", "helia": "^2.0.1", "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", @@ -72,7 +71,7 @@ }, "browser": { "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", - "go-ipfs": false + "kubo": false }, "private": true } diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 867de492..f94bc9df 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -5,6 +5,7 @@ import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' +import { bitswap } from 'helia/block-brokers' import { createLibp2p, type Libp2pOptions } from 'libp2p' import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' @@ -39,7 +40,10 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise 1) { + output.nsecs = mtimeLike[1] + } + + return output + } + + if (typeof mtimeLike.Seconds === 'number') { + const output: Mtime = { + secs: BigInt(mtimeLike.Seconds) + } + + if (mtimeLike.FractionalNanoseconds != null) { + output.nsecs = mtimeLike.FractionalNanoseconds + } + + return output + } + + throw new Error('Cannot convert object to mtime') +} + +function dateToTimespec (date: Date): Mtime { + const ms = date.getTime() + const secs = Math.floor(ms / 1000) + + return { + secs: BigInt(secs), + nsecs: (ms - (secs * 1000)) * 1000 + } +} + +function isMtime (obj: any): obj is Mtime { + return typeof obj.secs === 'bigint' +} diff --git a/packages/unixfs/test/utils/glob-source.spec.ts b/packages/unixfs/test/utils/glob-source.spec.ts index ebb09c9c..0021597f 100644 --- a/packages/unixfs/test/utils/glob-source.spec.ts +++ b/packages/unixfs/test/utils/glob-source.spec.ts @@ -7,6 +7,8 @@ import { expect } from 'aegir/chai' import all from 'it-all' import { isNode } from 'wherearewe' import { globSource } from '../../src/utils/glob-source.js' +import { toMtime } from '../../src/utils/to-mtime.js' +import type { Mtime } from 'ipfs-unixfs' function fixtureDir (): string { const filename = fileURLToPath(import.meta.url) @@ -23,8 +25,8 @@ function findMode (file: string): number { return fs.statSync(fixture(file)).mode } -function findMtime (file: string): Date { - return fs.statSync(fixture(file)).mtime +function findMtime (file: string): Mtime { + return toMtime(fs.statSync(fixture(file)).mtime) } describe('glob-source', () => { @@ -228,28 +230,28 @@ describe('glob-source', () => { } const result = await all(globSource(fixtureDir(), '{dir,dir/**/*}', { - mtime: new Date(5) + mtime: toMtime(new Date(5)) })) expect(result).to.have.lengthOf(6) expect(result).to.containSubset([{ path: '/dir', - mtime: new Date(5) + mtime: toMtime(new Date(5)) }, { path: '/dir/file-1.txt', - mtime: new Date(5) + mtime: toMtime(new Date(5)) }, { path: '/dir/file-2.js', - mtime: new Date(5) + mtime: toMtime(new Date(5)) }, { path: '/dir/file-3.css', - mtime: new Date(5) + mtime: toMtime(new Date(5)) }, { path: '/dir/nested-dir', - mtime: new Date(5) + mtime: toMtime(new Date(5)) }, { path: '/dir/nested-dir/other.txt', - mtime: new Date(5) + mtime: toMtime(new Date(5)) }]) }) @@ -274,7 +276,7 @@ describe('glob-source', () => { mtime: [5, 0] })) - expect(result).to.have.deep.nested.property('[0].mtime', [5, 0]) + expect(result).to.have.deep.nested.property('[0].mtime', toMtime([5, 0])) }) it('overrides mtime for file with UnixFS timespec', async function () { @@ -286,6 +288,6 @@ describe('glob-source', () => { mtime: { Seconds: 5, FractionalNanoseconds: 0 } })) - expect(result).to.have.deep.nested.property('[0].mtime', { Seconds: 5, FractionalNanoseconds: 0 }) + expect(result).to.have.deep.nested.property('[0].mtime', toMtime({ Seconds: 5, FractionalNanoseconds: 0 })) }) }) diff --git a/packages/unixfs/test/utils/to-mtime.spec.ts b/packages/unixfs/test/utils/to-mtime.spec.ts new file mode 100644 index 00000000..d670497b --- /dev/null +++ b/packages/unixfs/test/utils/to-mtime.spec.ts @@ -0,0 +1,61 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { toMtime } from '../../src/utils/to-mtime.js' + +describe('to-mtime', () => { + it('should survive undefined', async function () { + const result = toMtime() + + expect(result).to.equal(undefined) + }) + + it('should convert a date', async function () { + const input = new Date() + const result = toMtime(input) + + expect(result?.secs).to.equal(BigInt(Math.floor(input.getTime() / 1000))) + }) + + it('should convert a timespec', async function () { + const input = { + Seconds: 100 + } + const result = toMtime(input) + + expect(result?.secs).to.equal(BigInt(input.Seconds)) + expect(result?.nsecs).to.be.undefined() + }) + + it('should convert a timespec with fractional nanoseconds', async function () { + const input = { + Seconds: 100, + FractionalNanoseconds: 5 + } + const result = toMtime(input) + + expect(result?.secs).to.equal(BigInt(input.Seconds)) + expect(result?.nsecs).to.equal(input.FractionalNanoseconds) + }) + + it('should convert a mtime', async function () { + const input = { + secs: 100n + } + const result = toMtime(input) + + expect(result?.secs).to.equal(input.secs) + expect(result?.nsecs).to.be.undefined() + }) + + it('should convert a mtime with fractional nanoseconds', async function () { + const input = { + secs: 100n, + nsecs: 5 + } + const result = toMtime(input) + + expect(result?.secs).to.equal(input.secs) + expect(result?.nsecs).to.equal(input.nsecs) + }) +}) From ffe7629c5fb81b5d9baade91a328d9494af4d829 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sun, 3 Dec 2023 09:44:18 +0000 Subject: [PATCH 215/253] chore(release): 1.4.3 [skip ci] ## [@helia/unixfs-v1.4.3](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.2...@helia/unixfs-v1.4.3) (2023-12-03) ### Bug Fixes * convert date to mtime in glob source ([#106](https://github.com/ipfs/helia-unixfs/issues/106)) ([cd9e903](https://github.com/ipfs/helia-unixfs/commit/cd9e903c2ccac61372eaa64a61b4a8f3d79f9d4a)) ### Dependencies * **dev:** bump aegir from 40.0.13 to 41.0.0 ([#105](https://github.com/ipfs/helia-unixfs/issues/105)) ([2421ee2](https://github.com/ipfs/helia-unixfs/commit/2421ee2b4440446160e1a665bc5ecfc92d2b64de)) --- packages/unixfs/CHANGELOG.md | 12 ++++++++++++ packages/unixfs/package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index 4aca9ab0..dac40a95 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,15 @@ +## [@helia/unixfs-v1.4.3](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.2...@helia/unixfs-v1.4.3) (2023-12-03) + + +### Bug Fixes + +* convert date to mtime in glob source ([#106](https://github.com/ipfs/helia-unixfs/issues/106)) ([cd9e903](https://github.com/ipfs/helia-unixfs/commit/cd9e903c2ccac61372eaa64a61b4a8f3d79f9d4a)) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#105](https://github.com/ipfs/helia-unixfs/issues/105)) ([2421ee2](https://github.com/ipfs/helia-unixfs/commit/2421ee2b4440446160e1a665bc5ecfc92d2b64de)) + ## [@helia/unixfs-v1.4.2](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.1...@helia/unixfs-v1.4.2) (2023-09-14) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 5c1bcc9b..23f09931 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.4.2", + "version": "1.4.3", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", From 1b0b2ef05c5cbd78c3b5d5629237200a69bbd5dd Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Sun, 3 Dec 2023 09:44:31 +0000 Subject: [PATCH 216/253] chore: update sibling dependencies --- packages/unixfs/README.md | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/unixfs/README.md b/packages/unixfs/README.md index 22527bdb..ac26f11f 100644 --- a/packages/unixfs/README.md +++ b/packages/unixfs/README.md @@ -13,6 +13,49 @@ > A Helia-compatible wrapper for UnixFS +# About + +`@helia/unixfs` is an implementation of a filesystem compatible with Helia. + +See the interface for all available operations. + +## Example + +```typescript +import { createHelia } from 'helia' +import { unixfs } from '@helia/unixfs' + +const helia = createHelia({ + // ... helia config +}) +const fs = unixfs(helia) + +// create an empty dir and a file, then add the file to the dir +const emptyDirCid = await fs.addDirectory() +const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) +const updateDirCid = await fs.cp(fileCid, emptyDirCid, 'foo.txt') + +// or doing the same thing as a stream +for await (const entry of fs.addAll([{ + path: 'foo.txt', + content: Uint8Array.from([0, 1, 2, 3]) +}])) { + console.info(entry) +} +``` + +## Example + +Recursively adding a directory (Node.js-compatibly environments only): + +```typescript +import { globSource } from '@helia/unixfs' + +for await (const entry of fs.addAll(globSource('path/to/containing/dir', 'glob-pattern'))) { + console.info(entry) +} +``` + ## Table of contents - [Install](#install) From 2ac0e8b26556b73961e67191c564ac2b18d32b31 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 5 Dec 2023 23:19:21 +0000 Subject: [PATCH 217/253] feat!: support DNS over HTTPS and DNS-JSON over HTTPS (#55) Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> BREAKING CHANGE: alters the options object passed to the `ipns` factory function --- .gitignore | 3 + package.json | 2 +- packages/interop/.aegir.js | 2 +- packages/interop/package.json | 5 +- packages/interop/test/dht.spec.ts | 8 +- packages/interop/test/fixtures/create-kubo.ts | 6 +- packages/interop/test/pubsub.spec.ts | 8 +- packages/ipns/package.json | 15 +- packages/ipns/src/dns-resolvers/default.ts | 9 ++ .../src/dns-resolvers/dns-json-over-https.ts | 90 +++++++++++ .../ipns/src/dns-resolvers/dns-over-https.ts | 146 ++++++++++++++++++ packages/ipns/src/dns-resolvers/index.ts | 2 + .../src/dns-resolvers/resolver.browser.ts | 50 ++++++ packages/ipns/src/dns-resolvers/resolver.ts | 25 +++ packages/ipns/src/index.ts | 134 ++++++++++++++-- packages/ipns/src/utils/dns.ts | 126 +++++++++++++++ .../src/utils/resolve-dns-link.browser.ts | 61 -------- packages/ipns/src/utils/resolve-dns-link.ts | 65 -------- packages/ipns/test/dns-resolvers.spec.ts | 86 +++++++++++ packages/ipns/test/publish.spec.ts | 2 +- packages/ipns/test/resolve.spec.ts | 19 +-- packages/ipns/test/resolveDns.spec.ts | 41 +++++ 22 files changed, 729 insertions(+), 176 deletions(-) create mode 100644 packages/ipns/src/dns-resolvers/default.ts create mode 100644 packages/ipns/src/dns-resolvers/dns-json-over-https.ts create mode 100644 packages/ipns/src/dns-resolvers/dns-over-https.ts create mode 100644 packages/ipns/src/dns-resolvers/index.ts create mode 100644 packages/ipns/src/dns-resolvers/resolver.browser.ts create mode 100644 packages/ipns/src/dns-resolvers/resolver.ts create mode 100644 packages/ipns/src/utils/dns.ts delete mode 100644 packages/ipns/src/utils/resolve-dns-link.browser.ts delete mode 100644 packages/ipns/src/utils/resolve-dns-link.ts create mode 100644 packages/ipns/test/dns-resolvers.spec.ts create mode 100644 packages/ipns/test/resolveDns.spec.ts diff --git a/.gitignore b/.gitignore index 7ad9e674..7d54fd58 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ node_modules package-lock.json yarn.lock .vscode +.env +.envrc +.tool-versions diff --git a/package.json b/package.json index e9b39b85..c95b525f 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^41.1.14", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index cc52af59..381983c0 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -12,7 +12,7 @@ export default { host: '127.0.0.1', port: ipfsdPort }, { - ipfsBin: (await import('go-ipfs')).default.path(), + ipfsBin: (await import('kubo')).default.path(), kuboRpcModule: kuboRpcClient, ipfsOptions: { config: { diff --git a/packages/interop/package.json b/packages/interop/package.json index 8b13403c..a224d226 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -60,16 +60,15 @@ "@libp2p/peer-id-factory": "^3.0.3", "@libp2p/tcp": "^8.0.4", "@libp2p/websockets": "^7.0.4", - "aegir": "^41.0.0", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "go-ipfs": "^0.22.0", "helia": "^2.0.1", "ipfsd-ctl": "^13.0.0", "ipns": "^7.0.1", "it-all": "^3.0.2", "it-last": "^3.0.1", "it-map": "^3.0.3", + "kubo": "^0.24.0", "kubo-rpc-client": "^3.0.0", "libp2p": "^0.46.6", "merge-options": "^3.0.4", @@ -79,7 +78,7 @@ }, "browser": { "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", - "go-ipfs": false + "kubo": false }, "private": true } diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index 6c07ea3c..bef69b9b 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -131,9 +131,11 @@ keyTypes.forEach(type => { message: 'Kubo could not find Helia on the DHT' }) - name = ipns(helia, [ - dht(helia) - ]) + name = ipns(helia, { + routers: [ + dht(helia) + ] + }) } afterEach(async () => { diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index 518c8a68..0c7c5a5f 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -1,7 +1,5 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */ -// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped -import * as goIpfs from 'go-ipfs' import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl' +import * as kubo from 'kubo' import * as kuboRpcClient from 'kubo-rpc-client' import mergeOptions from 'merge-options' import { isElectronMain, isNode } from 'wherearewe' @@ -9,7 +7,7 @@ import { isElectronMain, isNode } from 'wherearewe' export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { const opts = mergeOptions({ kuboRpcModule: kuboRpcClient, - ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + ipfsBin: isNode || isElectronMain ? kubo.path() : undefined, test: true, endpoint: process.env.IPFSD_SERVER, ipfsOptions: { diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index 22fff4a3..ff32652d 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -52,9 +52,11 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { // connect the two nodes await connect(helia, kubo, '/meshsub/1.1.0') - name = ipns(helia, [ - pubsub(helia) - ]) + name = ipns(helia, { + routers: [ + pubsub(helia) + ] + }) }) afterEach(async () => { diff --git a/packages/ipns/package.json b/packages/ipns/package.json index cb19b4a5..38b87b6c 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -46,6 +46,10 @@ "./routing": { "types": "./dist/src/routing/index.d.ts", "import": "./dist/src/routing/index.js" + }, + "./dns-resolvers": { + "types": "./dist/src/dns-resolvers/index.d.ts", + "import": "./dist/src/dns-resolvers/index.js" } }, "eslintConfig": { @@ -155,10 +159,11 @@ "release": "aegir release" }, "dependencies": { - "@libp2p/interface": "^0.1.2", "@libp2p/kad-dht": "^10.0.11", "@libp2p/logger": "^3.0.2", "@libp2p/peer-id": "^3.0.2", + "dns-over-http-resolver": "^2.1.3", + "dns-packet": "^5.6.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", "ipns": "^7.0.1", @@ -169,13 +174,17 @@ "uint8arrays": "^4.0.3" }, "devDependencies": { + "@libp2p/interface": "^0.1.4", "@libp2p/peer-id-factory": "^3.0.3", - "aegir": "^41.0.0", + "@types/dns-packet": "^5.6.4", "datastore-core": "^9.0.3", "sinon": "^17.0.0", "sinon-ts": "^1.0.0" }, "browser": { - "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" + "./dist/src/dns-resolvers/resolver.js": "./dist/src/dns-resolvers/resolver.browser.js" + }, + "typedoc": { + "entryPoint": "./src/index.ts" } } diff --git a/packages/ipns/src/dns-resolvers/default.ts b/packages/ipns/src/dns-resolvers/default.ts new file mode 100644 index 00000000..0586b72c --- /dev/null +++ b/packages/ipns/src/dns-resolvers/default.ts @@ -0,0 +1,9 @@ +import { MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js' +import resolve from './resolver.js' +import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js' + +export function defaultResolver (): DNSResolver { + return async (domain: string, options: ResolveDnsLinkOptions = {}): Promise => { + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options) + } +} diff --git a/packages/ipns/src/dns-resolvers/dns-json-over-https.ts b/packages/ipns/src/dns-resolvers/dns-json-over-https.ts new file mode 100644 index 00000000..ece490df --- /dev/null +++ b/packages/ipns/src/dns-resolvers/dns-json-over-https.ts @@ -0,0 +1,90 @@ +/* eslint-env browser */ + +import PQueue from 'p-queue' +import { CustomProgressEvent } from 'progress-events' +import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js' +import { TLRU } from '../utils/tlru.js' +import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js' + +// Avoid sending multiple queries for the same hostname by caching results +const cache = new TLRU(1000) +// This TTL will be used if the remote service does not return one +const ttl = 60 * 1000 + +/** + * Uses the RFC 8427 'application/dns-json' content-type to resolve DNS queries. + * + * Supports and server that uses the same schema as Google's DNS over HTTPS + * resolver. + * + * This resolver needs fewer dependencies than the regular DNS-over-HTTPS + * resolver so can result in a smaller bundle size and consequently is preferred + * for browser use. + * + * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/ + * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers + * @see https://dnsprivacy.org/public_resolvers/ + * @see https://datatracker.ietf.org/doc/html/rfc8427 + */ +export function dnsJsonOverHttps (url: string): DNSResolver { + // browsers limit concurrent connections per host, + // we don't want preload calls to exhaust the limit (~6) + const httpQueue = new PQueue({ concurrency: 4 }) + + const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise => { + const searchParams = new URLSearchParams() + searchParams.set('name', fqdn) + searchParams.set('type', 'TXT') + + const query = searchParams.toString() + + // try cache first + if (options.nocache !== true && cache.has(query)) { + const response = cache.get(query) + + if (response != null) { + options.onProgress?.(new CustomProgressEvent('dnslink:cache', { detail: response })) + return response + } + } + + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: fqdn })) + + // query DNS-JSON over HTTPS server + const response = await httpQueue.add(async () => { + const res = await fetch(`${url}?${searchParams}`, { + headers: { + accept: 'application/dns-json' + }, + signal: options.signal + }) + + if (res.status !== 200) { + throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`) + } + + const query = new URL(res.url).search.slice(1) + const json: DNSResponse = await res.json() + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: json })) + + const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json) + + cache.set(query, ipfsPath, answer.TTL ?? ttl) + + return ipfsPath + }, { + signal: options.signal + }) + + if (response == null) { + throw new Error('No DNS response received') + } + + return response + } + + return async (domain: string, options: ResolveDnsLinkOptions = {}) => { + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options) + } +} diff --git a/packages/ipns/src/dns-resolvers/dns-over-https.ts b/packages/ipns/src/dns-resolvers/dns-over-https.ts new file mode 100644 index 00000000..b373f06f --- /dev/null +++ b/packages/ipns/src/dns-resolvers/dns-over-https.ts @@ -0,0 +1,146 @@ +/* eslint-env browser */ + +import { Buffer } from 'buffer' +import dnsPacket, { type DecodedPacket } from 'dns-packet' +import { base64url } from 'multiformats/bases/base64' +import PQueue from 'p-queue' +import { CustomProgressEvent } from 'progress-events' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js' +import { TLRU } from '../utils/tlru.js' +import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js' + +// Avoid sending multiple queries for the same hostname by caching results +const cache = new TLRU(1000) +// This TTL will be used if the remote service does not return one +const ttl = 60 * 1000 + +/** + * Uses the RFC 1035 'application/dns-message' content-type to resolve DNS + * queries. + * + * This resolver needs more dependencies than the non-standard + * DNS-JSON-over-HTTPS resolver so can result in a larger bundle size and + * consequently is not preferred for browser use. + * + * @see https://datatracker.ietf.org/doc/html/rfc1035 + * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-wireformat/ + * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers + * @see https://dnsprivacy.org/public_resolvers/ + */ +export function dnsOverHttps (url: string): DNSResolver { + // browsers limit concurrent connections per host, + // we don't want preload calls to exhaust the limit (~6) + const httpQueue = new PQueue({ concurrency: 4 }) + + const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise => { + const dnsQuery = dnsPacket.encode({ + type: 'query', + id: 0, + flags: dnsPacket.RECURSION_DESIRED, + questions: [{ + type: 'TXT', + name: fqdn + }] + }) + + const searchParams = new URLSearchParams() + searchParams.set('dns', base64url.encode(dnsQuery).substring(1)) + + const query = searchParams.toString() + + // try cache first + if (options.nocache !== true && cache.has(query)) { + const response = cache.get(query) + + if (response != null) { + options.onProgress?.(new CustomProgressEvent('dnslink:cache', { detail: response })) + return response + } + } + + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: fqdn })) + + // query DNS over HTTPS server + const response = await httpQueue.add(async () => { + const res = await fetch(`${url}?${searchParams}`, { + headers: { + accept: 'application/dns-message' + }, + signal: options.signal + }) + + if (res.status !== 200) { + throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`) + } + + const query = new URL(res.url).search.slice(1) + const buf = await res.arrayBuffer() + // map to expected response format + const json = toDNSResponse(dnsPacket.decode(Buffer.from(buf))) + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: json })) + + const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json) + + cache.set(query, ipfsPath, answer.TTL ?? ttl) + + return ipfsPath + }, { + signal: options.signal + }) + + if (response == null) { + throw new Error('No DNS response received') + } + + return response + } + + return async (domain: string, options: ResolveDnsLinkOptions = {}) => { + return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options) + } +} + +function toDNSResponse (response: DecodedPacket): DNSResponse { + const txtType = 16 + + return { + Status: 0, + TC: response.flag_tc ?? false, + RD: response.flag_rd ?? false, + RA: response.flag_ra ?? false, + AD: response.flag_ad ?? false, + CD: response.flag_cd ?? false, + Question: response.questions?.map(q => ({ + name: q.name, + type: txtType + })) ?? [], + Answer: response.answers?.map(a => { + if (a.type !== 'TXT' || a.data.length < 1) { + return { + name: a.name, + type: txtType, + TTL: 0, + data: 'invalid' + } + } + + if (!Buffer.isBuffer(a.data[0])) { + return { + name: a.name, + type: txtType, + TTL: a.ttl ?? ttl, + data: String(a.data[0]) + } + } + + return { + name: a.name, + type: txtType, + TTL: a.ttl ?? ttl, + data: uint8ArrayToString(a.data[0]) + } + }) ?? [] + } +} diff --git a/packages/ipns/src/dns-resolvers/index.ts b/packages/ipns/src/dns-resolvers/index.ts new file mode 100644 index 00000000..ee8c0db8 --- /dev/null +++ b/packages/ipns/src/dns-resolvers/index.ts @@ -0,0 +1,2 @@ +export { dnsOverHttps } from './dns-over-https.js' +export { dnsJsonOverHttps } from './dns-json-over-https.js' diff --git a/packages/ipns/src/dns-resolvers/resolver.browser.ts b/packages/ipns/src/dns-resolvers/resolver.browser.ts new file mode 100644 index 00000000..561672f7 --- /dev/null +++ b/packages/ipns/src/dns-resolvers/resolver.browser.ts @@ -0,0 +1,50 @@ +import Resolver from 'dns-over-http-resolver' +import PQueue from 'p-queue' +import { CustomProgressEvent } from 'progress-events' +import { resolveFn, type DNSResponse } from '../utils/dns.js' +import { TLRU } from '../utils/tlru.js' +import type { DNSResolver } from '../index.js' + +const cache = new TLRU(1000) +// We know browsers themselves cache DNS records for at least 1 minute, +// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426 +const ttl = 60 * 1000 + +// browsers limit concurrent connections per host, +// we don't want to exhaust the limit (~6) +const httpQueue = new PQueue({ concurrency: 4 }) + +const resolve: DNSResolver = async function resolve (domain, options = {}) { + const resolver = new Resolver({ maxCache: 0 }) + // try cache first + if (options.nocache !== true && cache.has(domain)) { + const response = cache.get(domain) + + if (response != null) { + options?.onProgress?.(new CustomProgressEvent('dnslink:cache', { detail: response })) + return response + } + } + + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: domain })) + + // Add the query to the queue + const response = await httpQueue.add(async () => { + const dnslinkRecord = await resolveFn(resolver, domain) + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: dnslinkRecord })) + cache.set(domain, dnslinkRecord, ttl) + + return dnslinkRecord + }, { + signal: options?.signal + }) + + if (response == null) { + throw new Error('No DNS response received') + } + + return response +} + +export default resolve diff --git a/packages/ipns/src/dns-resolvers/resolver.ts b/packages/ipns/src/dns-resolvers/resolver.ts new file mode 100644 index 00000000..9b63f864 --- /dev/null +++ b/packages/ipns/src/dns-resolvers/resolver.ts @@ -0,0 +1,25 @@ +import { Resolver } from 'dns/promises' +import { CustomProgressEvent } from 'progress-events' +import { resolveFn, type DNSResponse } from '../utils/dns.js' +import type { DNSResolver } from '../index.js' + +const resolve: DNSResolver = async function resolve (domain, options = {}) { + const resolver = new Resolver() + const listener = (): void => { + resolver.cancel() + } + + options.signal?.addEventListener('abort', listener) + + try { + options.onProgress?.(new CustomProgressEvent('dnslink:query', { detail: domain })) + const dnslinkRecord = await resolveFn(resolver, domain) + + options.onProgress?.(new CustomProgressEvent('dnslink:answer', { detail: dnslinkRecord })) + return dnslinkRecord + } finally { + options.signal?.removeEventListener('abort', listener) + } +} + +export default resolve diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index dedc2277..cb8ca507 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -5,16 +5,20 @@ * * @example * + * With {@link IPNSRouting} routers: + * * ```typescript * import { createHelia } from 'helia' * import { dht, pubsub } from '@helia/ipns/routing' * import { unixfs } from '@helia/unixfs' * * const helia = await createHelia() - * const name = ipns(helia, [ - * dht(helia), - * pubsub(helia) - * ]) + * const name = ipns(helia, { + * routers: [ + * dht(helia), + * pubsub(helia) + * ] + * }) * * // create a public key to publish as an IPNS name * const keyInfo = await helia.libp2p.keychain.createKey('my-key') @@ -33,13 +37,76 @@ * * @example * + * With default {@link DNSResolver} resolvers: + * + * ```typescript + * import { createHelia } from 'helia' + * import { dht, pubsub } from '@helia/ipns/routing' + * import { unixfs } from '@helia/unixfs' + * + * const helia = await createHelia() + * const name = ipns(helia, { + * resolvers: [ + * dnsOverHttps('https://private-dns-server.me/dns-query'), + * ] + * }) + * + * const cid = name.resolveDns('some-domain-with-dnslink-entry.com') + * ``` + * + * @example + * + * Calling `resolveDns` with the `@helia/ipns` instance: + * * ```typescript - * // resolve a CID from a TXT record in a DNS zone file, eg: - * // > dig ipfs.io TXT + * // resolve a CID from a TXT record in a DNS zone file, using the default + * // resolver for the current platform eg: + * // > dig _dnslink.ipfs.io TXT * // ;; ANSWER SECTION: - * // ipfs.io. 435 IN TXT "dnslink=/ipfs/Qmfoo" + * // _dnslink.ipfs.io. 60 IN TXT "dnslink=/ipns/website.ipfs.io" + * // > dig _dnslink.website.ipfs.io TXT + * // ;; ANSWER SECTION: + * // _dnslink.website.ipfs.io. 60 IN TXT "dnslink=/ipfs/QmWebsite" * * const cid = name.resolveDns('ipfs.io') + * + * console.info(cid) + * // QmWebsite + * ``` + * + * @example + * + * This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This + * uses binary DNS records so requires extra dependencies to process the + * response which can increase browser bundle sizes. + * + * If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. + * + * ```typescript + * // use DNS-Over-HTTPS + * import { dnsOverHttps } from '@helia/ipns/dns-resolvers' + * + * const cid = name.resolveDns('ipfs.io', { + * resolvers: [ + * dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + * ] + * }) + * ``` + * + * @example + * + * DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can + * result in a smaller browser bundle due to the response being plain JSON. + * + * ```typescript + * // use DNS-JSON-Over-HTTPS + * import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' + * + * const cid = name.resolveDns('ipfs.io', { + * resolvers: [ + * dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + * ] + * }) * ``` */ @@ -51,9 +118,10 @@ import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' import { CID } from 'multiformats/cid' import { CustomProgressEvent } from 'progress-events' +import { defaultResolver } from './dns-resolvers/default.js' import { localStore, type LocalStore } from './routing/local-store.js' -import { resolveDnslink } from './utils/resolve-dns-link.js' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' +import type { DNSResponse } from './utils/dns.js' import type { AbortOptions } from '@libp2p/interface' import type { PeerId } from '@libp2p/interface/peer-id' import type { Datastore } from 'interface-datastore' @@ -83,6 +151,11 @@ export type RepublishProgressEvents = ProgressEvent<'ipns:republish:success', IPNSRecord> | ProgressEvent<'ipns:republish:error', { record: IPNSRecord, err: Error }> +export type ResolveDnsLinkProgressEvents = + ProgressEvent<'dnslink:cache', string> | + ProgressEvent<'dnslink:query', string> | + ProgressEvent<'dnslink:answer', DNSResponse> + export interface PublishOptions extends AbortOptions, ProgressOptions { /** * Time duration of the record in ms (default: 24hrs) @@ -108,11 +181,35 @@ export interface ResolveOptions extends AbortOptions, ProgressOptions { + /** + * Do not use cached DNS entries (default: false) + */ + nocache?: boolean +} + +export interface DNSResolver { + (domain: string, options?: ResolveDnsLinkOptions): Promise +} + +export interface ResolveDNSOptions extends AbortOptions, ProgressOptions { + /** + * Do not query the network for the IPNS record (default: false) + */ + offline?: boolean + /** * Do not use cached DNS entries (default: false) */ nocache?: boolean + + /** + * These resolvers will be used to resolve the dnslink entries, if unspecified node will + * fall back to the `dns` module and browsers fall back to querying google/cloudflare DoH + * + * @see https://github.com/ipfs/helia-ipns/pull/55#discussion_r1270096881 + */ + resolvers?: DNSResolver[] } export interface RepublishOptions extends AbortOptions, ProgressOptions { @@ -157,10 +254,12 @@ class DefaultIPNS implements IPNS { private readonly routers: IPNSRouting[] private readonly localStore: LocalStore private timeout?: ReturnType + private readonly defaultResolvers: DNSResolver[] - constructor (components: IPNSComponents, routers: IPNSRouting[] = []) { + constructor (components: IPNSComponents, routers: IPNSRouting[] = [], resolvers: DNSResolver[] = []) { this.routers = routers this.localStore = localStore(components.datastore) + this.defaultResolvers = resolvers.length > 0 ? resolvers : [defaultResolver()] } async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { @@ -201,7 +300,11 @@ class DefaultIPNS implements IPNS { } async resolveDns (domain: string, options: ResolveDNSOptions = {}): Promise { - const dnslink = await resolveDnslink(domain, options) + const resolvers = options.resolvers ?? this.defaultResolvers + + const dnslink = await Promise.any( + resolvers.map(async resolver => resolver(domain, options)) + ) return this.#resolve(dnslink, options) } @@ -298,8 +401,13 @@ class DefaultIPNS implements IPNS { } } -export function ipns (components: IPNSComponents, routers: IPNSRouting[] = []): IPNS { - return new DefaultIPNS(components, routers) +export interface IPNSOptions { + routers?: IPNSRouting[] + resolvers?: DNSResolver[] +} + +export function ipns (components: IPNSComponents, { routers = [], resolvers = [] }: IPNSOptions): IPNS { + return new DefaultIPNS(components, routers, resolvers) } export { ipnsValidator } diff --git a/packages/ipns/src/utils/dns.ts b/packages/ipns/src/utils/dns.ts new file mode 100644 index 00000000..0590efcf --- /dev/null +++ b/packages/ipns/src/utils/dns.ts @@ -0,0 +1,126 @@ +import { CodeError } from '@libp2p/interface/errors' +import * as isIPFS from 'is-ipfs' +import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js' + +export interface Question { + name: string + type: number +} + +export interface Answer { + name: string + type: number + TTL: number + data: string +} + +export interface DNSResponse { + Status: number + TC: boolean + RD: boolean + RA: boolean + AD: boolean + CD: boolean + Question: Question[] + Answer?: Answer[] +} + +export const ipfsPathForAnswer = (answer: Answer): string => { + let data = answer.data + + if (data.startsWith('"')) { + data = data.substring(1) + } + + if (data.endsWith('"')) { + data = data.substring(0, data.length - 1) + } + + return data.replace('dnslink=', '') +} + +export const ipfsPathAndAnswer = (domain: string, response: DNSResponse): { ipfsPath: string, answer: Answer } => { + const answer = findDNSLinkAnswer(domain, response) + + return { + ipfsPath: ipfsPathForAnswer(answer), + answer + } +} + +export const findDNSLinkAnswer = (domain: string, response: DNSResponse): Answer => { + const answer = response.Answer?.filter(a => a.data.includes('dnslink=/ipfs') || a.data.includes('dnslink=/ipns')).pop() + + if (answer == null) { + throw new CodeError(`No dnslink records found for domain: ${domain}`, 'ERR_DNSLINK_NOT_FOUND') + } + + return answer +} + +export const MAX_RECURSIVE_DEPTH = 32 + +export const recursiveResolveDnslink = async (domain: string, depth: number, resolve: DNSResolver, options: ResolveDnsLinkOptions = {}): Promise => { + if (depth === 0) { + throw new Error('recursion limit exceeded') + } + + let dnslinkRecord: string + + try { + dnslinkRecord = await resolve(domain, options) + } catch (err: any) { + // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error + if (err.code !== 'ENOTFOUND' && err.code !== 'ERR_DNSLINK_NOT_FOUND' && err.code !== 'ENODATA') { + throw err + } + + if (domain.startsWith('_dnslink.')) { + // The supplied domain contains a _dnslink component + // Check the non-_dnslink domain + domain = domain.replace('_dnslink.', '') + } else { + // Check the _dnslink subdomain + domain = `_dnslink.${domain}` + } + + // If this throws then we propagate the error + dnslinkRecord = await resolve(domain, options) + } + + const result = dnslinkRecord.replace('dnslink=', '') + // result is now a `/ipfs/` or `/ipns/` string + const domainOrCID = result.split('/')[2] // e.g. ["", "ipfs", ""] + const isIPFSCID = isIPFS.cid(domainOrCID) + + // if the result is a CID, or depth is 1, we've reached the end of the recursion + // if depth is 1, another recursive call will be made, but it would throw. + // we could return if depth is 1 and allow users to handle, but that may be a breaking change + if (isIPFSCID) { + return result + } + + return recursiveResolveDnslink(domainOrCID, depth - 1, resolve, options) +} + +interface DnsResolver { + resolveTxt(domain: string): Promise +} + +const DNSLINK_REGEX = /^dnslink=.+$/ +export const resolveFn = async (resolver: DnsResolver, domain: string): Promise => { + const records = await resolver.resolveTxt(domain) + const dnslinkRecords = records.flat() + .filter(record => DNSLINK_REGEX.test(record)) + + // we now have dns text entries as an array of strings + // only records passing the DNSLINK_REGEX text are included + // TODO: support multiple dnslink records + const dnslinkRecord = dnslinkRecords[0] + + if (dnslinkRecord == null) { + throw new CodeError(`No dnslink records found for domain: ${domain}`, 'ERR_DNSLINK_NOT_FOUND') + } + + return dnslinkRecord +} diff --git a/packages/ipns/src/utils/resolve-dns-link.browser.ts b/packages/ipns/src/utils/resolve-dns-link.browser.ts deleted file mode 100644 index 3a6d7074..00000000 --- a/packages/ipns/src/utils/resolve-dns-link.browser.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* eslint-env browser */ - -import PQueue from 'p-queue' -import { TLRU } from './tlru.js' -import type { AbortOptions } from '@libp2p/interface' - -// Avoid sending multiple queries for the same hostname by caching results -const cache = new TLRU<{ Path: string, Message: string }>(1000) -// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884 -// However we know browsers themselves cache DNS records for at least 1 minute, -// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426 -const ttl = 60 * 1000 - -// browsers limit concurrent connections per host, -// we don't want preload calls to exhaust the limit (~6) -const httpQueue = new PQueue({ concurrency: 4 }) - -const ipfsPath = (response: { Path: string, Message: string }): string => { - if (response.Path != null) { - return response.Path - } - throw new Error(response.Message) -} - -export interface ResolveDnsLinkOptions extends AbortOptions { - nocache?: boolean -} - -export async function resolveDnslink (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise { // eslint-disable-line require-await - const resolve = async (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise => { - // @ts-expect-error - URLSearchParams does not take boolean options, only strings - const searchParams = new URLSearchParams(opts) - searchParams.set('arg', fqdn) - - // try cache first - const query = searchParams.toString() - if (opts.nocache !== true && cache.has(query)) { - const response = cache.get(query) - - if (response != null) { - return ipfsPath(response) - } - } - - // fallback to delegated DNS resolver - const response = await httpQueue.add(async () => { - // Delegated HTTP resolver sending DNSLink queries to ipfs.io - // TODO: replace hardcoded host with configurable DNS over HTTPS: https://github.com/ipfs/js-ipfs/issues/2212 - const res = await fetch(`https://ipfs.io/api/v0/dns?${searchParams}`) - const query = new URL(res.url).search.slice(1) - const json = await res.json() - cache.set(query, json, ttl) - - return json - }) - - return ipfsPath(response) - } - - return resolve(fqdn, opts) -} diff --git a/packages/ipns/src/utils/resolve-dns-link.ts b/packages/ipns/src/utils/resolve-dns-link.ts deleted file mode 100644 index 1975cf9f..00000000 --- a/packages/ipns/src/utils/resolve-dns-link.ts +++ /dev/null @@ -1,65 +0,0 @@ -import dns from 'dns' -import { promisify } from 'util' -import * as isIPFS from 'is-ipfs' -import type { AbortOptions } from '@libp2p/interface' - -const MAX_RECURSIVE_DEPTH = 32 - -export async function resolveDnslink (domain: string, options: AbortOptions = {}): Promise { - return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, options) -} - -async function recursiveResolveDnslink (domain: string, depth: number, options: AbortOptions = {}): Promise { - if (depth === 0) { - throw new Error('recursion limit exceeded') - } - - let dnslinkRecord - - try { - dnslinkRecord = await resolve(domain) - } catch (err: any) { - // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error - if (err.code !== 'ENOTFOUND' && err.code !== 'ERR_DNSLINK_NOT_FOUND' && err.code !== 'ENODATA') { - throw err - } - - if (domain.startsWith('_dnslink.')) { - // The supplied domain contains a _dnslink component - // Check the non-_dnslink domain - dnslinkRecord = await resolve(domain.replace('_dnslink.', '')) - } else { - // Check the _dnslink subdomain - const _dnslinkDomain = `_dnslink.${domain}` - // If this throws then we propagate the error - dnslinkRecord = await resolve(_dnslinkDomain) - } - } - - const result = dnslinkRecord.replace('dnslink=', '') - const domainOrCID = result.split('/')[2] - const isIPFSCID = isIPFS.cid(domainOrCID) - - if (isIPFSCID || depth === 0) { - return result - } - - return recursiveResolveDnslink(domainOrCID, depth - 1, options) -} - -async function resolve (domain: string, options: AbortOptions = {}): Promise { - const DNSLINK_REGEX = /^dnslink=.+$/ - const records = await promisify(dns.resolveTxt)(domain) - const dnslinkRecords = records.reduce((rs, r) => rs.concat(r), []) - .filter(record => DNSLINK_REGEX.test(record)) - - const dnslinkRecord = dnslinkRecords[0] - - // we now have dns text entries as an array of strings - // only records passing the DNSLINK_REGEX text are included - if (dnslinkRecord == null) { - throw new Error(`No dnslink records found for domain: ${domain}`) - } - - return dnslinkRecord -} diff --git a/packages/ipns/test/dns-resolvers.spec.ts b/packages/ipns/test/dns-resolvers.spec.ts new file mode 100644 index 00000000..6913933b --- /dev/null +++ b/packages/ipns/test/dns-resolvers.spec.ts @@ -0,0 +1,86 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { defaultResolver } from '../src/dns-resolvers/default.js' +import { dnsJsonOverHttps } from '../src/dns-resolvers/dns-json-over-https.js' +import { dnsOverHttps } from '../src/dns-resolvers/dns-over-https.js' +import type { DNSResolver } from '../src/index.js' + +const resolvers: Record = { + 'dns-json-over-https': dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), + 'dns-over-https': dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), + default: defaultResolver() +} + +describe('dns resolvers', () => { + Object.entries(resolvers).forEach(([name, resolver]) => { + it(`${name} should resolve`, async () => { + const result = await resolver('ipfs.io') + + expect(result).to.startWith('/ipfs') + }) + + it(`${name} should cache results`, async function () { + if (name === 'default' && globalThis.Buffer != null) { + // node dns uses OS-level caching + this.skip() + } + + let usedCache = false + + // resolve once + await resolver('ipfs.io') + + const makeCall = async (): Promise => { + await resolver('ipfs.io', { + onProgress: (evt) => { + if (evt.type.includes('dnslink:cache')) { + usedCache = true + } + } + }) + } + + for (let i = 0; i < 15; i++) { + // resolve again, should use the cache + // TTL can be pretty low which means this can be flaky if executed more slowly than the TTL + await makeCall() + if (usedCache) { + break + } + } + + expect(usedCache).to.be.true() + }) + + it(`${name} should skip cache results`, async function () { + if (name === 'default' && globalThis.Buffer != null) { + // node dns uses OS-level caching + this.skip() + } + + let usedCache = false + + // resolve once + await resolver('ipfs.io') + + // resolve again, should skip the cache + await resolver('ipfs.io', { + nocache: true, + onProgress: (evt) => { + if (evt.type.includes('dnslink:cache')) { + usedCache = true + } + } + }) + + expect(usedCache).to.be.false() + }) + + it(`${name} should abort if signal is aborted`, async () => { + const signal = AbortSignal.timeout(1) + + await expect(resolver('ipfs.io', { nocache: true, signal })).to.eventually.be.rejected() + }) + }) +}) diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index 184e835b..82c966d6 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -20,7 +20,7 @@ describe('publish', () => { routing = stubInterface() routing.get.throws(new Error('Not found')) - name = ipns({ datastore }, [routing]) + name = ipns({ datastore }, { routers: [routing] }) }) it('should publish an IPNS record with the default params', async function () { diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index e605da32..cd02d830 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -25,7 +25,7 @@ describe('resolve', () => { routing = stubInterface() routing.get.throws(new Error('Not found')) - name = ipns({ datastore }, [routing]) + name = ipns({ datastore }, { routers: [routing] }) }) it('should resolve a record', async () => { @@ -75,23 +75,6 @@ describe('resolve', () => { expect(resolvedValue.toString()).to.equal(cid.toV1().toString()) }) - it('should resolve /ipns/tableflip.io', async function () { - const domain = 'tableflip.io' - - try { - const resolvedValue = await name.resolveDns(domain) - - expect(resolvedValue).to.be.an.instanceOf(CID) - } catch (err: any) { - // happens when running tests offline - if (err.message.includes(`ECONNREFUSED ${domain}`) === true) { - return this.skip() - } - - throw err - } - }) - it('should emit progress events', async function () { const onProgress = Sinon.stub() const key = await createEd25519PeerId() diff --git a/packages/ipns/test/resolveDns.spec.ts b/packages/ipns/test/resolveDns.spec.ts new file mode 100644 index 00000000..a2288d90 --- /dev/null +++ b/packages/ipns/test/resolveDns.spec.ts @@ -0,0 +1,41 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import { type Datastore } from 'interface-datastore' +import { stub } from 'sinon' +import { type StubbedInstance, stubInterface } from 'sinon-ts' +import { type IPNSRouting, ipns } from '../src/index.js' + +describe('resolveDns', () => { + let routing: StubbedInstance + let datastore: Datastore + + beforeEach(async () => { + datastore = new MemoryDatastore() + routing = stubInterface() + routing.get.throws(new Error('Not found')) + }) + + it('should use resolvers passed in constructor', async () => { + const stubbedResolver1 = stub().returns('dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + + const name = ipns({ datastore }, { routers: [routing], resolvers: [stubbedResolver1] }) + const result = await name.resolveDns('foobar.baz', { nocache: true, offline: true }) + expect(stubbedResolver1.called).to.be.true() + expect(stubbedResolver1.calledWith('foobar.baz')).to.be.true() + expect(result.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + }) + + it('should allow overriding of resolvers passed in constructor', async () => { + const stubbedResolver1 = stub().returns('dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const stubbedResolver2 = stub().returns('dnslink=/ipfs/bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq') + + const name = ipns({ datastore }, { routers: [routing], resolvers: [stubbedResolver1] }) + const result = await name.resolveDns('foobar.baz', { nocache: true, offline: true, resolvers: [stubbedResolver2] }) + expect(stubbedResolver1.called).to.be.false() + expect(stubbedResolver2.called).to.be.true() + expect(stubbedResolver2.calledWith('foobar.baz')).to.be.true() + expect(result.toString()).to.equal('bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq') + }) +}) From ce35431cd5afea0f96efe003a8ba92a39d9a21fa Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 5 Dec 2023 23:25:28 +0000 Subject: [PATCH 218/253] chore(release): 3.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/ipns-v3.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.3...@helia/ipns-v3.0.0) (2023-12-05) ### ⚠ BREAKING CHANGES * alters the options object passed to the `ipns` factory function ### Features * support DNS over HTTPS and DNS-JSON over HTTPS ([#55](https://github.com/ipfs/helia-ipns/issues/55)) ([2ac0e8b](https://github.com/ipfs/helia-ipns/commit/2ac0e8b26556b73961e67191c564ac2b18d32b31)) --- packages/ipns/CHANGELOG.md | 11 +++++++++++ packages/ipns/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index bcd19099..909489aa 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/ipns-v3.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.3...@helia/ipns-v3.0.0) (2023-12-05) + + +### ⚠ BREAKING CHANGES + +* alters the options object passed to the `ipns` factory function + +### Features + +* support DNS over HTTPS and DNS-JSON over HTTPS ([#55](https://github.com/ipfs/helia-ipns/issues/55)) ([2ac0e8b](https://github.com/ipfs/helia-ipns/commit/2ac0e8b26556b73961e67191c564ac2b18d32b31)) + ## [@helia/ipns-v2.0.3](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.2...@helia/ipns-v2.0.3) (2023-10-26) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 38b87b6c..ddf47a1d 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "2.0.3", + "version": "3.0.0", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From d0d84f07db9338ccc8245167929bd71f4cb8b238 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Tue, 5 Dec 2023 23:25:39 +0000 Subject: [PATCH 219/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- packages/ipns/README.md | 110 ++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index a224d226..50f46d9f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,7 +53,7 @@ "@chainsafe/libp2p-noise": "^13.0.0", "@chainsafe/libp2p-yamux": "^5.0.0", "@helia/interface": "^2.0.0", - "@helia/ipns": "^2.0.0", + "@helia/ipns": "^3.0.0", "@libp2p/interface": "^0.1.2", "@libp2p/kad-dht": "^10.0.4", "@libp2p/peer-id": "^3.0.2", diff --git a/packages/ipns/README.md b/packages/ipns/README.md index cdca30ff..d5ad8aaf 100644 --- a/packages/ipns/README.md +++ b/packages/ipns/README.md @@ -13,6 +13,116 @@ > An implementation of IPNS for Helia +# About + +IPNS operations using a Helia node + +## Example + +With IPNSRouting routers: + +```typescript +import { createHelia } from 'helia' +import { dht, pubsub } from '@helia/ipns/routing' +import { unixfs } from '@helia/unixfs' + +const helia = await createHelia() +const name = ipns(helia, { + routers: [ + dht(helia), + pubsub(helia) + ] +}) + +// create a public key to publish as an IPNS name +const keyInfo = await helia.libp2p.keychain.createKey('my-key') +const peerId = await helia.libp2p.keychain.exportPeerId(keyInfo.name) + +// store some data to publish +const fs = unixfs(helia) +const cid = await fs.add(Uint8Array.from([0, 1, 2, 3, 4])) + +// publish the name +await name.publish(peerId, cid) + +// resolve the name +const cid = name.resolve(peerId) +``` + +## Example + +With default DNSResolver resolvers: + +```typescript +import { createHelia } from 'helia' +import { dht, pubsub } from '@helia/ipns/routing' +import { unixfs } from '@helia/unixfs' + +const helia = await createHelia() +const name = ipns(helia, { + resolvers: [ + dnsOverHttps('https://private-dns-server.me/dns-query'), + ] +}) + +const cid = name.resolveDns('some-domain-with-dnslink-entry.com') +``` + +## Example + +Calling `resolveDns` with the `@helia/ipns` instance: + +```typescript +// resolve a CID from a TXT record in a DNS zone file, using the default +// resolver for the current platform eg: +// > dig _dnslink.ipfs.io TXT +// ;; ANSWER SECTION: +// _dnslink.ipfs.io. 60 IN TXT "dnslink=/ipns/website.ipfs.io" +// > dig _dnslink.website.ipfs.io TXT +// ;; ANSWER SECTION: +// _dnslink.website.ipfs.io. 60 IN TXT "dnslink=/ipfs/QmWebsite" + +const cid = name.resolveDns('ipfs.io') + +console.info(cid) +// QmWebsite +``` + +## Example + +This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This +uses binary DNS records so requires extra dependencies to process the +response which can increase browser bundle sizes. + +If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. + +```typescript +// use DNS-Over-HTTPS +import { dnsOverHttps } from '@helia/ipns/dns-resolvers' + +const cid = name.resolveDns('ipfs.io', { + resolvers: [ + dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + ] +}) +``` + +## Example + +DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can +result in a smaller browser bundle due to the response being plain JSON. + +```typescript +// use DNS-JSON-Over-HTTPS +import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' + +const cid = name.resolveDns('ipfs.io', { + resolvers: [ + dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + ] +}) +``` + ## Table of contents - [Install](#install) From f66dd71f18dab57471749e6a708917ea291d05e1 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 6 Dec 2023 11:38:52 -0800 Subject: [PATCH 220/253] chore: fix docs and aegir dep in subpackages (#142) --- packages/interop/package.json | 1 + packages/ipns/CHANGELOG.md | 35 +++++++++++++++++++ packages/ipns/package.json | 1 + packages/ipns/src/index.ts | 4 ++- ...resolveDns.spec.ts => resolve-dns.spec.ts} | 0 5 files changed, 40 insertions(+), 1 deletion(-) rename packages/ipns/test/{resolveDns.spec.ts => resolve-dns.spec.ts} (100%) diff --git a/packages/interop/package.json b/packages/interop/package.json index 50f46d9f..f27fc5e1 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -60,6 +60,7 @@ "@libp2p/peer-id-factory": "^3.0.3", "@libp2p/tcp": "^8.0.4", "@libp2p/websockets": "^7.0.4", + "aegir": "^41.1.14", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", "helia": "^2.0.1", diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 909489aa..8d5f61f3 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -5,6 +5,41 @@ * alters the options object passed to the `ipns` factory function +#### Before + +```typescript +import { createHelia } from 'helia' +import { ipns } from '@helia/ipns' +import { dht, pubsub } from '@helia/ipns/routing' +import { unixfs } from '@helia/unixfs' + +const helia = await createHelia() +const name = ipns(helia, [ + dht(helia), + pubsub(helia) +]) +``` + +#### After + +```typescript +import { createHelia } from 'helia' +import { ipns } from '@helia/ipns' +import { dnsOverHttps } from '@helia/ipns/dns-resolvers' +import { unixfs } from '@helia/unixfs' + +const helia = await createHelia() +const name = ipns(helia, { + routers: [ + dht(helia), + pubsub(helia) + ], + resolvers: [ + dnsOverHttps('https://private-dns-server.me/dns-query'), + ] +}) +``` + ### Features * support DNS over HTTPS and DNS-JSON over HTTPS ([#55](https://github.com/ipfs/helia-ipns/issues/55)) ([2ac0e8b](https://github.com/ipfs/helia-ipns/commit/2ac0e8b26556b73961e67191c564ac2b18d32b31)) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index ddf47a1d..e5353661 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -177,6 +177,7 @@ "@libp2p/interface": "^0.1.4", "@libp2p/peer-id-factory": "^3.0.3", "@types/dns-packet": "^5.6.4", + "aegir": "^41.1.14", "datastore-core": "^9.0.3", "sinon": "^17.0.0", "sinon-ts": "^1.0.0" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index cb8ca507..93ce0cab 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -9,6 +9,7 @@ * * ```typescript * import { createHelia } from 'helia' + * import { ipns } from '@helia/ipns' * import { dht, pubsub } from '@helia/ipns/routing' * import { unixfs } from '@helia/unixfs' * @@ -41,8 +42,9 @@ * * ```typescript * import { createHelia } from 'helia' - * import { dht, pubsub } from '@helia/ipns/routing' + * import { ipns } from '@helia/ipns' * import { unixfs } from '@helia/unixfs' + * import { dnsOverHttps } from '@helia/ipns/dns-resolvers' * * const helia = await createHelia() * const name = ipns(helia, { diff --git a/packages/ipns/test/resolveDns.spec.ts b/packages/ipns/test/resolve-dns.spec.ts similarity index 100% rename from packages/ipns/test/resolveDns.spec.ts rename to packages/ipns/test/resolve-dns.spec.ts From 6ab5ddcecb377bf61b6a4566292249fd5dc3d2c7 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Wed, 6 Dec 2023 19:44:46 +0000 Subject: [PATCH 221/253] chore: update sibling dependencies --- packages/ipns/README.md | 49 +++-------------------------------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/packages/ipns/README.md b/packages/ipns/README.md index d5ad8aaf..025a5fa2 100644 --- a/packages/ipns/README.md +++ b/packages/ipns/README.md @@ -23,6 +23,7 @@ With IPNSRouting routers: ```typescript import { createHelia } from 'helia' +import { ipns } from '@helia/ipns' import { dht, pubsub } from '@helia/ipns/routing' import { unixfs } from '@helia/unixfs' @@ -55,8 +56,9 @@ With default DNSResolver resolvers: ```typescript import { createHelia } from 'helia' -import { dht, pubsub } from '@helia/ipns/routing' +import { ipns } from '@helia/ipns' import { unixfs } from '@helia/unixfs' +import { dnsOverHttps } from '@helia/ipns/dns-resolvers' const helia = await createHelia() const name = ipns(helia, { @@ -122,48 +124,3 @@ const cid = name.resolveDns('ipfs.io', { ] }) ``` - -## Table of contents - -- [Install](#install) - - [Browser ` -``` - -## API Docs - -- - -## License - -Licensed under either of - -- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) -- MIT ([LICENSE-MIT](LICENSE-MIT) / ) - -## Contribute - -Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). - -Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. - -Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). - -Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. - -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) From d732db9f4fea23aa11456d451f02d4f143846ba3 Mon Sep 17 00:00:00 2001 From: Flamenco Date: Thu, 7 Dec 2023 19:46:31 -0500 Subject: [PATCH 222/253] docs: fix typo (#113) Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- packages/ipns/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 93ce0cab..bed58b4c 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -225,7 +225,7 @@ export interface IPNS { /** * Creates an IPNS record signed by the passed PeerId that will resolve to the passed value * - * If the valid is a PeerId, a recursive IPNS record will be created. + * If the value is a PeerId, a recursive IPNS record will be created. */ publish(key: PeerId, value: CID | PeerId, options?: PublishOptions): Promise From ae6354bda124ea7665e565ef6360790e592f187e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 8 Dec 2023 00:52:03 +0000 Subject: [PATCH 223/253] chore(release): 3.0.1 [skip ci] ## [@helia/ipns-v3.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v3.0.0...@helia/ipns-v3.0.1) (2023-12-08) ### Trivial Changes * fix docs and aegir dep in subpackages ([#142](https://github.com/ipfs/helia-ipns/issues/142)) ([f66dd71](https://github.com/ipfs/helia-ipns/commit/f66dd71f18dab57471749e6a708917ea291d05e1)) * update sibling dependencies ([6ab5ddc](https://github.com/ipfs/helia-ipns/commit/6ab5ddcecb377bf61b6a4566292249fd5dc3d2c7)) * update sibling dependencies ([d0d84f0](https://github.com/ipfs/helia-ipns/commit/d0d84f07db9338ccc8245167929bd71f4cb8b238)) ### Documentation * fix typo ([#113](https://github.com/ipfs/helia-ipns/issues/113)) ([d732db9](https://github.com/ipfs/helia-ipns/commit/d732db9f4fea23aa11456d451f02d4f143846ba3)) --- packages/ipns/CHANGELOG.md | 14 ++++++++++++++ packages/ipns/package.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index 8d5f61f3..ab5ae038 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,17 @@ +## [@helia/ipns-v3.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v3.0.0...@helia/ipns-v3.0.1) (2023-12-08) + + +### Trivial Changes + +* fix docs and aegir dep in subpackages ([#142](https://github.com/ipfs/helia-ipns/issues/142)) ([f66dd71](https://github.com/ipfs/helia-ipns/commit/f66dd71f18dab57471749e6a708917ea291d05e1)) +* update sibling dependencies ([6ab5ddc](https://github.com/ipfs/helia-ipns/commit/6ab5ddcecb377bf61b6a4566292249fd5dc3d2c7)) +* update sibling dependencies ([d0d84f0](https://github.com/ipfs/helia-ipns/commit/d0d84f07db9338ccc8245167929bd71f4cb8b238)) + + +### Documentation + +* fix typo ([#113](https://github.com/ipfs/helia-ipns/issues/113)) ([d732db9](https://github.com/ipfs/helia-ipns/commit/d732db9f4fea23aa11456d451f02d4f143846ba3)) + ## [@helia/ipns-v3.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v2.0.3...@helia/ipns-v3.0.0) (2023-12-05) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index e5353661..80aadf25 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "3.0.0", + "version": "3.0.1", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", From 546ecf023bd619d32e187fa6a55d39fcf12e4bbe Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 29 Dec 2023 21:51:48 +0100 Subject: [PATCH 224/253] fix: make @libp2p/interface a dependency (#159) This should be a dep not a dev dep. Refs #158 --- packages/ipns/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 80aadf25..9eceaad1 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -159,6 +159,7 @@ "release": "aegir release" }, "dependencies": { + "@libp2p/interface": "^0.1.4", "@libp2p/kad-dht": "^10.0.11", "@libp2p/logger": "^3.0.2", "@libp2p/peer-id": "^3.0.2", @@ -174,7 +175,6 @@ "uint8arrays": "^4.0.3" }, "devDependencies": { - "@libp2p/interface": "^0.1.4", "@libp2p/peer-id-factory": "^3.0.3", "@types/dns-packet": "^5.6.4", "aegir": "^41.1.14", From 001247c6fc38ff3d810736371de901e5e1099f26 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 08:46:26 +0100 Subject: [PATCH 225/253] feat!: update helia to v3 and multiformats to v13 (#147) Updates all deps and fixes linting errors. BREAKING CHANGE: uses multiformats v13 and helia v3 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 2 ++ .github/workflows/semantic-pull-request.yml | 12 ++++++++ README.md | 17 ++++------- package.json | 4 +++ packages/interop/README.md | 22 ++++----------- packages/interop/package.json | 28 +++++++++++-------- packages/interop/test/bitswap.spec.ts | 6 ++-- packages/interop/test/files.spec.ts | 7 +++-- .../test/fixtures/create-helia.browser.ts | 10 ++++--- .../interop/test/fixtures/create-helia.ts | 10 ++++--- packages/unixfs/README.md | 26 ++++++----------- packages/unixfs/package.json | 22 +++++++++------ packages/unixfs/src/index.ts | 6 ++-- 14 files changed, 91 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 35d87d10..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -9,7 +9,9 @@ on: permissions: contents: write + id-token: write packages: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/README.md b/README.md index 871658ff..a7d9d1c6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-# @helia/unixfs +# @helia/unixfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,30 +13,23 @@ > An implementation of Unixfs for Helia -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/interop`](./packages/interop) Interop tests for @helia/unixfs - [`/packages/unixfs`](./packages/unixfs) A Helia-compatible wrapper for UnixFS -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues). diff --git a/package.json b/package.json index 6960c4c8..5979579b 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-unixfs/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], diff --git a/packages/interop/README.md b/packages/interop/README.md index bb31762f..32d069a1 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/unixfs-interop +# @helia/unixfs-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,13 @@ > Interop tests for @helia/unixfs -## Table of contents - -- [Install](#install) - - [Browser ` ``` -## API Docs - -- - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index c75c60f4..db25629b 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/unixfs", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-unixfs/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-unixfs.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-unixfs/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -48,25 +52,27 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^13.0.1", - "@chainsafe/libp2p-yamux": "^5.0.0", - "@helia/interface": "^2.0.0", + "@chainsafe/libp2p-noise": "^14.1.0", + "@chainsafe/libp2p-yamux": "^6.0.1", + "@helia/interface": "^3.0.0", "@helia/unixfs": "^1.0.0", - "@libp2p/tcp": "^8.0.6", - "@libp2p/websockets": "^7.0.6", - "aegir": "^41.0.0", + "@libp2p/identify": "^1.0.9", + "@libp2p/interface": "^1.1.1", + "@libp2p/tcp": "^9.0.7", + "@libp2p/websockets": "^8.0.7", + "aegir": "^42.0.1", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "kubo": "^0.24.0", - "helia": "^2.0.1", + "helia": "^3.0.0", "ipfs-core-types": "^0.14.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^4.0.1", + "kubo": "^0.25.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.46.10", + "libp2p": "^1.0.10", "merge-options": "^3.0.4", - "multiformats": "^12.1.1", + "multiformats": "^13.0.0", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/test/bitswap.spec.ts b/packages/interop/test/bitswap.spec.ts index 9104246b..7990e7de 100644 --- a/packages/interop/test/bitswap.spec.ts +++ b/packages/interop/test/bitswap.spec.ts @@ -3,14 +3,16 @@ import { type UnixFS, unixfs } from '@helia/unixfs' import { expect } from 'aegir/chai' import toBuffer from 'it-to-buffer' +import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' describe('unixfs bitswap interop', () => { - let helia: Helia + let helia: Helia let unixFs: UnixFS let kubo: Controller @@ -74,7 +76,7 @@ describe('unixfs bitswap interop', () => { const { cid } = await kubo.api.add(candidate.content) - const bytes = await toBuffer(unixFs.cat(cid)) + const bytes = await toBuffer(unixFs.cat(CID.parse(cid.toString()))) expect(bytes).to.equalBytes(toBuffer(input)) }) diff --git a/packages/interop/test/files.spec.ts b/packages/interop/test/files.spec.ts index 4988f3ac..db329bf4 100644 --- a/packages/interop/test/files.spec.ts +++ b/packages/interop/test/files.spec.ts @@ -4,16 +4,17 @@ import { type AddOptions, type UnixFS, unixfs } from '@helia/unixfs' import { expect } from 'aegir/chai' import { fixedSize } from 'ipfs-unixfs-importer/chunker' import { balanced } from 'ipfs-unixfs-importer/layout' +import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' -import type { CID } from 'multiformats/cid' describe('unixfs interop', () => { - let helia: Helia + let helia: Helia let unixFs: UnixFS let kubo: Controller @@ -26,7 +27,7 @@ describe('unixfs interop', () => { async function importToKubo (data: FileCandidate, opts?: KuboAddOptions): Promise { const result = await kubo.api.add(data.content, opts) - return result.cid + return CID.parse(result.cid.toString()) } async function expectSameCid (data: () => FileCandidate, heliaOpts: Partial = {}, kuboOpts: KuboAddOptions = {}): Promise { diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index f94bc9df..0dce0519 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,16 +1,18 @@ import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { identify } from '@libp2p/identify' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { bitswap } from 'helia/block-brokers' -import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' +import { createLibp2p } from 'libp2p' import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' +import type { Libp2pOptions } from 'libp2p' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise> { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -28,7 +30,7 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise> { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() @@ -25,7 +27,7 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise

-# @helia/unixfs +# @helia/unixfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -19,7 +19,7 @@ See the interface for all available operations. -## Example +## Example - Creating files and directories ```typescript import { createHelia } from 'helia' @@ -44,9 +44,9 @@ for await (const entry of fs.addAll([{ } ``` -## Example +## Example - Recursively adding a directory -Recursively adding a directory (Node.js-compatibly environments only): +Node.js-compatibly environments only: ```typescript import { globSource } from '@helia/unixfs' @@ -56,21 +56,13 @@ for await (const entry of fs.addAll(globSource('path/to/containing/dir', 'glob-p } ``` -## Table of contents - -- [Install](#install) - - [Browser ` ``` -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-unixfs/issues). diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 23f09931..1bda316c 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -3,7 +3,7 @@ "version": "1.4.3", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-unixfs/tree/master/packages/unixfs#readme", + "homepage": "https://github.com/ipfs/helia-unixfs/tree/main/packages/unixfs#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-unixfs.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-unixfs/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -155,13 +159,12 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@ipld/dag-pb": "^4.0.0", - "@libp2p/interface": "^0.1.2", - "@libp2p/logger": "^3.0.2", + "@libp2p/interface": "^1.0.2", + "@libp2p/logger": "^4.0.2", "@multiformats/murmur3": "^2.1.2", "hamt-sharding": "^3.0.2", - "interface-blockstore": "^5.0.0", "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^13.1.0", "ipfs-unixfs-importer": "^15.1.0", @@ -169,20 +172,21 @@ "it-last": "^3.0.1", "it-pipe": "^3.0.1", "merge-options": "^3.0.4", - "multiformats": "^12.1.1", + "multiformats": "^13.0.0", "progress-events": "^1.0.0", - "sparse-array": "^1.3.2" + "sparse-array": "^1.3.2", + "uint8arrays": "^5.0.1" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "blockstore-core": "^4.0.1", "delay": "^6.0.0", + "interface-blockstore": "^5.2.9", "iso-url": "^1.2.1", "it-all": "^3.0.1", "it-drain": "^3.0.1", "it-first": "^3.0.1", "it-to-buffer": "^4.0.1", - "uint8arrays": "^4.0.3", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/unixfs/src/index.ts b/packages/unixfs/src/index.ts index 6019dff9..57eb0f6b 100644 --- a/packages/unixfs/src/index.ts +++ b/packages/unixfs/src/index.ts @@ -5,7 +5,7 @@ * * See the {@link UnixFS UnixFS interface} for all available operations. * - * @example + * @example Creating files and directories * * ```typescript * import { createHelia } from 'helia' @@ -30,9 +30,9 @@ * } * ``` * - * @example + * @example Recursively adding a directory * - * Recursively adding a directory (Node.js-compatibly environments only): + * Node.js-compatibly environments only: * * ```typescript * import { globSource } from '@helia/unixfs' From afc27d46c41612d04aa87c8201ae85536891d5f1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 07:54:05 +0000 Subject: [PATCH 226/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/unixfs-v2.0.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.3...@helia/unixfs-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3 ### Features * update helia to v3 and multiformats to v13 ([#147](https://github.com/ipfs/helia-unixfs/issues/147)) ([001247c](https://github.com/ipfs/helia-unixfs/commit/001247c6fc38ff3d810736371de901e5e1099f26)) ### Trivial Changes * update sibling dependencies ([1b0b2ef](https://github.com/ipfs/helia-unixfs/commit/1b0b2ef05c5cbd78c3b5d5629237200a69bbd5dd)) --- packages/unixfs/CHANGELOG.md | 16 ++++++++++++++++ packages/unixfs/package.json | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/unixfs/CHANGELOG.md b/packages/unixfs/CHANGELOG.md index dac40a95..15ac02e7 100644 --- a/packages/unixfs/CHANGELOG.md +++ b/packages/unixfs/CHANGELOG.md @@ -1,3 +1,19 @@ +## [@helia/unixfs-v2.0.0](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.3...@helia/unixfs-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3 + +### Features + +* update helia to v3 and multiformats to v13 ([#147](https://github.com/ipfs/helia-unixfs/issues/147)) ([001247c](https://github.com/ipfs/helia-unixfs/commit/001247c6fc38ff3d810736371de901e5e1099f26)) + + +### Trivial Changes + +* update sibling dependencies ([1b0b2ef](https://github.com/ipfs/helia-unixfs/commit/1b0b2ef05c5cbd78c3b5d5629237200a69bbd5dd)) + ## [@helia/unixfs-v1.4.3](https://github.com/ipfs/helia-unixfs/compare/@helia/unixfs-v1.4.2...@helia/unixfs-v1.4.3) (2023-12-03) diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 1bda316c..9cef82a1 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/unixfs", - "version": "1.4.3", + "version": "2.0.0", "description": "A Helia-compatible wrapper for UnixFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-unixfs/tree/main/packages/unixfs#readme", From 12f4c75988cb57d9db5a04657c0712cc84e93bd7 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 07:54:16 +0000 Subject: [PATCH 227/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index db25629b..e58fd2d7 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -55,7 +55,7 @@ "@chainsafe/libp2p-noise": "^14.1.0", "@chainsafe/libp2p-yamux": "^6.0.1", "@helia/interface": "^3.0.0", - "@helia/unixfs": "^1.0.0", + "@helia/unixfs": "^2.0.0", "@libp2p/identify": "^1.0.9", "@libp2p/interface": "^1.1.1", "@libp2p/tcp": "^9.0.7", From 9f7dc0a0581524531501fc062fefb6ba26d99c02 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 8 Jan 2024 09:00:25 +0100 Subject: [PATCH 228/253] feat!: update helia to v3 and multiformats to v13 Updates all deps and fixes linitng errors. BREAKING CHANGE: uses multiformats v13 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 2 + .github/workflows/semantic-pull-request.yml | 12 ++++++ README.md | 17 +++----- package.json | 6 ++- packages/interop/README.md | 11 ++--- packages/interop/package.json | 12 ++++-- packages/interop/typedoc.json | 5 --- packages/mfs/README.md | 47 +++++++++++++++------ packages/mfs/package.json | 24 ++++++----- 10 files changed, 84 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml delete mode 100644 packages/interop/typedoc.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 35d87d10..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -9,7 +9,9 @@ on: permissions: contents: write + id-token: write packages: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/README.md b/README.md index b24df40e..681daf4d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-# @helia/mfs +# @helia/mfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,30 +13,23 @@ > A mutable filesystem powered by Helia -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/interop`](./packages/interop) Interop tests for @helia/mfs - [`/packages/mfs`](./packages/mfs) A mutable filesystem powered by Helia -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-mfs/issues). diff --git a/package.json b/package.json index ee3e238f..4369065e 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-mfs/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/interop/README.md b/packages/interop/README.md index 644f58c5..b81c629a 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/mfs-interop +# @helia/mfs-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,19 +13,14 @@ > Interop tests for @helia/mfs -## Table of contents - -- [License](#license) -- [Contribute](#contribute) - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-mfs/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index e7047557..73aceb0b 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/mfs", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-mfs/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-mfs.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-mfs/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -49,11 +53,11 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@helia/mfs": "^1.0.0", - "aegir": "^41.0.3", + "aegir": "^42.0.1", "go-ipfs": "^0.22.0", - "helia": "^2.0.3", + "helia": "^3.0.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", diff --git a/packages/interop/typedoc.json b/packages/interop/typedoc.json deleted file mode 100644 index f599dc72..00000000 --- a/packages/interop/typedoc.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "entryPoints": [ - "./src/index.ts" - ] -} diff --git a/packages/mfs/README.md b/packages/mfs/README.md index f5c0318f..59a7d4e4 100644 --- a/packages/mfs/README.md +++ b/packages/mfs/README.md @@ -4,7 +4,7 @@

-# @helia/mfs +# @helia/mfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,42 @@ > A mutable filesystem powered by Helia -## Table of contents +# About -- [Install](#install) - - [Browser ` ``` -## API Docs +# API Docs -- +- -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-mfs/issues). diff --git a/packages/mfs/package.json b/packages/mfs/package.json index 00bf6715..d5c2757e 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -3,7 +3,7 @@ "version": "1.0.2", "description": "A mutable filesystem powered by Helia", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-mfs/tree/master/packages/mfs#readme", + "homepage": "https://github.com/ipfs/helia-mfs/tree/main/packages/mfs#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-mfs.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-mfs/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -135,28 +139,28 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", - "@helia/unixfs": "^1.2.2", - "@ipld/dag-pb": "^4.0.0", + "@helia/interface": "^3.0.0", + "@helia/unixfs": "^2.0.0", "@libp2p/interfaces": "^3.3.1", - "@libp2p/logger": "^3.0.1", - "interface-blockstore": "^5.0.0", + "@libp2p/logger": "^4.0.4", "interface-datastore": "^8.2.2", "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^13.1.0", "ipfs-unixfs-importer": "^15.1.0", - "it-last": "^3.0.1", - "multiformats": "^12.0.1" + "multiformats": "^13.0.0" }, "devDependencies": { - "aegir": "^41.0.3", + "@ipld/dag-pb": "^4.0.7", + "aegir": "^42.0.1", "blockstore-core": "^4.0.1", "datastore-core": "^9.2.0", "delay": "^6.0.0", + "interface-blockstore": "^5.2.9", "it-all": "^3.0.1", "it-drain": "^3.0.1", "it-first": "^3.0.1", + "it-last": "^3.0.4", "it-to-buffer": "^4.0.1", - "uint8arrays": "^4.0.3" + "uint8arrays": "^5.0.1" } } From fca43ed1df31b63e25bc571b413789076859e24f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 08:09:40 +0000 Subject: [PATCH 229/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/mfs-v2.0.0](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.2...@helia/mfs-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 ### Features * update helia to v3 and multiformats to v13 ([9f7dc0a](https://github.com/ipfs/helia-mfs/commit/9f7dc0a0581524531501fc062fefb6ba26d99c02)) --- packages/mfs/CHANGELOG.md | 11 +++++++++++ packages/mfs/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/mfs/CHANGELOG.md b/packages/mfs/CHANGELOG.md index b5a21fcc..f06929f5 100644 --- a/packages/mfs/CHANGELOG.md +++ b/packages/mfs/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/mfs-v2.0.0](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.2...@helia/mfs-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 + +### Features + +* update helia to v3 and multiformats to v13 ([9f7dc0a](https://github.com/ipfs/helia-mfs/commit/9f7dc0a0581524531501fc062fefb6ba26d99c02)) + ## [@helia/mfs-v1.0.2](https://github.com/ipfs/helia-mfs/compare/@helia/mfs-v1.0.1...@helia/mfs-v1.0.2) (2023-10-07) diff --git a/packages/mfs/package.json b/packages/mfs/package.json index d5c2757e..10b878ef 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -1,6 +1,6 @@ { "name": "@helia/mfs", - "version": "1.0.2", + "version": "2.0.0", "description": "A mutable filesystem powered by Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-mfs/tree/main/packages/mfs#readme", From bebe2efbd70d504c3e691dee713339254b24ea05 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 08:09:50 +0000 Subject: [PATCH 230/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 73aceb0b..3e732cc0 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -54,7 +54,7 @@ }, "devDependencies": { "@helia/interface": "^3.0.0", - "@helia/mfs": "^1.0.0", + "@helia/mfs": "^2.0.0", "aegir": "^42.0.1", "go-ipfs": "^0.22.0", "helia": "^3.0.0", From a0381b95051bbf3edfa4f53e0ae2d5f43c1e4382 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 09:28:43 +0100 Subject: [PATCH 231/253] feat!: update helia to v3 and multiformats to v13 (#167) Updates all deps and fixes linting errors. Renames the `dht` routing to `libp2p` since it's used any and all configured libp2p Content Routers instead of the low-level DHT interface for several releases now. BREAKING CHANGE: uses multiformats v13 and helia v3, renames `dht` routing to `libp2p` --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 2 + .github/workflows/semantic-pull-request.yml | 12 ++++ .github/workflows/stale.yml | 13 +++++ .gitignore | 3 - README.md | 15 ++--- package.json | 6 +- packages/interop/README.md | 25 +-------- packages/interop/package.json | 42 ++++++++------ .../interop/test/fixtures/create-helia.ts | 4 +- .../interop/test/fixtures/create-peer-ids.ts | 2 +- packages/interop/test/fixtures/key-types.ts | 2 +- .../test/{dht.spec.ts => libp2p.spec.ts} | 40 ++++++++------ packages/interop/test/pubsub.spec.ts | 19 ++++--- packages/ipns/README.md | 55 ++++++++++++++++--- packages/ipns/package.json | 38 +++++++------ packages/ipns/src/index.ts | 23 ++++---- packages/ipns/src/routing/index.ts | 6 +- .../ipns/src/routing/{dht.ts => libp2p.ts} | 24 ++++---- packages/ipns/src/routing/pubsub.ts | 21 ++++--- packages/ipns/src/utils/dns.ts | 2 +- packages/ipns/typedoc.json | 1 + 22 files changed, 210 insertions(+), 147 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml create mode 100644 .github/workflows/stale.yml rename packages/interop/test/{dht.spec.ts => libp2p.spec.ts} (82%) rename packages/ipns/src/routing/{dht.ts => libp2p.ts} (58%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 35d87d10..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -9,7 +9,9 @@ on: permissions: contents: write + id-token: write packages: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..16d65d72 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,13 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 diff --git a/.gitignore b/.gitignore index 7d54fd58..7ad9e674 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,3 @@ node_modules package-lock.json yarn.lock .vscode -.env -.envrc -.tool-versions diff --git a/README.md b/README.md index 4523f8f3..c4a56555 100644 --- a/README.md +++ b/README.md @@ -13,30 +13,23 @@ > An implementation of IPNS for Helia -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/interop`](./packages/interop) Interop tests for @helia/ipns - [`/packages/ipns`](./packages/ipns) An implementation of IPNS for Helia -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). diff --git a/package.json b/package.json index c95b525f..e75278db 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-ipns/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.1.14", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/interop/README.md b/packages/interop/README.md index fbdb8140..c2ea5500 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -13,35 +13,14 @@ > Interop tests for @helia/ipns -## Table of contents - -- [Install](#install) - - [Browser ` -``` - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index f27fc5e1..bc49eff4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/ipns", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-ipns/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-ipns.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-ipns/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -49,32 +53,34 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-gossipsub": "^10.1.0", - "@chainsafe/libp2p-noise": "^13.0.0", - "@chainsafe/libp2p-yamux": "^5.0.0", - "@helia/interface": "^2.0.0", + "@chainsafe/libp2p-gossipsub": "^11.0.1", + "@chainsafe/libp2p-noise": "^14.1.0", + "@chainsafe/libp2p-yamux": "^6.0.1", + "@helia/interface": "^3.0.0", "@helia/ipns": "^3.0.0", - "@libp2p/interface": "^0.1.2", - "@libp2p/kad-dht": "^10.0.4", - "@libp2p/peer-id": "^3.0.2", - "@libp2p/peer-id-factory": "^3.0.3", - "@libp2p/tcp": "^8.0.4", - "@libp2p/websockets": "^7.0.4", - "aegir": "^41.1.14", + "@libp2p/identify": "^1.0.9", + "@libp2p/interface": "^1.1.1", + "@libp2p/kad-dht": "^12.0.2", + "@libp2p/keychain": "^4.0.5", + "@libp2p/peer-id": "^4.0.4", + "@libp2p/peer-id-factory": "^4.0.3", + "@libp2p/tcp": "^9.0.10", + "@libp2p/websockets": "^8.0.10", + "aegir": "^42.0.1", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "helia": "^2.0.1", + "helia": "^3.0.0", "ipfsd-ctl": "^13.0.0", - "ipns": "^7.0.1", + "ipns": "^8.0.0", "it-all": "^3.0.2", "it-last": "^3.0.1", "it-map": "^3.0.3", - "kubo": "^0.24.0", + "kubo": "^0.25.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.46.6", + "libp2p": "^1.1.1", "merge-options": "^3.0.4", - "multiformats": "^12.0.1", - "uint8arrays": "^4.0.3", + "multiformats": "^13.0.0", + "uint8arrays": "^5.0.1", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index a3826ef6..0a8ea11e 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -6,10 +6,10 @@ import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' import type { Helia } from '@helia/interface' +import type { Identify } from '@libp2p/identify' import type { Libp2p } from '@libp2p/interface' -import type { IdentifyService } from 'libp2p/identify' -export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { +export async function createHeliaNode (config: Libp2pOptions = {}): Promise>> { const blockstore = new MemoryBlockstore() const datastore = new MemoryDatastore() diff --git a/packages/interop/test/fixtures/create-peer-ids.ts b/packages/interop/test/fixtures/create-peer-ids.ts index 01faee2f..60e0094b 100644 --- a/packages/interop/test/fixtures/create-peer-ids.ts +++ b/packages/interop/test/fixtures/create-peer-ids.ts @@ -3,7 +3,7 @@ import map from 'it-map' import { sha256 } from 'multiformats/hashes/sha2' import { compare as uint8ArrayCompare } from 'uint8arrays/compare' import { xor as uint8ArrayXor } from 'uint8arrays/xor' -import type { PeerId } from '@libp2p/interface/peer-id' +import type { PeerId } from '@libp2p/interface' /** * Sort peers by distance to the KadID of the passed buffer diff --git a/packages/interop/test/fixtures/key-types.ts b/packages/interop/test/fixtures/key-types.ts index 4cd79919..cadea32b 100644 --- a/packages/interop/test/fixtures/key-types.ts +++ b/packages/interop/test/fixtures/key-types.ts @@ -1,4 +1,4 @@ -import type { PeerIdType } from '@libp2p/interface/peer-id' +import type { PeerIdType } from '@libp2p/interface' export const keyTypes: PeerIdType[] = [ 'Ed25519', diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/libp2p.spec.ts similarity index 82% rename from packages/interop/test/dht.spec.ts rename to packages/interop/test/libp2p.spec.ts index bef69b9b..87cfc9d5 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/libp2p.spec.ts @@ -1,14 +1,16 @@ /* eslint-env mocha */ import { ipns } from '@helia/ipns' -import { dht } from '@helia/ipns/routing' -import { kadDHT, type DualKadDHT } from '@libp2p/kad-dht' +import { libp2p } from '@helia/ipns/routing' +import { identify } from '@libp2p/identify' +import { kadDHT, removePublicAddressesMapper, type KadDHT } from '@libp2p/kad-dht' +import { keychain, type Keychain } from '@libp2p/keychain' +import { peerIdFromString } from '@libp2p/peer-id' import { createEd25519PeerId, createRSAPeerId, createSecp256k1PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' import last from 'it-last' -import { identifyService } from 'libp2p/identify' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' @@ -23,13 +25,12 @@ import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' -import type { Libp2p } from '@libp2p/interface' +import type { Libp2p, PeerId } from '@libp2p/interface' import type { Controller } from 'ipfsd-ctl' -import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' keyTypes.forEach(type => { - describe(`dht routing with ${type} keys`, () => { - let helia: Helia> + describe(`libp2p routing with ${type} keys`, () => { + let helia: Helia> let kubo: Controller let name: IPNS @@ -51,7 +52,7 @@ keyTypes.forEach(type => { helia = await createHeliaNode({ services: { - identify: identifyService(), + identify: identify(), dht: kadDHT({ validators: { ipns: ipnsValidator @@ -60,8 +61,14 @@ keyTypes.forEach(type => { ipns: ipnsSelector }, // skips waiting for the initial self-query to find peers - allowQueryWithZeroPeers: true - }) + allowQueryWithZeroPeers: true, + + // use lan-only mode + protocol: '/ipfs/lan/kad/1.0.0', + peerInfoMapper: removePublicAddressesMapper, + clientMode: false + }), + keychain: keychain() } }) kubo = await createKuboNode() @@ -83,10 +90,10 @@ keyTypes.forEach(type => { const [closest] = await sortClosestPeers(routingKey, [ helia.libp2p.peerId, - kubo.peer.id + peerIdFromString(kubo.peer.id.toString()) ]) - if (resolver === 'kubo' && closest.equals(kubo.peer.id)) { + if (resolver === 'kubo' && closest.equals(peerIdFromString(kubo.peer.id.toString()))) { break } @@ -102,7 +109,7 @@ keyTypes.forEach(type => { await waitFor(async () => { let found = false - for await (const event of helia.libp2p.services.dht.findPeer(kubo.peer.id)) { + for await (const event of helia.libp2p.services.dht.findPeer(peerIdFromString(kubo.peer.id.toString()))) { if (event.name === 'FINAL_PEER') { found = true } @@ -118,6 +125,7 @@ keyTypes.forEach(type => { await waitFor(async () => { let found = false + // @ts-expect-error kubo deps are out of date for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) { if (event.name === 'FINAL_PEER') { found = true @@ -133,7 +141,7 @@ keyTypes.forEach(type => { name = ipns(helia, { routers: [ - dht(helia) + libp2p(helia) ] }) } @@ -152,11 +160,11 @@ keyTypes.forEach(type => { await createNodes('kubo') const keyName = 'my-ipns-key' - await helia.libp2p.keychain.importPeer(keyName, key) + await helia.libp2p.services.keychain.importPeer(keyName, key) await name.publish(key, value) - const resolved = await last(kubo.api.name.resolve(key)) + const resolved = await last(kubo.api.name.resolve(key.toString())) if (resolved == null) { throw new Error('kubo failed to resolve name') diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index ff32652d..b8feb78b 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -4,10 +4,11 @@ import { gossipsub } from '@chainsafe/libp2p-gossipsub' import { ipns } from '@helia/ipns' import { pubsub } from '@helia/ipns/routing' +import { identify } from '@libp2p/identify' +import { keychain, type Keychain } from '@libp2p/keychain' import { peerIdFromKeys } from '@libp2p/peer-id' import { expect } from 'aegir/chai' import last from 'it-last' -import { identifyService } from 'libp2p/identify' import { base36 } from 'multiformats/bases/base36' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' @@ -23,6 +24,7 @@ import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' +import type { Identify } from '@libp2p/identify' import type { Libp2p } from '@libp2p/interface' import type { PubSub } from '@libp2p/interface/pubsub' import type { Controller } from 'ipfsd-ctl' @@ -34,15 +36,16 @@ const LIBP2P_KEY_CODEC = 0x72 // resolution because Kubo will use the DHT as well keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { describe(`pubsub routing with ${keyType} keys`, () => { - let helia: Helia> + let helia: Helia> let kubo: Controller let name: IPNS beforeEach(async () => { - helia = await createHeliaNode({ + helia = await createHeliaNode<{ identify: Identify, pubsub: PubSub, keychain: Keychain }>({ services: { - identify: identifyService(), - pubsub: gossipsub() + identify: identify(), + pubsub: gossipsub(), + keychain: keychain() } }) kubo = await createKuboNode({ @@ -75,8 +78,8 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { const cid = CID.createV1(raw.code, digest) const keyName = 'my-ipns-key' - await helia.libp2p.keychain.createKey(keyName, keyType) - const peerId = await helia.libp2p.keychain.exportPeerId(keyName) + await helia.libp2p.services.keychain.createKey(keyName, keyType) + const peerId = await helia.libp2p.services.keychain.exportPeerId(keyName) if (peerId.publicKey == null) { throw new Error('No public key present') @@ -87,6 +90,7 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { .with.property('message', 'PublishError.InsufficientPeers') // should fail to resolve the first time as kubo was not subscribed to the pubsub channel + // @ts-expect-error kubo deps are out of date await expect(last(kubo.api.name.resolve(peerId, { timeout: 100 }))).to.eventually.be.undefined() @@ -107,6 +111,7 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { await name.publish(peerId, cid) // kubo should now be able to resolve IPNS name + // @ts-expect-error kubo deps are out of date const resolved = await last(kubo.api.name.resolve(peerId, { timeout: 100 })) diff --git a/packages/ipns/README.md b/packages/ipns/README.md index 025a5fa2..5819b723 100644 --- a/packages/ipns/README.md +++ b/packages/ipns/README.md @@ -17,27 +17,27 @@ IPNS operations using a Helia node -## Example +## Example - Using libp2p and pubsub routers With IPNSRouting routers: ```typescript import { createHelia } from 'helia' import { ipns } from '@helia/ipns' -import { dht, pubsub } from '@helia/ipns/routing' +import { libp2p, pubsub } from '@helia/ipns/routing' import { unixfs } from '@helia/unixfs' const helia = await createHelia() const name = ipns(helia, { routers: [ - dht(helia), + libp2p(helia), pubsub(helia) ] }) // create a public key to publish as an IPNS name -const keyInfo = await helia.libp2p.keychain.createKey('my-key') -const peerId = await helia.libp2p.keychain.exportPeerId(keyInfo.name) +const keyInfo = await helia.libp2p.services.keychain.createKey('my-key') +const peerId = await helia.libp2p.services.keychain.exportPeerId(keyInfo.name) // store some data to publish const fs = unixfs(helia) @@ -50,7 +50,7 @@ await name.publish(peerId, cid) const cid = name.resolve(peerId) ``` -## Example +## Example - Using custom DNS over HTTPS resolvers With default DNSResolver resolvers: @@ -70,7 +70,7 @@ const name = ipns(helia, { const cid = name.resolveDns('some-domain-with-dnslink-entry.com') ``` -## Example +## Example - Resolving a domain with a dnslink entry Calling `resolveDns` with the `@helia/ipns` instance: @@ -90,7 +90,7 @@ console.info(cid) // QmWebsite ``` -## Example +## Example - Using DNS-Over-HTTPS This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This uses binary DNS records so requires extra dependencies to process the @@ -109,7 +109,7 @@ const cid = name.resolveDns('ipfs.io', { }) ``` -## Example +## Example - Using DNS-JSON-Over-HTTPS DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can result in a smaller browser bundle due to the response being plain JSON. @@ -124,3 +124,40 @@ const cid = name.resolveDns('ipfs.io', { ] }) ``` + +# Install + +```console +$ npm i @helia/ipns +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 9eceaad1..cc8f801f 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -3,7 +3,7 @@ "version": "3.0.1", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", + "homepage": "https://github.com/ipfs/helia-ipns/tree/main/packages/ipns#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-ipns.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-ipns/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -43,13 +47,13 @@ "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js" }, - "./routing": { - "types": "./dist/src/routing/index.d.ts", - "import": "./dist/src/routing/index.js" - }, "./dns-resolvers": { "types": "./dist/src/dns-resolvers/index.d.ts", "import": "./dist/src/dns-resolvers/index.js" + }, + "./routing": { + "types": "./dist/src/routing/index.d.ts", + "import": "./dist/src/routing/index.js" } }, "eslintConfig": { @@ -159,28 +163,28 @@ "release": "aegir release" }, "dependencies": { - "@libp2p/interface": "^0.1.4", - "@libp2p/kad-dht": "^10.0.11", - "@libp2p/logger": "^3.0.2", - "@libp2p/peer-id": "^3.0.2", - "dns-over-http-resolver": "^2.1.3", + "@libp2p/interface": "^1.1.1", + "@libp2p/kad-dht": "^12.0.2", + "@libp2p/logger": "^4.0.4", + "@libp2p/peer-id": "^4.0.4", + "dns-over-http-resolver": "^3.0.0", "dns-packet": "^5.6.0", "hashlru": "^2.3.0", "interface-datastore": "^8.0.0", - "ipns": "^7.0.1", + "ipns": "^8.0.0", "is-ipfs": "^8.0.1", - "multiformats": "^12.0.1", - "p-queue": "^7.3.0", + "multiformats": "^13.0.0", + "p-queue": "^8.0.1", "progress-events": "^1.0.0", - "uint8arrays": "^4.0.3" + "uint8arrays": "^5.0.1" }, "devDependencies": { - "@libp2p/peer-id-factory": "^3.0.3", + "@libp2p/peer-id-factory": "^4.0.3", "@types/dns-packet": "^5.6.4", - "aegir": "^41.1.14", + "aegir": "^42.0.1", "datastore-core": "^9.0.3", "sinon": "^17.0.0", - "sinon-ts": "^1.0.0" + "sinon-ts": "^2.0.0" }, "browser": { "./dist/src/dns-resolvers/resolver.js": "./dist/src/dns-resolvers/resolver.browser.js" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index bed58b4c..16dcee31 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -3,27 +3,27 @@ * * IPNS operations using a Helia node * - * @example + * @example Using libp2p and pubsub routers * * With {@link IPNSRouting} routers: * * ```typescript * import { createHelia } from 'helia' * import { ipns } from '@helia/ipns' - * import { dht, pubsub } from '@helia/ipns/routing' + * import { libp2p, pubsub } from '@helia/ipns/routing' * import { unixfs } from '@helia/unixfs' * * const helia = await createHelia() * const name = ipns(helia, { * routers: [ - * dht(helia), + * libp2p(helia), * pubsub(helia) * ] * }) * * // create a public key to publish as an IPNS name - * const keyInfo = await helia.libp2p.keychain.createKey('my-key') - * const peerId = await helia.libp2p.keychain.exportPeerId(keyInfo.name) + * const keyInfo = await helia.libp2p.services.keychain.createKey('my-key') + * const peerId = await helia.libp2p.services.keychain.exportPeerId(keyInfo.name) * * // store some data to publish * const fs = unixfs(helia) @@ -36,7 +36,7 @@ * const cid = name.resolve(peerId) * ``` * - * @example + * @example Using custom DNS over HTTPS resolvers * * With default {@link DNSResolver} resolvers: * @@ -56,7 +56,7 @@ * const cid = name.resolveDns('some-domain-with-dnslink-entry.com') * ``` * - * @example + * @example Resolving a domain with a dnslink entry * * Calling `resolveDns` with the `@helia/ipns` instance: * @@ -76,7 +76,7 @@ * // QmWebsite * ``` * - * @example + * @example Using DNS-Over-HTTPS * * This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This * uses binary DNS records so requires extra dependencies to process the @@ -95,7 +95,7 @@ * }) * ``` * - * @example + * @example Using DNS-JSON-Over-HTTPS * * DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can * result in a smaller browser bundle due to the response being plain JSON. @@ -112,7 +112,7 @@ * ``` */ -import { CodeError } from '@libp2p/interface/errors' +import { CodeError } from '@libp2p/interface' import { logger } from '@libp2p/logger' import { peerIdFromString } from '@libp2p/peer-id' import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' @@ -124,8 +124,7 @@ import { defaultResolver } from './dns-resolvers/default.js' import { localStore, type LocalStore } from './routing/local-store.js' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' import type { DNSResponse } from './utils/dns.js' -import type { AbortOptions } from '@libp2p/interface' -import type { PeerId } from '@libp2p/interface/peer-id' +import type { AbortOptions, PeerId } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' import type { IPNSRecord } from 'ipns' import type { ProgressEvent, ProgressOptions } from 'progress-events' diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts index 232c2212..b29d15cc 100644 --- a/packages/ipns/src/routing/index.ts +++ b/packages/ipns/src/routing/index.ts @@ -1,4 +1,4 @@ -import type { DHTProgressEvents } from './dht.js' +import type { Libp2pContentRoutingProgressEvents } from './libp2p.js' import type { DatastoreProgressEvents } from './local-store.js' import type { PubSubProgressEvents } from './pubsub.js' import type { AbortOptions } from '@libp2p/interface' @@ -19,8 +19,8 @@ export interface IPNSRouting { export type IPNSRoutingEvents = DatastoreProgressEvents | - DHTProgressEvents | + Libp2pContentRoutingProgressEvents | PubSubProgressEvents -export { dht } from './dht.js' +export { libp2p } from './libp2p.js' export { pubsub } from './pubsub.js' diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/libp2p.ts similarity index 58% rename from packages/ipns/src/routing/dht.ts rename to packages/ipns/src/routing/libp2p.ts index ea13b79f..cb015b5d 100644 --- a/packages/ipns/src/routing/dht.ts +++ b/packages/ipns/src/routing/libp2p.ts @@ -1,21 +1,21 @@ import { CustomProgressEvent, type ProgressEvent } from 'progress-events' import type { GetOptions, PutOptions } from './index.js' import type { IPNSRouting } from '../index.js' -import type { ContentRouting } from '@libp2p/interface/content-routing' +import type { ContentRouting } from '@libp2p/interface' -export interface DHTRoutingComponents { +export interface Libp2pContentRoutingComponents { libp2p: { contentRouting: ContentRouting } } -export type DHTProgressEvents = - ProgressEvent<'ipns:routing:dht:error', Error> +export type Libp2pContentRoutingProgressEvents = + ProgressEvent<'ipns:routing:libp2p:error', Error> -export class DHTRouting implements IPNSRouting { +export class Libp2pContentRouting implements IPNSRouting { private readonly contentRouting: ContentRouting - constructor (components: DHTRoutingComponents) { + constructor (components: Libp2pContentRoutingComponents) { this.contentRouting = components.libp2p.contentRouting } @@ -23,7 +23,7 @@ export class DHTRouting implements IPNSRouting { try { await this.contentRouting.put(routingKey, marshaledRecord, options) } catch (err: any) { - options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) + options.onProgress?.(new CustomProgressEvent('ipns:routing:libp2p:error', err)) } } @@ -31,13 +31,17 @@ export class DHTRouting implements IPNSRouting { try { return await this.contentRouting.get(routingKey, options) } catch (err: any) { - options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) + options.onProgress?.(new CustomProgressEvent('ipns:routing:libp2p:error', err)) } throw new Error('Not found') } } -export function dht (components: DHTRoutingComponents): IPNSRouting { - return new DHTRouting(components) +/** + * The libp2p routing uses any available Content Routers configured on the + * passed libp2p node. This could be KadDHT, HTTP API Delegated Routing, etc. + */ +export function libp2p (components: Libp2pContentRoutingComponents): IPNSRouting { + return new Libp2pContentRouting(components) } diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts index 9b08141c..43770696 100644 --- a/packages/ipns/src/routing/pubsub.ts +++ b/packages/ipns/src/routing/pubsub.ts @@ -1,4 +1,4 @@ -import { CodeError } from '@libp2p/interface/errors' +import { CodeError } from '@libp2p/interface' import { logger } from '@libp2p/logger' import { peerIdToRoutingKey } from 'ipns' import { ipnsSelector } from 'ipns/selector' @@ -9,8 +9,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { localStore, type LocalStore } from './local-store.js' import type { GetOptions, IPNSRouting, PutOptions } from './index.js' -import type { PeerId } from '@libp2p/interface/peer-id' -import type { Message, PublishResult, PubSub } from '@libp2p/interface/pubsub' +import type { PeerId, Message, PublishResult, PubSub } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' const log = logger('helia:ipns:routing:pubsub') @@ -30,14 +29,6 @@ export type PubSubProgressEvents = ProgressEvent<'ipns:pubsub:subscribe', { topic: string }> | ProgressEvent<'ipns:pubsub:error', Error> -/** - * This IPNS routing receives IPNS record updates via dedicated - * pubsub topic. - * - * Note we must first be subscribed to the topic in order to receive - * updated records, so the first call to `.get` should be expected - * to fail! - */ class PubSubRouting implements IPNSRouting { private subscriptions: string[] private readonly localStore: LocalStore @@ -192,6 +183,14 @@ function topicToKey (topic: string): Uint8Array { return uint8ArrayFromString(key, 'base64url') } +/** + * This IPNS routing receives IPNS record updates via dedicated + * pubsub topic. + * + * Note we must first be subscribed to the topic in order to receive + * updated records, so the first call to `.get` should be expected + * to fail! + */ export function pubsub (components: PubsubRoutingComponents): IPNSRouting { return new PubSubRouting(components) } diff --git a/packages/ipns/src/utils/dns.ts b/packages/ipns/src/utils/dns.ts index 0590efcf..a913aa6b 100644 --- a/packages/ipns/src/utils/dns.ts +++ b/packages/ipns/src/utils/dns.ts @@ -1,4 +1,4 @@ -import { CodeError } from '@libp2p/interface/errors' +import { CodeError } from '@libp2p/interface' import * as isIPFS from 'is-ipfs' import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js' diff --git a/packages/ipns/typedoc.json b/packages/ipns/typedoc.json index 0c509d8d..6873a0a8 100644 --- a/packages/ipns/typedoc.json +++ b/packages/ipns/typedoc.json @@ -1,6 +1,7 @@ { "entryPoints": [ "./src/index.ts", + "./src/dns-resolvers/index.ts", "./src/routing/index.ts" ] } From 8af090eeeb96a35e5ac8527a27f41d292a6ab908 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 08:35:54 +0000 Subject: [PATCH 232/253] chore(release): 4.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/ipns-v4.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v3.0.1...@helia/ipns-v4.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3, renames `dht` routing to `libp2p` ### Features * update helia to v3 and multiformats to v13 ([#167](https://github.com/ipfs/helia-ipns/issues/167)) ([a0381b9](https://github.com/ipfs/helia-ipns/commit/a0381b95051bbf3edfa4f53e0ae2d5f43c1e4382)) ### Bug Fixes * make @libp2p/interface a dependency ([#159](https://github.com/ipfs/helia-ipns/issues/159)) ([546ecf0](https://github.com/ipfs/helia-ipns/commit/546ecf023bd619d32e187fa6a55d39fcf12e4bbe)), closes [#158](https://github.com/ipfs/helia-ipns/issues/158) --- packages/ipns/CHANGELOG.md | 16 ++++++++++++++++ packages/ipns/package.json | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/ipns/CHANGELOG.md b/packages/ipns/CHANGELOG.md index ab5ae038..f0651d43 100644 --- a/packages/ipns/CHANGELOG.md +++ b/packages/ipns/CHANGELOG.md @@ -1,3 +1,19 @@ +## [@helia/ipns-v4.0.0](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v3.0.1...@helia/ipns-v4.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3, renames `dht` routing to `libp2p` + +### Features + +* update helia to v3 and multiformats to v13 ([#167](https://github.com/ipfs/helia-ipns/issues/167)) ([a0381b9](https://github.com/ipfs/helia-ipns/commit/a0381b95051bbf3edfa4f53e0ae2d5f43c1e4382)) + + +### Bug Fixes + +* make @libp2p/interface a dependency ([#159](https://github.com/ipfs/helia-ipns/issues/159)) ([546ecf0](https://github.com/ipfs/helia-ipns/commit/546ecf023bd619d32e187fa6a55d39fcf12e4bbe)), closes [#158](https://github.com/ipfs/helia-ipns/issues/158) + ## [@helia/ipns-v3.0.1](https://github.com/ipfs/helia-ipns/compare/@helia/ipns-v3.0.0...@helia/ipns-v3.0.1) (2023-12-08) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index cc8f801f..820165f9 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -1,6 +1,6 @@ { "name": "@helia/ipns", - "version": "3.0.1", + "version": "4.0.0", "description": "An implementation of IPNS for Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-ipns/tree/main/packages/ipns#readme", From 5b0ed5228bfc81b4d7f2fd15c356cbbace9e79f6 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 08:36:09 +0000 Subject: [PATCH 233/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index bc49eff4..14976ae2 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -57,7 +57,7 @@ "@chainsafe/libp2p-noise": "^14.1.0", "@chainsafe/libp2p-yamux": "^6.0.1", "@helia/interface": "^3.0.0", - "@helia/ipns": "^3.0.0", + "@helia/ipns": "^4.0.0", "@libp2p/identify": "^1.0.9", "@libp2p/interface": "^1.1.1", "@libp2p/kad-dht": "^12.0.2", From 6405c3487879614dc4dd7308b15c946d644e0488 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 10:20:59 +0100 Subject: [PATCH 234/253] feat!: update helia to v3 and multiformats to v13 (#52) Updates all deps and fixes linting errors. BREAKING CHANGE: uses multiformats v13 and helia v3 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 2 + .github/workflows/semantic-pull-request.yml | 12 ++++ .github/workflows/stale.yml | 25 ++----- README.md | 15 ++--- package.json | 6 +- packages/car/README.md | 74 +++++++++++++++++---- packages/car/package.json | 24 +++---- packages/car/src/index.ts | 4 +- packages/car/typedoc.json | 3 +- packages/interop/README.md | 32 +-------- packages/interop/package.json | 18 +++-- packages/interop/test/index.spec.ts | 3 +- 13 files changed, 123 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 35d87d10..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -9,7 +9,9 @@ on: permissions: contents: write + id-token: write packages: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 6f6d895d..16d65d72 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,25 +2,12 @@ name: Close and mark stale issue on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write jobs: stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' - close-issue-message: 'This issue was closed because it is missing author input.' - stale-issue-label: 'kind/stale' - any-of-labels: 'need/author-input' - exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' - days-before-issue-stale: 6 - days-before-issue-close: 7 - enable-statistics: true + uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 diff --git a/README.md b/README.md index 3386fefe..daa19068 100644 --- a/README.md +++ b/README.md @@ -13,30 +13,23 @@ > Import/export car files from Helia -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/car`](./packages/car) Import/export car files from Helia - [`/packages/interop`](./packages/interop) Interop tests for @helia/car -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-car/issues). diff --git a/package.json b/package.json index c54a5ef0..259ea39c 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-car/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/car/README.md b/packages/car/README.md index 72de958e..96119adb 100644 --- a/packages/car/README.md +++ b/packages/car/README.md @@ -4,7 +4,7 @@

-# @helia/car +# @helia/car [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,71 @@ > Import/export car files from Helia -## Table of contents +# About -- [Install](#install) - - [Browser ` ``` -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-car/issues). diff --git a/packages/car/package.json b/packages/car/package.json index 2ef8852d..c22d2517 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -3,7 +3,7 @@ "version": "1.0.4", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/car#readme", + "homepage": "https://github.com/ipfs/helia-car/tree/main/packages/car#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-car.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-car/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -42,10 +46,6 @@ ".": { "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js" - }, - "./errors": { - "types": "./dist/src/errors.d.ts", - "import": "./dist/src/errors.js" } }, "eslintConfig": { @@ -155,24 +155,24 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@ipld/car": "^5.1.1", "@ipld/dag-pb": "^4.0.0", "@libp2p/interfaces": "^3.3.1", "cborg": "^4.0.3", - "interface-blockstore": "^5.0.0", + "it-drain": "^3.0.5", "it-map": "^3.0.3", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "p-defer": "^4.0.0", - "p-queue": "^7.3.4", + "p-queue": "^8.0.1", "progress-events": "^1.0.0" }, "devDependencies": { - "@helia/unixfs": "^1.2.2", - "aegir": "^41.0.0", + "@helia/unixfs": "^2.0.0", + "aegir": "^42.0.1", "blockstore-core": "^4.0.1", + "interface-blockstore": "^5.2.9", "ipfs-unixfs-importer": "^15.1.0", - "it-drain": "^3.0.1", "it-to-buffer": "^4.0.2" } } diff --git a/packages/car/src/index.ts b/packages/car/src/index.ts index 7813a2ac..a699fa32 100644 --- a/packages/car/src/index.ts +++ b/packages/car/src/index.ts @@ -7,7 +7,7 @@ * * By default it supports `dag-pb`, `dag-cbor`, `dag-json` and `raw` CIDs, more esoteric DAG walkers can be passed as an init option. * - * @example + * @example Exporting a DAG as a CAR file * * ```typescript * import { createHelia } from 'helia' @@ -36,7 +36,7 @@ * await c.export(cid, writer) * ``` * - * @example + * @example Importing all blocks from a CAR file * * ```typescript * import { createHelia } from 'helia' diff --git a/packages/car/typedoc.json b/packages/car/typedoc.json index d63e8ce5..f599dc72 100644 --- a/packages/car/typedoc.json +++ b/packages/car/typedoc.json @@ -1,6 +1,5 @@ { "entryPoints": [ - "./src/index.ts", - "./src/errors.ts" + "./src/index.ts" ] } diff --git a/packages/interop/README.md b/packages/interop/README.md index 254d35ee..af227b18 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/car-interop +# @helia/car-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,40 +13,14 @@ > Interop tests for @helia/car -## Table of contents - -- [Install](#install) - - [Browser ` -``` - -## API Docs - -- - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-car/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index 485965a8..9a177419 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/car", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-car/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-car/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-car.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-car/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -50,19 +54,19 @@ }, "devDependencies": { "@helia/car": "^1.0.0", - "@helia/interface": "^2.0.0", - "@helia/unixfs": "^1.2.2", + "@helia/interface": "^3.0.0", + "@helia/unixfs": "^2.0.0", "@ipld/car": "^5.1.1", - "@libp2p/websockets": "^7.0.5", - "aegir": "^41.0.0", + "@libp2p/websockets": "^8.0.10", + "aegir": "^42.0.1", "go-ipfs": "^0.22.0", - "helia": "^2.0.1", + "helia": "^3.0.0", "ipfs-unixfs-importer": "^15.1.0", "ipfsd-ctl": "^13.0.0", "it-to-buffer": "^4.0.2", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "p-defer": "^4.0.0", "wherearewe": "^2.0.1" }, diff --git a/packages/interop/test/index.spec.ts b/packages/interop/test/index.spec.ts index ca992bf7..41b8992c 100644 --- a/packages/interop/test/index.spec.ts +++ b/packages/interop/test/index.spec.ts @@ -5,6 +5,7 @@ import { type UnixFS, unixfs } from '@helia/unixfs' import { CarReader } from '@ipld/car' import { expect } from 'aegir/chai' import toBuffer from 'it-to-buffer' +import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import { memoryCarWriter } from './fixtures/memory-car.js' @@ -91,6 +92,6 @@ describe('car interop', () => { await c.import(reader) - expect(await toBuffer(u.cat(cid))).to.equalBytes(toBuffer(input)) + expect(await toBuffer(u.cat(CID.parse(cid.toString())))).to.equalBytes(toBuffer(input)) }) }) From ae7cbc9a16a267cb0f6d7cecd381f919430afaea Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 10:21:05 +0100 Subject: [PATCH 235/253] feat!: update helia to v3 and multiformats to v13 (#87) Updates all deps and fixes linting errors. BREAKING CHANGE: uses multiformats v13 and helia v3 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 191 ++---------------- .github/workflows/semantic-pull-request.yml | 12 ++ README.md | 17 +- package.json | 6 +- packages/interop/README.md | 32 +-- packages/interop/package.json | 33 +-- .../test/fixtures/create-helia.browser.ts | 4 +- .../interop/test/fixtures/create-helia.ts | 4 +- packages/interop/test/strings.spec.ts | 3 +- packages/strings/README.md | 41 +++- packages/strings/package.json | 26 +-- 12 files changed, 123 insertions(+), 248 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index a15c3791..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,182 +1,27 @@ name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - run: npm run --if-present doc-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + id-token: write + packages: write + pull-requests: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/README.md b/README.md index 522fa061..c3c36015 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-# @helia/strings +# @helia/strings [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,30 +13,23 @@ > Add/get IPLD blocks containing strings with your Helia node -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/interop`](./packages/interop) Interop tests for @helia/strings - [`/packages/strings`](./packages/strings) Add/get IPLD blocks containing strings with your Helia node -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). diff --git a/package.json b/package.json index 850b254a..2b1aa5a7 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-strings/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/interop/README.md b/packages/interop/README.md index 473db66f..a0dc16ce 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/strings-interop +# @helia/strings-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,40 +13,14 @@ > Interop tests for @helia/strings -## Table of contents - -- [Install](#install) - - [Browser ` -``` - -## API Docs - -- - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index 99c135c6..b661b320 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/strings", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-strings/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-strings.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-strings/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -31,8 +35,9 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { - "package": true, - "sourceType": "module" + "project": true, + "sourceType": "module", + "package": true } }, "scripts": { @@ -49,23 +54,25 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^13.0.1", - "@chainsafe/libp2p-yamux": "^5.0.0", - "@helia/interface": "^2.0.0", + "@chainsafe/libp2p-noise": "^14.1.0", + "@chainsafe/libp2p-yamux": "^6.0.1", + "@helia/interface": "^3.0.0", "@helia/strings": "^1.0.0", - "@libp2p/tcp": "^8.0.8", - "@libp2p/websockets": "^7.0.8", - "aegir": "^41.0.0", + "@libp2p/identify": "^1.0.9", + "@libp2p/tcp": "^9.0.10", + "@libp2p/websockets": "^8.0.10", + "aegir": "^42.0.1", "blockstore-core": "^4.0.1", "datastore-core": "^9.0.3", - "helia": "^2.0.3", + "helia": "^3.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", - "kubo": "^0.23.0", + "kubo": "^0.25.0", "kubo-rpc-client": "^3.0.0", - "libp2p": "^0.46.12", + "libp2p": "^1.1.1", "merge-options": "^3.0.4", - "uint8arrays": "^4.0.3", + "multiformats": "^13.0.0", + "uint8arrays": "^5.0.1", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 867de492..5f0f5828 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,12 +1,12 @@ import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { identify } from '@libp2p/identify' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createHelia } from 'helia' import { createLibp2p, type Libp2pOptions } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -27,7 +27,7 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise { @@ -24,7 +24,7 @@ export async function createHeliaNode (config: Libp2pOptions = {}): Promise { it('should add to kubo and fetch from helia', async () => { const input = 'hello world' const cid = await kubo.api.block.put(uint8ArrayFromString(input)) - const output = await str.get(cid) + const output = await str.get(CID.parse(cid.toString())) expect(output).to.equal(input) }) diff --git a/packages/strings/README.md b/packages/strings/README.md index bbca118d..3a0a845b 100644 --- a/packages/strings/README.md +++ b/packages/strings/README.md @@ -4,7 +4,7 @@

-# @helia/strings +# @helia/strings [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -17,7 +17,7 @@ `@helia/strings` makes working with strings Helia simple & straightforward. -See the Strings interface for all available operations. +See the interface for all available operations. ## Example @@ -33,3 +33,40 @@ const string = await str.get(cid) console.info(string) // hello world ``` + +# Install + +```console +$ npm i @helia/strings +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-strings/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/strings/package.json b/packages/strings/package.json index 96ceac99..d7135073 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -3,7 +3,7 @@ "version": "1.0.1", "description": "Add/get IPLD blocks containing strings with your Helia node", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-strings/tree/master/packages/strings#readme", + "homepage": "https://github.com/ipfs/helia-strings/tree/main/packages/strings#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-strings.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-strings/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -31,8 +35,9 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { - "package": true, - "sourceType": "module" + "project": true, + "sourceType": "module", + "package": true } }, "release": { @@ -136,18 +141,15 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@libp2p/interfaces": "^3.3.1", - "interface-blockstore": "^5.0.0", - "multiformats": "^12.1.2", + "multiformats": "^13.0.0", "progress-events": "^1.0.0", - "uint8arrays": "^4.0.3" + "uint8arrays": "^5.0.1" }, "devDependencies": { - "aegir": "^41.0.0", - "blockstore-core": "^4.0.1" - }, - "typedoc": { - "entryPoint": "./src/index.ts" + "aegir": "^42.0.1", + "blockstore-core": "^4.0.1", + "interface-blockstore": "^5.0.0" } } From e3dc5867ffc4de0dd3b05b56eb1b0ce98d50dcb1 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 10:21:10 +0100 Subject: [PATCH 236/253] feat!: update helia to v3 and multiformats to v13 (#46) Updates all deps and fixes linting errors. BREAKING CHANGE: uses multiformats v13 and helia v3 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 191 ++------------------ .github/workflows/semantic-pull-request.yml | 12 ++ README.md | 17 +- package.json | 6 +- packages/interop/README.md | 32 +--- packages/interop/package.json | 16 +- packages/interop/test/json.spec.ts | 3 +- packages/json/README.md | 39 ++-- packages/json/package.json | 16 +- 10 files changed, 93 insertions(+), 241 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index a15c3791..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,182 +1,27 @@ name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - run: npm run --if-present doc-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + id-token: write + packages: write + pull-requests: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/README.md b/README.md index dc2a35b2..d5d7e7ae 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-# @helia/json +# @helia/json [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,30 +13,23 @@ > Add/get IPLD blocks containing json with your Helia node -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/interop`](./packages/interop) Interop tests for @helia/json - [`/packages/json`](./packages/json) Add/get IPLD blocks containing json with your Helia node -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-json/issues). diff --git a/package.json b/package.json index 510cb429..74138c5d 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-json/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/interop/README.md b/packages/interop/README.md index 29719cd9..50cae693 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/json-interop +# @helia/json-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,40 +13,14 @@ > Interop tests for @helia/json -## Table of contents - -- [Install](#install) - - [Browser ` -``` - -## API Docs - -- - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-json/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index 8def232a..993e4d8f 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/json", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-json/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-json.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-json/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -49,17 +53,17 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@helia/json": "^1.0.0", - "@libp2p/websockets": "^7.0.5", - "aegir": "^41.0.0", + "@libp2p/websockets": "^8.0.10", + "aegir": "^42.0.1", "go-ipfs": "^0.22.0", - "helia": "^2.0.1", + "helia": "^3.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/test/json.spec.ts b/packages/interop/test/json.spec.ts index 6dbd7983..55cc9107 100644 --- a/packages/interop/test/json.spec.ts +++ b/packages/interop/test/json.spec.ts @@ -2,6 +2,7 @@ import { json, type JSON, type AddOptions } from '@helia/json' import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' import * as jsonCodec from 'multiformats/codecs/json' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' @@ -57,7 +58,7 @@ describe('json interop', () => { it('should add to kubo and fetch from helia', async () => { const input = { hello: 'world' } const cid = await kubo.api.block.put(jsonCodec.encode(input)) - const output = await j.get(cid) + const output = await j.get(CID.parse(cid.toString())) expect(output).to.deep.equal(input) }) diff --git a/packages/json/README.md b/packages/json/README.md index 98fa3fb8..b05a5f0d 100644 --- a/packages/json/README.md +++ b/packages/json/README.md @@ -4,7 +4,7 @@

-# @helia/json +# @helia/json [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,36 @@ > Add/get IPLD blocks containing json with your Helia node -## Table of contents +# About -- [Install](#install) - - [Browser ` ``` -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-json/issues). diff --git a/packages/json/package.json b/packages/json/package.json index 208945c5..f442a6df 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -3,7 +3,7 @@ "version": "1.0.3", "description": "Add/get IPLD blocks containing json with your Helia node", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-json/tree/master/packages/json#readme", + "homepage": "https://github.com/ipfs/helia-json/tree/main/packages/json#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-json.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-json/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -135,14 +139,14 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@libp2p/interfaces": "^3.3.1", - "interface-blockstore": "^5.0.0", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^41.0.0", - "blockstore-core": "^4.0.1" + "aegir": "^42.0.1", + "blockstore-core": "^4.0.1", + "interface-blockstore": "^5.0.0" } } From 3c7d9d4a8e74e1a808c265fbc6ecbdc24f0f3da9 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 10:21:14 +0100 Subject: [PATCH 237/253] feat!: update helia to v3 and multiformats to v13 (#45) Updates all deps and fixes linting errors. BREAKING CHANGE: uses multiformats v13 and helia v3 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 191 ++------------------ .github/workflows/semantic-pull-request.yml | 12 ++ README.md | 17 +- package.json | 6 +- packages/dag-json/README.md | 39 ++-- packages/dag-json/package.json | 16 +- packages/interop/README.md | 32 +--- packages/interop/package.json | 16 +- packages/interop/test/dag-json.spec.ts | 3 +- 10 files changed, 93 insertions(+), 241 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index a15c3791..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,182 +1,27 @@ name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - run: npm run --if-present doc-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + id-token: write + packages: write + pull-requests: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/README.md b/README.md index 3d1bb81e..62fd6261 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-# @helia/dag-json +# @helia/dag-json [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,30 +13,23 @@ > Add/get IPLD blocks containing dag-json with your Helia node -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure +# Packages - [`/packages/dag-json`](./packages/dag-json) Add/get IPLD blocks containing dag-json with your Helia node - [`/packages/interop`](./packages/interop) Interop tests for @helia/dag-json -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-json/issues). diff --git a/package.json b/package.json index 4a1e19bd..73b90e9d 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-dag-json/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/dag-json/README.md b/packages/dag-json/README.md index eb50d3ee..af4a4a11 100644 --- a/packages/dag-json/README.md +++ b/packages/dag-json/README.md @@ -4,7 +4,7 @@

-# @helia/dag-json +# @helia/dag-json [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,36 @@ > Add/get IPLD blocks containing dag-json with your Helia node -## Table of contents +# About -- [Install](#install) - - [Browser ` ``` -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-json/issues). diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index 7c95ad4c..dfd303b5 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -3,7 +3,7 @@ "version": "1.0.3", "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/dag-json#readme", + "homepage": "https://github.com/ipfs/helia-dag-json/tree/main/packages/dag-json#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-dag-json.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-dag-json/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -135,15 +139,15 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@ipld/dag-json": "^10.0.1", "@libp2p/interfaces": "^3.3.1", - "interface-blockstore": "^5.0.0", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^41.0.0", - "blockstore-core": "^4.0.1" + "aegir": "^42.0.1", + "blockstore-core": "^4.0.1", + "interface-blockstore": "^5.0.0" } } diff --git a/packages/interop/README.md b/packages/interop/README.md index b722442a..6f5562b0 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/dag-json-interop +# @helia/dag-json-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,40 +13,14 @@ > Interop tests for @helia/dag-json -## Table of contents - -- [Install](#install) - - [Browser ` -``` - -## API Docs - -- - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-json/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index 79e5604f..bd468e14 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/dag-json", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-dag-json/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-dag-json/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-dag-json.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-dag-json/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -50,16 +54,16 @@ }, "devDependencies": { "@helia/dag-json": "^1.0.0", - "@helia/interface": "^2.0.0", - "@libp2p/websockets": "^7.0.5", - "aegir": "^41.0.0", + "@helia/interface": "^3.0.0", + "@libp2p/websockets": "^8.0.10", + "aegir": "^42.0.1", "go-ipfs": "^0.22.0", - "helia": "^2.0.1", + "helia": "^3.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/test/dag-json.spec.ts b/packages/interop/test/dag-json.spec.ts index 59bbcc03..a16338cb 100644 --- a/packages/interop/test/dag-json.spec.ts +++ b/packages/interop/test/dag-json.spec.ts @@ -2,6 +2,7 @@ import { dagJson, type DAGJSON, type AddOptions } from '@helia/dag-json' import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' import * as codec from 'multiformats/codecs/json' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' @@ -57,7 +58,7 @@ describe('dag-json interop', () => { it('should add to kubo and fetch from helia', async () => { const input = { hello: 'world' } const cid = await kubo.api.block.put(codec.encode(input)) - const output = await d.get(cid) + const output = await d.get(CID.parse(cid.toString())) expect(output).to.deep.equal(input) }) From f078447b6eba4c3d404d62bb930757aa1c0efe74 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 8 Jan 2024 10:23:37 +0100 Subject: [PATCH 238/253] feat!: update helia to v3 and multiformats to v13 (#45) Updates all deps and fixes linting errors. BREAKING CHANGE: uses multiformats v13 and helia v3 --- .github/dependabot.yml | 2 +- .github/workflows/js-test-and-release.yml | 191 ++------------------ .github/workflows/semantic-pull-request.yml | 12 ++ README.md | 21 +-- package.json | 6 +- packages/dag-cbor/README.md | 39 ++-- packages/dag-cbor/package.json | 17 +- packages/interop/README.md | 22 +-- packages/interop/package.json | 16 +- packages/interop/test/dag-json.spec.ts | 3 +- 10 files changed, 100 insertions(+), 229 deletions(-) create mode 100644 .github/workflows/semantic-pull-request.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bc3b42d..d401a774 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: schedule: interval: daily time: "10:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 20 commit-message: prefix: "deps" prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index a15c3791..1d7ff79c 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -1,182 +1,27 @@ name: test & maybe release + on: push: branches: - main pull_request: + workflow_dispatch: -jobs: - - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present lint - - run: npm run --if-present dep-check - - run: npm run --if-present doc-check - - test-node: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:node - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: node - - test-chrome: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome - - test-chrome-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: chrome-webworker +permissions: + contents: write + id-token: write + packages: write + pull-requests: write - test-firefox: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true - test-firefox-webworker: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: firefox-webworker - - test-webkit: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit - - test-webkit-webworker: - needs: check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - node: [lts/*] - fail-fast: true - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npm run --if-present test:webkit-webworker - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: webkit-webworker - - test-electron-main: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-main - - test-electron-renderer: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 - with: - flags: electron-renderer - - release: - needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - - uses: ipfs/aegir/actions/cache-node-modules@master - - uses: ipfs/aegir/actions/docker-login@master - with: - docker-token: ${{ secrets.DOCKER_TOKEN }} - docker-username: ${{ secrets.DOCKER_USERNAME }} - - run: npm run --if-present release - env: - GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} +jobs: + js-test-and-release: + uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + secrets: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml new file mode 100644 index 00000000..bd00f090 --- /dev/null +++ b/.github/workflows/semantic-pull-request.yml @@ -0,0 +1,12 @@ +name: Semantic PR + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 diff --git a/README.md b/README.md index f99f7491..3d28e607 100644 --- a/README.md +++ b/README.md @@ -4,39 +4,34 @@

-# @helia/dag-cbor +# @helia/dag-cbor [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-dag-cbor.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-dag-cbor) [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-dag-cbor/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-dag-cbor/actions/workflows/js-test-and-release.yml?query=branch%3Amain) -> Add/get IPLD blocks containing json with your Helia node - -## Table of contents - -- [Structure](#structure) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) +> Add/get IPLD blocks containing dag-cbor with your Helia node -## Structure +# Packages - [`/packages/dag-cbor`](./packages/dag-cbor) Add/get IPLD blocks containing dag-cbor with your Helia node - [`/packages/interop`](./packages/interop) Interop tests for @helia/dag-cbor -## API Docs +> Add/get IPLD blocks containing json with your Helia node + +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-cbor/issues). diff --git a/package.json b/package.json index 2a1d8c4a..789dbf28 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-dag-cbor/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "ipfs" ], @@ -36,7 +40,7 @@ "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop" }, "devDependencies": { - "aegir": "^41.0.0", + "aegir": "^42.0.1", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/dag-cbor/README.md b/packages/dag-cbor/README.md index 6350b669..6b5078c3 100644 --- a/packages/dag-cbor/README.md +++ b/packages/dag-cbor/README.md @@ -4,7 +4,7 @@

-# @helia/dag-cbor +# @helia/dag-cbor [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,36 @@ > Add/get IPLD blocks containing dag-cbor with your Helia node -## Table of contents +# About -- [Install](#install) - - [Browser ` ``` -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-cbor/issues). diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 7aab7be2..74257436 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -3,7 +3,7 @@ "version": "1.0.3", "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/dag-cbor#readme", + "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/main/packages/dag-cbor#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-dag-cbor.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-dag-cbor/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -31,6 +35,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -135,15 +140,15 @@ "release": "aegir release" }, "dependencies": { - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@ipld/dag-cbor": "^9.0.0", "@libp2p/interfaces": "^3.3.1", - "interface-blockstore": "^5.0.0", - "multiformats": "^12.0.1", + "multiformats": "^13.0.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^41.0.0", - "blockstore-core": "^4.0.1" + "aegir": "^42.0.1", + "blockstore-core": "^4.0.1", + "interface-blockstore": "^5.0.0" } } diff --git a/packages/interop/README.md b/packages/interop/README.md index b7fddf73..807fb091 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,7 +4,7 @@

-# @helia/dag-cbor-interop +# @helia/dag-cbor-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -13,21 +13,13 @@ > Interop tests for @helia/dag-cbor -## Table of contents - -- [Install](#install) - - [Browser ` ``` -## API Docs - -- - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-dag-cbor/issues). diff --git a/packages/interop/package.json b/packages/interop/package.json index 4115d050..eef7dffd 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Interop tests for @helia/dag-cbor", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/master/packages/interop#readme", + "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/main/packages/interop#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/helia-dag-cbor.git" @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/helia-dag-cbor/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "keywords": [ "IPFS" ], @@ -31,6 +35,7 @@ "eslintConfig": { "extends": "ipfs", "parserOptions": { + "project": true, "sourceType": "module" } }, @@ -49,16 +54,17 @@ }, "devDependencies": { "@helia/dag-cbor": "^1.0.0", - "@helia/interface": "^2.0.0", + "@helia/interface": "^3.0.0", "@ipld/dag-cbor": "^9.0.0", - "@libp2p/websockets": "^7.0.5", - "aegir": "^41.0.0", + "@libp2p/websockets": "^8.0.10", + "aegir": "^42.0.1", "go-ipfs": "^0.22.0", - "helia": "^2.0.1", + "helia": "^3.0.0", "ipfs-core-types": "^0.14.0", "ipfsd-ctl": "^13.0.0", "kubo-rpc-client": "^3.0.0", "merge-options": "^3.0.4", + "multiformats": "^13.0.0", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/test/dag-json.spec.ts b/packages/interop/test/dag-json.spec.ts index dd4ef3e1..6bbc4cf7 100644 --- a/packages/interop/test/dag-json.spec.ts +++ b/packages/interop/test/dag-json.spec.ts @@ -3,6 +3,7 @@ import { dagCbor, type DAGCBOR, type AddOptions } from '@helia/dag-cbor' import * as codec from '@ipld/dag-cbor' import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' @@ -57,7 +58,7 @@ describe('dag-json interop', () => { it('should add to kubo and fetch from helia', async () => { const input = { hello: 'world' } const cid = await kubo.api.block.put(codec.encode(input)) - const output = await d.get(cid) + const output = await d.get(CID.parse(cid.toString())) expect(output).to.deep.equal(input) }) From 89ca5a5ae6d440178d6c0e56f3cf84974e3a33c8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 09:27:03 +0000 Subject: [PATCH 239/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/car-v2.0.0](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.4...@helia/car-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3 ### Features * update helia to v3 and multiformats to v13 ([#52](https://github.com/ipfs/helia-car/issues/52)) ([6405c34](https://github.com/ipfs/helia-car/commit/6405c3487879614dc4dd7308b15c946d644e0488)) --- packages/car/CHANGELOG.md | 11 +++++++++++ packages/car/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/car/CHANGELOG.md b/packages/car/CHANGELOG.md index b718dda8..d14a9084 100644 --- a/packages/car/CHANGELOG.md +++ b/packages/car/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/car-v2.0.0](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.4...@helia/car-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3 + +### Features + +* update helia to v3 and multiformats to v13 ([#52](https://github.com/ipfs/helia-car/issues/52)) ([6405c34](https://github.com/ipfs/helia-car/commit/6405c3487879614dc4dd7308b15c946d644e0488)) + ## [@helia/car-v1.0.4](https://github.com/ipfs/helia-car/compare/@helia/car-v1.0.3...@helia/car-v1.0.4) (2023-10-07) diff --git a/packages/car/package.json b/packages/car/package.json index c22d2517..df7d36b8 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -1,6 +1,6 @@ { "name": "@helia/car", - "version": "1.0.4", + "version": "2.0.0", "description": "Import/export car files from Helia", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-car/tree/main/packages/car#readme", From d7b502f9ed0d5c49a4ff1ee6f2bfdefa77730cbe Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 09:27:14 +0000 Subject: [PATCH 240/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 9a177419..509c57ed 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,7 +53,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/car": "^1.0.0", + "@helia/car": "^2.0.0", "@helia/interface": "^3.0.0", "@helia/unixfs": "^2.0.0", "@ipld/car": "^5.1.1", From e0c295617ce10d343eebccd6c975a391deb4fed9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 09:28:26 +0000 Subject: [PATCH 241/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/strings-v2.0.0](https://github.com/ipfs/helia-strings/compare/@helia/strings-v1.0.1...@helia/strings-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3 ### Features * update helia to v3 and multiformats to v13 ([#87](https://github.com/ipfs/helia-strings/issues/87)) ([ae7cbc9](https://github.com/ipfs/helia-strings/commit/ae7cbc9a16a267cb0f6d7cecd381f919430afaea)) ### Trivial Changes * update sibling dependencies ([1184ea6](https://github.com/ipfs/helia-strings/commit/1184ea695987cee7f922b7954c8bc626bc035dba)) --- packages/strings/CHANGELOG.md | 16 ++++++++++++++++ packages/strings/package.json | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/strings/CHANGELOG.md b/packages/strings/CHANGELOG.md index 8e971df1..63cf240b 100644 --- a/packages/strings/CHANGELOG.md +++ b/packages/strings/CHANGELOG.md @@ -1,3 +1,19 @@ +## [@helia/strings-v2.0.0](https://github.com/ipfs/helia-strings/compare/@helia/strings-v1.0.1...@helia/strings-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3 + +### Features + +* update helia to v3 and multiformats to v13 ([#87](https://github.com/ipfs/helia-strings/issues/87)) ([ae7cbc9](https://github.com/ipfs/helia-strings/commit/ae7cbc9a16a267cb0f6d7cecd381f919430afaea)) + + +### Trivial Changes + +* update sibling dependencies ([1184ea6](https://github.com/ipfs/helia-strings/commit/1184ea695987cee7f922b7954c8bc626bc035dba)) + ## [@helia/strings-v1.0.1](https://github.com/ipfs/helia-strings/compare/@helia/strings-v1.0.0...@helia/strings-v1.0.1) (2023-10-06) diff --git a/packages/strings/package.json b/packages/strings/package.json index d7135073..0cdd5f7b 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -1,6 +1,6 @@ { "name": "@helia/strings", - "version": "1.0.1", + "version": "2.0.0", "description": "Add/get IPLD blocks containing strings with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-strings/tree/main/packages/strings#readme", From 1a24e86ef308161ae21ed3ffb6bf94f80f2483b8 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 09:28:38 +0000 Subject: [PATCH 242/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- packages/strings/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index b661b320..adf30ad4 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -57,7 +57,7 @@ "@chainsafe/libp2p-noise": "^14.1.0", "@chainsafe/libp2p-yamux": "^6.0.1", "@helia/interface": "^3.0.0", - "@helia/strings": "^1.0.0", + "@helia/strings": "^2.0.0", "@libp2p/identify": "^1.0.9", "@libp2p/tcp": "^9.0.10", "@libp2p/websockets": "^8.0.10", diff --git a/packages/strings/README.md b/packages/strings/README.md index 3a0a845b..f4a4beb9 100644 --- a/packages/strings/README.md +++ b/packages/strings/README.md @@ -17,7 +17,7 @@ `@helia/strings` makes working with strings Helia simple & straightforward. -See the interface for all available operations. +See the Strings interface for all available operations. ## Example From bfddf3da7c11476cb201aea66a0edccd574b3701 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 09:29:44 +0000 Subject: [PATCH 243/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/dag-json-v2.0.0](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.3...@helia/dag-json-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3 ### Features * update helia to v3 and multiformats to v13 ([#45](https://github.com/ipfs/helia-dag-json/issues/45)) ([3c7d9d4](https://github.com/ipfs/helia-dag-json/commit/3c7d9d4a8e74e1a808c265fbc6ecbdc24f0f3da9)) --- packages/dag-json/CHANGELOG.md | 11 +++++++++++ packages/dag-json/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/dag-json/CHANGELOG.md b/packages/dag-json/CHANGELOG.md index 03d7ae87..6f7d9133 100644 --- a/packages/dag-json/CHANGELOG.md +++ b/packages/dag-json/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/dag-json-v2.0.0](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.3...@helia/dag-json-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3 + +### Features + +* update helia to v3 and multiformats to v13 ([#45](https://github.com/ipfs/helia-dag-json/issues/45)) ([3c7d9d4](https://github.com/ipfs/helia-dag-json/commit/3c7d9d4a8e74e1a808c265fbc6ecbdc24f0f3da9)) + ## [@helia/dag-json-v1.0.3](https://github.com/ipfs/helia-dag-json/compare/@helia/dag-json-v1.0.2...@helia/dag-json-v1.0.3) (2023-10-07) diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index dfd303b5..2aa0b512 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-json", - "version": "1.0.3", + "version": "2.0.0", "description": "Add/get IPLD blocks containing dag-json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-json/tree/main/packages/dag-json#readme", From 3ba2a284497a9209cfe166ec7f37b77f0c5f49e5 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 09:30:02 +0000 Subject: [PATCH 244/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index bd468e14..a551beef 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,7 +53,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/dag-json": "^1.0.0", + "@helia/dag-json": "^2.0.0", "@helia/interface": "^3.0.0", "@libp2p/websockets": "^8.0.10", "aegir": "^42.0.1", From b4a6e80b567d98a6377c6dd8bf54fbd8d4a3875d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 09:30:55 +0000 Subject: [PATCH 245/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/json-v2.0.0](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.3...@helia/json-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3 ### Features * update helia to v3 and multiformats to v13 ([#46](https://github.com/ipfs/helia-json/issues/46)) ([e3dc586](https://github.com/ipfs/helia-json/commit/e3dc5867ffc4de0dd3b05b56eb1b0ce98d50dcb1)) --- packages/json/CHANGELOG.md | 11 +++++++++++ packages/json/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/json/CHANGELOG.md b/packages/json/CHANGELOG.md index 0b84cecf..661239fe 100644 --- a/packages/json/CHANGELOG.md +++ b/packages/json/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/json-v2.0.0](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.3...@helia/json-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3 + +### Features + +* update helia to v3 and multiformats to v13 ([#46](https://github.com/ipfs/helia-json/issues/46)) ([e3dc586](https://github.com/ipfs/helia-json/commit/e3dc5867ffc4de0dd3b05b56eb1b0ce98d50dcb1)) + ## [@helia/json-v1.0.3](https://github.com/ipfs/helia-json/compare/@helia/json-v1.0.2...@helia/json-v1.0.3) (2023-10-07) diff --git a/packages/json/package.json b/packages/json/package.json index f442a6df..663bf6f0 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -1,6 +1,6 @@ { "name": "@helia/json", - "version": "1.0.3", + "version": "2.0.0", "description": "Add/get IPLD blocks containing json with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-json/tree/main/packages/json#readme", From d6e234179e1b960bf283c2b2709be1638c0a0632 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 09:31:07 +0000 Subject: [PATCH 246/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 993e4d8f..059a8098 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -54,7 +54,7 @@ }, "devDependencies": { "@helia/interface": "^3.0.0", - "@helia/json": "^1.0.0", + "@helia/json": "^2.0.0", "@libp2p/websockets": "^8.0.10", "aegir": "^42.0.1", "go-ipfs": "^0.22.0", From 547ca94a4b380a89c8766d305294cd54f852618c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 8 Jan 2024 09:32:37 +0000 Subject: [PATCH 247/253] chore(release): 2.0.0 [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [@helia/dag-cbor-v2.0.0](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.3...@helia/dag-cbor-v2.0.0) (2024-01-08) ### ⚠ BREAKING CHANGES * uses multiformats v13 and helia v3 ### Features * update helia to v3 and multiformats to v13 ([#45](https://github.com/ipfs/helia-dag-cbor/issues/45)) ([f078447](https://github.com/ipfs/helia-dag-cbor/commit/f078447b6eba4c3d404d62bb930757aa1c0efe74)) --- packages/dag-cbor/CHANGELOG.md | 11 +++++++++++ packages/dag-cbor/package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/dag-cbor/CHANGELOG.md b/packages/dag-cbor/CHANGELOG.md index 3a3b9f75..fa2aab1c 100644 --- a/packages/dag-cbor/CHANGELOG.md +++ b/packages/dag-cbor/CHANGELOG.md @@ -1,3 +1,14 @@ +## [@helia/dag-cbor-v2.0.0](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.3...@helia/dag-cbor-v2.0.0) (2024-01-08) + + +### ⚠ BREAKING CHANGES + +* uses multiformats v13 and helia v3 + +### Features + +* update helia to v3 and multiformats to v13 ([#45](https://github.com/ipfs/helia-dag-cbor/issues/45)) ([f078447](https://github.com/ipfs/helia-dag-cbor/commit/f078447b6eba4c3d404d62bb930757aa1c0efe74)) + ## [@helia/dag-cbor-v1.0.3](https://github.com/ipfs/helia-dag-cbor/compare/@helia/dag-cbor-v1.0.2...@helia/dag-cbor-v1.0.3) (2023-10-07) diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 74257436..2725e43c 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -1,6 +1,6 @@ { "name": "@helia/dag-cbor", - "version": "1.0.3", + "version": "2.0.0", "description": "Add/get IPLD blocks containing dag-cbor with your Helia node", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/helia-dag-cbor/tree/main/packages/dag-cbor#readme", From 6dad087768fd60a2676d632add8b5261fc704909 Mon Sep 17 00:00:00 2001 From: "aegir[bot]" Date: Mon, 8 Jan 2024 09:32:49 +0000 Subject: [PATCH 248/253] chore: update sibling dependencies --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index eef7dffd..51ada94b 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,7 +53,7 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/dag-cbor": "^1.0.0", + "@helia/dag-cbor": "^2.0.0", "@helia/interface": "^3.0.0", "@ipld/dag-cbor": "^9.0.0", "@libp2p/websockets": "^8.0.10", From 705e7c5cdace26e6ed8332c5e92d71158f3604eb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 8 Jan 2024 12:02:51 +0100 Subject: [PATCH 249/253] chore: rename mfs interop test file --- packages/interop/test/{index.spec.ts => mfs.spec.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/interop/test/{index.spec.ts => mfs.spec.ts} (100%) diff --git a/packages/interop/test/index.spec.ts b/packages/interop/test/mfs.spec.ts similarity index 100% rename from packages/interop/test/index.spec.ts rename to packages/interop/test/mfs.spec.ts From 9e4e7086e6902852180a01995db075336e109d29 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 8 Jan 2024 12:12:29 +0100 Subject: [PATCH 250/253] chore: rename car interop file --- packages/interop/test/{index.spec.ts => car.spec.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/interop/test/{index.spec.ts => car.spec.ts} (100%) diff --git a/packages/interop/test/index.spec.ts b/packages/interop/test/car.spec.ts similarity index 100% rename from packages/interop/test/index.spec.ts rename to packages/interop/test/car.spec.ts From 977a35497e986828fe70a1ce7025313c92cb4d3c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 8 Jan 2024 13:39:02 +0100 Subject: [PATCH 251/253] chore: update interop tests --- README.md | 42 ++++++--- packages/helia/package.json | 4 - packages/helia/src/index.ts | 4 + .../src/utils/libp2p-defaults.browser.ts | 2 +- packages/helia/src/utils/libp2p-defaults.ts | 2 +- packages/interop/package.json | 30 +++++-- packages/interop/test/car.spec.ts | 2 +- packages/interop/test/dag-cbor.spec.ts | 2 +- packages/interop/test/dag-json.spec.ts | 2 +- .../test/fixtures/create-helia.browser.ts | 89 +++++++++++-------- .../interop/test/fixtures/create-helia.ts | 77 ++++++++-------- .../test/fixtures/create-kubo.browser.ts | 3 +- packages/interop/test/fixtures/create-kubo.ts | 3 +- ...store.spec.ts => helia-blockstore.spec.ts} | 2 +- .../{hashes.spec.ts => helia-hashes.spec.ts} | 8 +- .../test/{pins.spec.ts => helia-pins.spec.ts} | 2 +- packages/interop/test/ipns-libp2p.spec.ts | 31 +------ packages/interop/test/ipns-pubsub.spec.ts | 19 +--- packages/interop/test/json.spec.ts | 2 +- packages/interop/test/mfs.spec.ts | 2 +- packages/interop/test/strings.spec.ts | 2 +- packages/interop/test/unixfs-bitswap.spec.ts | 2 +- packages/interop/test/unixfs-files.spec.ts | 2 +- 23 files changed, 171 insertions(+), 163 deletions(-) rename packages/interop/test/{blockstore.spec.ts => helia-blockstore.spec.ts} (97%) rename packages/interop/test/{hashes.spec.ts => helia-hashes.spec.ts} (94%) rename packages/interop/test/{pins.spec.ts => helia-pins.spec.ts} (98%) diff --git a/README.md b/README.md index bc37989f..00a2a568 100644 --- a/README.md +++ b/README.md @@ -159,24 +159,42 @@ graph TD; # 🏭 Code Structure -Helia embraces a modular approach and encourages users to bring their own implementations of interfacing libraries to suit their needs. Helia also ships supplemental libraries and tools including: +Helia embraces a modular approach and encourages users to bring their own implementations of various APIs to suit their needs. -- [`@helia/UnixFS`](./packages/unixfs) -- [`@helia/mfs`](./packages/mfs) -- [`@helia/ipns`](./packages/ipns) -- [`@helia/car`](./packages/car) -- [`@helia/strings`](./packages/strings) -- [`@helia/json`](./packages/json) -- [`@helia/dag-json`](./packages/dag-json) -- [`@helia/dag-cbor`](./packages/dag-cbor) +The basic Helia API is defined in: -These libraries are by no means the "one true implementation", but instead instead provide optionality depending on one's needs. - -This repo itself is made up of these packages: - [`/packages/interface`](./packages/interface) The Helia API + +The API is implemented by: + - [`/packages/helia`](./packages/helia) An implementation of the Helia API + +Helia also ships a number of supplemental libraries and tools. + +These libraries are not intended to be the "one true implementation" of any given API, but are made available for users to include depending on the need of their particular application: + +- [./packages/unixfs](./packages/unixfs) The `@helia/unixfs` module +- [./packages/mfs](./packages/mfs) The `@helia/mfs` module +- [./packages/ipns](./packages/ipns) `@helia/ipns` +- [./packages/car](./packages/car) `@helia/car` +- [./packages/strings](./packages/strings) `@helia/strings` +- [./packages/json](./packages/json) `@helia/json` +- [./packages/dag-json](./packages/dag-json) `@helia/dag-json` +- [./packages/dag-cbor](./packages/dag-cbor) `@helia/dag-cbor` + +An interface suite ensures everything is compatible: + - [`/packages/interop`](./packages/interop) Interop tests for Helia +## Other modules + +There are several other modules available outside this repo: + +- [`@helia/delegated-routing-v1-http-api`](https://github.com/ipfs/helia-delegated-routing-v1-http-api) An implementation of the [Delegated Routing v1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) including a server and a client +- [Helia WNFS](https://github.com/shovelers/helia-wnfs) a [WNFS](https://guide.fission.codes/developers/webnative/file-system-wnfs) implementation built on top of Helia +- [`@helia/remote-pinning`](https://github.com/ipfs/helia-remote-pinning) A Helia client for communicating with [IPFS Pinning Services](https://ipfs.github.io/pinning-services-api-spec/) +- [`@helia/http-gateway`](https://github.com/ipfs/helia-http-gateway) An implentation of the [IPFS HTTP Gateway API](https://docs.ipfs.tech/concepts/ipfs-gateway/#gateway-types) built with Helia + # 📣 Project status Helia v1 shipped in 202303 (see [releases](https://github.com/ipfs/helia/releases)), and development keeps on trucking as we work on initiatives in the [roadmap](#roadmap) and make performance improvements and bug fixes along the way. diff --git a/packages/helia/package.json b/packages/helia/package.json index 928b6f13..f0cee030 100644 --- a/packages/helia/package.json +++ b/packages/helia/package.json @@ -46,10 +46,6 @@ ".": { "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js" - }, - "./hashers": { - "types": "./dist/src/utils/default-hashers.d.ts", - "import": "./dist/src/utils/default-hashers.js" } }, "eslintConfig": { diff --git a/packages/helia/src/index.ts b/packages/helia/src/index.ts index f9d63277..c5a47f47 100644 --- a/packages/helia/src/index.ts +++ b/packages/helia/src/index.ts @@ -22,6 +22,7 @@ import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { HeliaImpl } from './helia.js' +import { libp2pDefaults } from './utils/libp2p-defaults.js' import { createLibp2p } from './utils/libp2p.js' import type { DefaultLibp2pServices } from './utils/libp2p-defaults.js' import type { Helia } from '@helia/interface' @@ -40,6 +41,9 @@ export * from '@helia/interface' export * from '@helia/interface/blocks' export * from '@helia/interface/pins' +export type { DefaultLibp2pServices } +export { libp2pDefaults } + /** * DAGWalkers take a block and yield CIDs encoded in that block */ diff --git a/packages/helia/src/utils/libp2p-defaults.browser.ts b/packages/helia/src/utils/libp2p-defaults.browser.ts index 0a1e71d6..253527dc 100644 --- a/packages/helia/src/utils/libp2p-defaults.browser.ts +++ b/packages/helia/src/utils/libp2p-defaults.browser.ts @@ -34,7 +34,7 @@ export interface DefaultLibp2pServices extends Record { pubsub: PubSub } -export function libp2pDefaults (options: Libp2pDefaultsOptions): Libp2pOptions { +export function libp2pDefaults (options: Libp2pDefaultsOptions = {}): Libp2pOptions { return { peerId: options.peerId, addresses: { diff --git a/packages/helia/src/utils/libp2p-defaults.ts b/packages/helia/src/utils/libp2p-defaults.ts index 549d04c4..2d1fb042 100644 --- a/packages/helia/src/utils/libp2p-defaults.ts +++ b/packages/helia/src/utils/libp2p-defaults.ts @@ -38,7 +38,7 @@ export interface DefaultLibp2pServices extends Record { upnp: unknown } -export function libp2pDefaults (options: Libp2pDefaultsOptions): Libp2pOptions { +export function libp2pDefaults (options: Libp2pDefaultsOptions = {}): Libp2pOptions { return { peerId: options.peerId, addresses: { diff --git a/packages/interop/package.json b/packages/interop/package.json index be61bb97..a0e3d86d 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,27 +53,41 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^14.0.0", - "@chainsafe/libp2p-yamux": "^6.0.1", "@helia/block-brokers": "^0.0.0", + "@helia/car": "^2.0.0", + "@helia/dag-cbor": "^2.0.0", + "@helia/dag-json": "^2.0.0", "@helia/interface": "^3.0.0", - "@libp2p/identify": "^1.0.1", + "@helia/ipns": "^4.0.0", + "@helia/json": "^2.0.0", + "@helia/mfs": "^2.0.0", + "@helia/strings": "^2.0.0", + "@helia/unixfs": "^2.0.0", + "@ipld/car": "^5.2.5", + "@ipld/dag-cbor": "^9.0.7", + "@libp2p/interface": "^1.1.1", + "@libp2p/kad-dht": "^12.0.2", + "@libp2p/keychain": "^4.0.5", "@libp2p/peer-id": "^4.0.3", - "@libp2p/tcp": "^9.0.1", + "@libp2p/peer-id-factory": "^4.0.3", "@libp2p/websockets": "^8.0.1", "@multiformats/sha3": "^3.0.0", "aegir": "^42.0.0", - "blockstore-core": "^4.0.0", - "datastore-core": "^9.0.0", "helia": "^3.0.0", + "ipfs-core-types": "^0.14.1", + "ipfs-unixfs-importer": "^15.2.3", "ipfsd-ctl": "^13.0.0", "it-all": "^3.0.4", "it-drain": "^3.0.5", + "it-last": "^3.0.4", + "it-map": "^3.0.5", "it-to-buffer": "^4.0.1", "kubo": "^0.25.0", "kubo-rpc-client": "^3.0.1", - "libp2p": "^1.0.1", - "multiformats": "^13.0.0" + "multiformats": "^13.0.0", + "p-defer": "^4.0.0", + "uint8arrays": "^5.0.1", + "wherearewe": "^2.0.1" }, "browser": { "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", diff --git a/packages/interop/test/car.spec.ts b/packages/interop/test/car.spec.ts index 41b8992c..0b108366 100644 --- a/packages/interop/test/car.spec.ts +++ b/packages/interop/test/car.spec.ts @@ -14,7 +14,7 @@ import type { Helia } from '@helia/interface' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' -describe('car interop', () => { +describe('@helia/car', () => { let helia: Helia let c: Car let u: UnixFS diff --git a/packages/interop/test/dag-cbor.spec.ts b/packages/interop/test/dag-cbor.spec.ts index 6bbc4cf7..f6aa223b 100644 --- a/packages/interop/test/dag-cbor.spec.ts +++ b/packages/interop/test/dag-cbor.spec.ts @@ -10,7 +10,7 @@ import type { Helia } from '@helia/interface' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' -describe('dag-json interop', () => { +describe('@helia/dag-cbor', () => { let helia: Helia let d: DAGCBOR let kubo: Controller diff --git a/packages/interop/test/dag-json.spec.ts b/packages/interop/test/dag-json.spec.ts index a16338cb..45b870b1 100644 --- a/packages/interop/test/dag-json.spec.ts +++ b/packages/interop/test/dag-json.spec.ts @@ -10,7 +10,7 @@ import type { Helia } from '@helia/interface' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' -describe('dag-json interop', () => { +describe('@helia/dag-json', () => { let helia: Helia let d: DAGJSON let kubo: Controller diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 8a48541f..642f5f2f 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,51 +1,62 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import { bitswap } from '@helia/block-brokers' -import { identify } from '@libp2p/identify' +import { ipnsValidator, ipnsSelector } from '@helia/ipns' +import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' -import { createHelia, type HeliaInit } from 'helia' -import { createLibp2p } from 'libp2p' +import { sha3512 } from '@multiformats/sha3' +import { createHelia, libp2pDefaults } from 'helia' import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' +import type { DefaultLibp2pServices } from 'helia' -export async function createHeliaNode (init?: Partial): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() +export async function createHeliaNode (): Promise>> { + const defaults = libp2pDefaults() - // dial-only in the browser until webrtc browser-to-browser arrives - const libp2p = await createLibp2p({ - transports: [ - webSockets({ - filter: all - }) - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identify() - }, - connectionGater: { - // allow dialing loopback - denyDialMultiaddr: () => false - } - }) + // allow dialing insecure WebSockets + defaults.transports?.pop() + defaults.transports = [ + ...(defaults.transports ?? []), + webSockets({ + filter: all + }) + ] + + // allow dialing loopback + defaults.connectionGater = { + denyDialMultiaddr: () => false + } - const helia = await createHelia({ - libp2p, + // use LAN DHT + defaults.services = { + ...(defaults.services ?? {}), + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + }, + // skips waiting for the initial self-query to find peers + allowQueryWithZeroPeers: true, + + protocol: '/ipfs/lan/kad/1.0.0', + peerInfoMapper: removePublicAddressesMapper, + clientMode: false + }) + } + + // remove services that are not used in tests + delete defaults.services.autoNAT + delete defaults.services.dcutr + delete defaults.services.delegatedRouting + + return createHelia>({ blockBrokers: [ bitswap() ], - blockstore, - datastore, - ...init + libp2p: defaults, + hashers: [ + sha3512 + ] }) - - return helia } diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index 559a3fec..301ce0bb 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -1,48 +1,49 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import { bitswap } from '@helia/block-brokers' -import { identify } from '@libp2p/identify' -import { tcp } from '@libp2p/tcp' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' -import { createHelia, type HeliaInit } from 'helia' -import { createLibp2p } from 'libp2p' +import { ipnsValidator, ipnsSelector } from '@helia/ipns' +import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht' +import { sha3512 } from '@multiformats/sha3' +import { createHelia, libp2pDefaults } from 'helia' import type { Helia } from '@helia/interface' +import type { Libp2p } from '@libp2p/interface' +import type { DefaultLibp2pServices } from 'helia' -export async function createHeliaNode (init?: Partial): Promise { - const blockstore = new MemoryBlockstore() - const datastore = new MemoryDatastore() +export async function createHeliaNode (): Promise>> { + const defaults = libp2pDefaults() + defaults.addresses = { + listen: [ + '/ip4/0.0.0.0/tcp/0' + ] + } + defaults.services = { + ...(defaults.services ?? {}), + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + }, + // skips waiting for the initial self-query to find peers + allowQueryWithZeroPeers: true, - const libp2p = await createLibp2p({ - addresses: { - listen: [ - '/ip4/0.0.0.0/tcp/0' - ] - }, - transports: [ - tcp() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ], - datastore, - services: { - identify: identify() - } - }) + protocol: '/ipfs/lan/kad/1.0.0', + peerInfoMapper: removePublicAddressesMapper, + clientMode: false + }) + } + + // remove services that are not used in tests + delete defaults.services.autoNAT + delete defaults.services.dcutr + delete defaults.services.delegatedRouting - const helia = await createHelia({ - libp2p, + return createHelia>({ blockBrokers: [ bitswap() ], - blockstore, - datastore, - ...init + libp2p: defaults, + hashers: [ + sha3512 + ] }) - - return helia } diff --git a/packages/interop/test/fixtures/create-kubo.browser.ts b/packages/interop/test/fixtures/create-kubo.browser.ts index 2e6645e4..48a6633b 100644 --- a/packages/interop/test/fixtures/create-kubo.browser.ts +++ b/packages/interop/test/fixtures/create-kubo.browser.ts @@ -15,6 +15,7 @@ export async function createKuboNode (): Promise { ] } } - } + }, + args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] }) } diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts index f05517ea..fa7718b4 100644 --- a/packages/interop/test/fixtures/create-kubo.ts +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -17,6 +17,7 @@ export async function createKuboNode (): Promise { ] } } - } + }, + args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] }) } diff --git a/packages/interop/test/blockstore.spec.ts b/packages/interop/test/helia-blockstore.spec.ts similarity index 97% rename from packages/interop/test/blockstore.spec.ts rename to packages/interop/test/helia-blockstore.spec.ts index fa08cfcb..401e4c73 100644 --- a/packages/interop/test/blockstore.spec.ts +++ b/packages/interop/test/helia-blockstore.spec.ts @@ -11,7 +11,7 @@ import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' import type { Controller } from 'ipfsd-ctl' -describe('blockstore', () => { +describe('helia - blockstore', () => { let helia: Helia let kubo: Controller diff --git a/packages/interop/test/hashes.spec.ts b/packages/interop/test/helia-hashes.spec.ts similarity index 94% rename from packages/interop/test/hashes.spec.ts rename to packages/interop/test/helia-hashes.spec.ts index 79715829..ef1ff42d 100644 --- a/packages/interop/test/hashes.spec.ts +++ b/packages/interop/test/helia-hashes.spec.ts @@ -11,16 +11,12 @@ import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' import type { Controller } from 'ipfsd-ctl' -describe('hashes', () => { +describe('helia - hashes', () => { let helia: Helia let kubo: Controller beforeEach(async () => { - helia = await createHeliaNode({ - hashers: [ - sha3512 - ] - }) + helia = await createHeliaNode() kubo = await createKuboNode() // connect the two nodes diff --git a/packages/interop/test/pins.spec.ts b/packages/interop/test/helia-pins.spec.ts similarity index 98% rename from packages/interop/test/pins.spec.ts rename to packages/interop/test/helia-pins.spec.ts index d443846f..f798ea06 100644 --- a/packages/interop/test/pins.spec.ts +++ b/packages/interop/test/helia-pins.spec.ts @@ -11,7 +11,7 @@ import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' import type { Controller } from 'ipfsd-ctl' -describe('pins', () => { +describe('helia - pins', () => { let helia: Helia let kubo: Controller diff --git a/packages/interop/test/ipns-libp2p.spec.ts b/packages/interop/test/ipns-libp2p.spec.ts index 87cfc9d5..a71c8055 100644 --- a/packages/interop/test/ipns-libp2p.spec.ts +++ b/packages/interop/test/ipns-libp2p.spec.ts @@ -2,14 +2,9 @@ import { ipns } from '@helia/ipns' import { libp2p } from '@helia/ipns/routing' -import { identify } from '@libp2p/identify' -import { kadDHT, removePublicAddressesMapper, type KadDHT } from '@libp2p/kad-dht' -import { keychain, type Keychain } from '@libp2p/keychain' import { peerIdFromString } from '@libp2p/peer-id' import { createEd25519PeerId, createRSAPeerId, createSecp256k1PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' -import { ipnsSelector } from 'ipns/selector' -import { ipnsValidator } from 'ipns/validator' import last from 'it-last' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' @@ -26,10 +21,12 @@ import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' import type { Libp2p, PeerId } from '@libp2p/interface' +import type { KadDHT } from '@libp2p/kad-dht' +import type { Keychain } from '@libp2p/keychain' import type { Controller } from 'ipfsd-ctl' keyTypes.forEach(type => { - describe(`libp2p routing with ${type} keys`, () => { + describe(`@helia/ipns - libp2p routing with ${type} keys`, () => { let helia: Helia> let kubo: Controller let name: IPNS @@ -50,27 +47,7 @@ keyTypes.forEach(type => { const digest = await sha256.digest(input) value = CID.createV1(raw.code, digest) - helia = await createHeliaNode({ - services: { - identify: identify(), - dht: kadDHT({ - validators: { - ipns: ipnsValidator - }, - selectors: { - ipns: ipnsSelector - }, - // skips waiting for the initial self-query to find peers - allowQueryWithZeroPeers: true, - - // use lan-only mode - protocol: '/ipfs/lan/kad/1.0.0', - peerInfoMapper: removePublicAddressesMapper, - clientMode: false - }), - keychain: keychain() - } - }) + helia = await createHeliaNode() kubo = await createKuboNode() // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key diff --git a/packages/interop/test/ipns-pubsub.spec.ts b/packages/interop/test/ipns-pubsub.spec.ts index b8feb78b..4884762f 100644 --- a/packages/interop/test/ipns-pubsub.spec.ts +++ b/packages/interop/test/ipns-pubsub.spec.ts @@ -1,11 +1,8 @@ /* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 5] */ -import { gossipsub } from '@chainsafe/libp2p-gossipsub' import { ipns } from '@helia/ipns' import { pubsub } from '@helia/ipns/routing' -import { identify } from '@libp2p/identify' -import { keychain, type Keychain } from '@libp2p/keychain' import { peerIdFromKeys } from '@libp2p/peer-id' import { expect } from 'aegir/chai' import last from 'it-last' @@ -24,9 +21,9 @@ import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' -import type { Identify } from '@libp2p/identify' import type { Libp2p } from '@libp2p/interface' import type { PubSub } from '@libp2p/interface/pubsub' +import type { Keychain } from '@libp2p/keychain' import type { Controller } from 'ipfsd-ctl' const LIBP2P_KEY_CODEC = 0x72 @@ -35,22 +32,14 @@ const LIBP2P_KEY_CODEC = 0x72 // component of the keypair, but that means we can't test pubsub // resolution because Kubo will use the DHT as well keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { - describe(`pubsub routing with ${keyType} keys`, () => { + describe(`@helia/ipns - pubsub routing with ${keyType} keys`, () => { let helia: Helia> let kubo: Controller let name: IPNS beforeEach(async () => { - helia = await createHeliaNode<{ identify: Identify, pubsub: PubSub, keychain: Keychain }>({ - services: { - identify: identify(), - pubsub: gossipsub(), - keychain: keychain() - } - }) - kubo = await createKuboNode({ - args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] - }) + helia = await createHeliaNode() + kubo = await createKuboNode() // connect the two nodes await connect(helia, kubo, '/meshsub/1.1.0') diff --git a/packages/interop/test/json.spec.ts b/packages/interop/test/json.spec.ts index 55cc9107..9cb702c0 100644 --- a/packages/interop/test/json.spec.ts +++ b/packages/interop/test/json.spec.ts @@ -10,7 +10,7 @@ import type { Helia } from '@helia/interface' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' -describe('json interop', () => { +describe('@helia/json', () => { let helia: Helia let j: JSON let kubo: Controller diff --git a/packages/interop/test/mfs.spec.ts b/packages/interop/test/mfs.spec.ts index 963f0474..58b48d78 100644 --- a/packages/interop/test/mfs.spec.ts +++ b/packages/interop/test/mfs.spec.ts @@ -7,7 +7,7 @@ import { createKuboNode } from './fixtures/create-kubo.js' import type { Helia } from '@helia/interface' import type { Controller } from 'ipfsd-ctl' -describe('mfs interop', () => { +describe('@helia/mfs', () => { let helia: Helia let fs: MFS let kubo: Controller diff --git a/packages/interop/test/strings.spec.ts b/packages/interop/test/strings.spec.ts index 894f6964..66f8bcdb 100644 --- a/packages/interop/test/strings.spec.ts +++ b/packages/interop/test/strings.spec.ts @@ -11,7 +11,7 @@ import type { Helia } from '@helia/interface' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' -describe('strings interop', () => { +describe('@helia/strings', () => { let helia: Helia let str: Strings let kubo: Controller diff --git a/packages/interop/test/unixfs-bitswap.spec.ts b/packages/interop/test/unixfs-bitswap.spec.ts index 7990e7de..948d61fb 100644 --- a/packages/interop/test/unixfs-bitswap.spec.ts +++ b/packages/interop/test/unixfs-bitswap.spec.ts @@ -11,7 +11,7 @@ import type { Libp2p } from '@libp2p/interface' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' -describe('unixfs bitswap interop', () => { +describe('@helia/unixfs - bitswap', () => { let helia: Helia let unixFs: UnixFS let kubo: Controller diff --git a/packages/interop/test/unixfs-files.spec.ts b/packages/interop/test/unixfs-files.spec.ts index db329bf4..45dd3d83 100644 --- a/packages/interop/test/unixfs-files.spec.ts +++ b/packages/interop/test/unixfs-files.spec.ts @@ -13,7 +13,7 @@ import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' -describe('unixfs interop', () => { +describe('@helia/unixfs - files', () => { let helia: Helia let unixFs: UnixFS let kubo: Controller From 2359d37eb10d4ea006e2d602fb28926ff7ea4a69 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 8 Jan 2024 13:51:23 +0100 Subject: [PATCH 252/253] chore: update gh actions --- .github/workflows/js-test-and-release.yml | 27 ----------------------- .github/workflows/main.yml | 2 +- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 .github/workflows/js-test-and-release.yml diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml deleted file mode 100644 index 1d7ff79c..00000000 --- a/.github/workflows/js-test-and-release.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: test & maybe release - -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -permissions: - contents: write - id-token: write - packages: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} - cancel-in-progress: true - -jobs: - js-test-and-release: - uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 - secrets: - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 496c495f..5c6dc2a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,7 +173,7 @@ jobs: id-token: write pull-requests: write steps: - - uses: google-github-actions/release-please-action@v4 + - uses: google-github-actions/release-please-action@v3 id: release with: token: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} From 43a8bcc4c5279925a7fafe248b44bb6de5d190f7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 8 Jan 2024 14:07:38 +0100 Subject: [PATCH 253/253] chore: fix firefox webworker and kubo --- packages/interop/test/car.spec.ts | 11 ++++++++--- .../interop/test/fixtures/create-helia.browser.ts | 3 +++ packages/interop/test/fixtures/create-helia.ts | 3 +++ packages/interop/test/unixfs-bitswap.spec.ts | 8 ++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/interop/test/car.spec.ts b/packages/interop/test/car.spec.ts index 0b108366..c31cf00d 100644 --- a/packages/interop/test/car.spec.ts +++ b/packages/interop/test/car.spec.ts @@ -4,6 +4,7 @@ import { car } from '@helia/car' import { type UnixFS, unixfs } from '@helia/unixfs' import { CarReader } from '@ipld/car' import { expect } from 'aegir/chai' +import drain from 'it-drain' import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' @@ -62,11 +63,15 @@ describe('@helia/car', () => { const buf = await writer.bytes() - kubo.api.dag.import([buf]) + await drain(kubo.api.dag.import([buf])) - const bytes = await toBuffer(kubo.api.cat(cid)) + const output: Uint8Array[] = [] - expect(bytes).to.equalBytes(toBuffer(input)) + for await (const b of kubo.api.cat(cid)) { + output.push(b) + } + + expect(toBuffer(output)).to.equalBytes(toBuffer(input)) }) it('should export a car from kubo, import and read the contents from Helia', async () => { diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 642f5f2f..49adc191 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -45,6 +45,9 @@ export async function createHeliaNode (): Promise { const cid = await unixFs.addFile(candidate) - const bytes = await toBuffer(kubo.api.cat(cid)) + const output: Uint8Array[] = [] - expect(bytes).to.equalBytes(toBuffer(input)) + for await (const b of kubo.api.cat(cid)) { + output.push(b) + } + + expect(toBuffer(output)).to.equalBytes(toBuffer(input)) }) it('should add a large file to kubo and fetch it from helia', async () => {