diff --git a/.circleci/config.yml b/.circleci/config.yml index f9fe22c4001..ffabab7ada7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,410 +1,37 @@ version: 2.1 - -aliases: - make_out_dirs: &make_out_dirs - run: mkdir -p /tmp/circleci-artifacts /tmp/circleci-workspace /tmp/circleci-test-results/{unit,sharness} - restore_gomod: &restore_gomod - restore_cache: - keys: - - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }} - - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}- - - v5-dep-{{ .Branch }}- - - v5-dep-master- - store_gomod: &store_gomod - save_cache: - key: v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }} - paths: - - ~/go/pkg/mod - - ~/.cache/go-build/ - only-version-tags: &only-version-tags - tags: - only: /^v[0-9].*/ - branches: - ignore: /.*/ - -default_environment: &default_environment - SERVICE: circle-ci - TRAVIS: 1 - CIRCLE: 1 - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - CIRCLE_ARTIFACTS: /tmp/circleci-artifacts - GIT_PAGER: cat - -executors: - golang: - docker: - - image: circleci/golang:1.15.2 - working_directory: ~/ipfs/go-ipfs - environment: - <<: *default_environment - GO111MODULE: "on" - TEST_NO_DOCKER: 1 - TEST_NO_FUSE: 1 - GOPATH: /home/circleci/go - TEST_VERBOSE: 1 - node: - docker: - - image: circleci/node:14 - working_directory: ~/ipfs/go-ipfs - environment: - <<: *default_environment - node-browsers: - docker: - - image: circleci/node:14-browsers - working_directory: ~/ipfs/go-ipfs - environment: - <<: *default_environment - NO_SANDBOX: true - IPFS_REUSEPORT: false - LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 - E2E_IPFSD_TYPE: go - dockerizer: - docker: - - image: circleci/golang:1.15.2 - environment: - IMAGE_NAME: ipfs/go-ipfs - WIP_IMAGE_TAG: wip - +setup: true +orbs: + continuation: circleci/continuation@0.2.0 jobs: - gobuild: - executor: golang - steps: - - checkout - - *make_out_dirs - - *restore_gomod - - run: - command: make cmd/ipfs-try-build - environment: - TEST_NO_FUSE: 0 - - run: - command: make cmd/ipfs-try-build - environment: - TEST_NO_FUSE: 1 - - *store_gomod - golint: - executor: golang - steps: - - checkout - - *make_out_dirs - - *restore_gomod - - run: | - make -O test_go_lint - - *store_gomod - gotest: - executor: golang - steps: - - checkout - - *make_out_dirs - - *restore_gomod - - - run: | - make -j 1 test/unit/gotest.junit.xml \ - && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - - run: - when: always - command: bash <(curl -s https://codecov.io/bash) -cF unittests -X search -f coverage/unit_tests.coverprofile - - - run: - when: always - command: mv "test/unit/gotest.junit.xml" /tmp/circleci-test-results/unit - - - *store_gomod - - - store_test_results: - path: /tmp/circleci-test-results - # Save artifacts - - store_artifacts: - path: /tmp/circleci-artifacts - - store_artifacts: - path: /tmp/circleci-test-results - sharness: - machine: - image: ubuntu-2004:202010-01 - working_directory: ~/ipfs/go-ipfs - environment: - <<: *default_environment - GO111MODULE: "on" - TEST_NO_DOCKER: 1 - TEST_NO_FUSE: 1 - GOPATH: /home/circleci/go - TEST_VERBOSE: 1 - steps: - - run: sudo apt update - - run: sudo apt install socat net-tools - - checkout - - - run: - mkdir rb-pinning-service-api && - cd rb-pinning-service-api && - git init && - git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git && - git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 && - git checkout FETCH_HEAD - - run: - cd rb-pinning-service-api && - docker-compose pull && - docker-compose up -d - - - *make_out_dirs - - *restore_gomod - - - run: - name: Setup Environment Variables - # we need the docker host IP; all ports exported by child containers can be accessed there. - command: echo "export DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV - - run: - echo DOCKER_HOST=$DOCKER_HOST && - make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 DOCKER_HOST=$DOCKER_HOST - - - run: - when: always - command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile - - - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness - # make sure we fail if there are test failures - - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0' - - - *store_gomod - - - store_test_results: - path: /tmp/circleci-test-results - # Save artifacts - - store_artifacts: - path: /tmp/circleci-artifacts - - store_artifacts: - path: /tmp/circleci-test-results - build: - executor: golang + generate-params: + executor: continuation/default steps: - checkout - - *make_out_dirs - - *restore_gomod - - run: - name: Building - command: make build - - run: - name: Storing - command: | - mkdir -p /tmp/circleci-workspace/bin - cp cmd/ipfs/ipfs /tmp/circleci-workspace/bin - - persist_to_workspace: - root: /tmp/circleci-workspace - paths: - - bin/ipfs - - *store_gomod - interop: - executor: node - parallelism: 4 - steps: - - *make_out_dirs - - attach_workspace: - at: /tmp/circleci-workspace - - run: - name: Installing dependencies - command: | - npm init -y - npm install ipfs@^0.52.2 - npm install ipfs-interop@^4.0.0 - npm install mocha-circleci-reporter@0.0.3 - working_directory: ~/ipfs/go-ipfs/interop - - run: - name: Running tests - command: | - mkdir -p /tmp/test-results/interop/ - export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)" - npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter - working_directory: ~/ipfs/go-ipfs/interop - environment: - IPFS_REUSEPORT: false - LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 - IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs - - store_test_results: - path: /tmp/test-results - go-ipfs-api: - executor: golang - steps: - - *make_out_dirs - - attach_workspace: - at: /tmp/circleci-workspace - - run: - name: Cloning - command: | - git clone https://github.com/ipfs/go-ipfs-api.git - git -C go-ipfs-api log -1 - - run: - name: Starting the daemon - command: /tmp/circleci-workspace/bin/ipfs daemon --init --enable-namesys-pubsub - background: true - - run: - name: Waiting for the daemon - no_output_timeout: 30s - command: | - while ! /tmp/circleci-workspace/bin/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do - sleep 1 - done - - restore_cache: - keys: - - v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }} - - v1-go-api- - - run: - command: go test -v ./... - working_directory: ~/ipfs/go-ipfs/go-ipfs-api - - save_cache: - key: v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }} - paths: - - ~/go/pkg/mod - - ~/.cache/go-build/ - - run: - name: Stopping the daemon - command: /tmp/circleci-workspace/bin/ipfs shutdown - go-ipfs-http-client: - executor: golang - steps: - - *make_out_dirs - - attach_workspace: - at: /tmp/circleci-workspace - - run: - name: Cloning - command: | - git clone https://github.com/ipfs/go-ipfs-http-client.git - git -C go-ipfs-http-client log -1 - - restore_cache: - keys: - - v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }} - - v1-http-client- - - run: - name: go test -v ./... - command: | - export PATH=/tmp/circleci-workspace/bin:$PATH - go test -v ./... - working_directory: ~/ipfs/go-ipfs/go-ipfs-http-client - - save_cache: - key: v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }} - paths: - - ~/go/pkg/mod - - ~/.cache/go-build/ - ipfs-webui: - executor: node-browsers - steps: - - *make_out_dirs - - attach_workspace: - at: /tmp/circleci-workspace - - run: - name: Cloning - command: | - git clone https://github.com/ipfs-shipyard/ipfs-webui.git - git -C ipfs-webui log -1 - - restore_cache: - keys: - - v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }} - - v1-ipfs-webui- - - run: - name: Installing dependencies - command: | - npm install - working_directory: ~/ipfs/go-ipfs/ipfs-webui - - run: - name: Running upstream tests (finish early if they fail) - command: | - npm test || circleci-agent step halt - working_directory: ~/ipfs/go-ipfs/ipfs-webui - - run: - name: Running tests with go-ipfs built from current commit - command: npm test - working_directory: ~/ipfs/go-ipfs/ipfs-webui - environment: - IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs - - save_cache: - key: v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }} - paths: - - ~/ipfs/go-ipfs/ipfs-webui/node_modules - docker-build: - executor: dockerizer - steps: - - checkout - - setup_remote_docker: - version: "19.03.13" - - run: - name: Build Docker image - command: | - docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG . - - run: - name: Archive Docker image - command: docker save -o go-ipfs-image.tar $IMAGE_NAME - - persist_to_workspace: - root: . - paths: - - ./go-ipfs-image.tar - docker-push: - executor: dockerizer - steps: - - checkout - - setup_remote_docker: - version: "19.03.13" - - attach_workspace: - at: /tmp/workspace - - run: - name: Load archived Docker image - command: docker load -i /tmp/workspace/go-ipfs-image.tar - - run: - name: Publish Docker Image to Docker Hub - command: | - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - ./bin/push-docker-tags.sh $(date -u +%F) "$CIRCLE_SHA1" "$CIRCLE_BRANCH" "$CIRCLE_TAG" - + - run: + name: Generate params + # for builds on the ipfs/go-ipfs repo, use 2xlarge for faster builds + # but since this is not available for many contributors, we otherwise use medium + command: | + echo $CIRCLE_REPOSITORY_URL + if [ "$CIRCLE_REPOSITORY_URL" = 'git@github.com:ipfs/go-ipfs.git' ]; then + resource_class=2xlarge + make_jobs=10 + else + resource_class=medium + make_jobs=3 + fi + cat \<<- EOF > params.json + { + "resource_class": "$resource_class", + "make_jobs": "$make_jobs" + } + EOF + cat params.json + - continuation/continue: + parameters: params.json + configuration_path: .circleci/main.yml workflows: version: 2 - - # Runs for all branches, but not on tags - # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag - test: - jobs: - - gobuild - - golint - - gotest - - sharness - - build - - interop: - requires: - - build - - go-ipfs-api: - requires: - - build - - go-ipfs-http-client: - requires: - - build - - ipfs-webui: - requires: - - build - - docker-build - - docker-push: - # Requires dockerhub credentials, from circleci context. - context: dockerhub - requires: - - docker-build - - golint - - gotest - - sharness - - interop - - go-ipfs-api - - go-ipfs-http-client - - ipfs-webui - filters: - branches: - only: - - master - - feat/stabilize-dht - - # NOTE: CircleCI only builds tags if you explicitly filter for them. That - # also means tag-based jobs can only depend on other tag-based jobs, so we - # use a separate workflow because every job needs to be tagged together. - # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag - docker-on-tag: + setup-workflow: jobs: - - docker-build: - filters: *only-version-tags - - docker-push: - context: dockerhub - filters: *only-version-tags - requires: - - docker-build + - generate-params diff --git a/.circleci/main.yml b/.circleci/main.yml new file mode 100644 index 00000000000..b6bd73a8646 --- /dev/null +++ b/.circleci/main.yml @@ -0,0 +1,366 @@ +version: 2.1 + +parameters: + resource_class: + type: string + default: medium + make_jobs: + type: string + default: 3 + +aliases: + make_out_dirs: &make_out_dirs + run: mkdir -p /tmp/circleci-artifacts /tmp/circleci-workspace /tmp/circleci-test-results/{unit,sharness} + restore_gomod: &restore_gomod + restore_cache: + keys: + - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }} + - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}- + - v5-dep-{{ .Branch }}- + - v5-dep-master- + store_gomod: &store_gomod + save_cache: + key: v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }} + paths: + - ~/go/pkg/mod + - ~/.cache/go-build/ + +default_environment: &default_environment + SERVICE: circle-ci + TRAVIS: 1 + CIRCLE: 1 + CIRCLE_TEST_REPORTS: /tmp/circleci-test-results + CIRCLE_ARTIFACTS: /tmp/circleci-artifacts + GIT_PAGER: cat + +executors: + golang: + docker: + - image: cimg/go:1.16.7 + working_directory: ~/ipfs/go-ipfs + environment: + <<: *default_environment + TEST_NO_DOCKER: 1 + TEST_NO_FUSE: 1 + TEST_VERBOSE: 1 + node: + docker: + - image: circleci/node:14 + working_directory: ~/ipfs/go-ipfs + environment: + <<: *default_environment + node-browsers: + docker: + - image: circleci/node:14-browsers + working_directory: ~/ipfs/go-ipfs + environment: + <<: *default_environment + NO_SANDBOX: true + IPFS_REUSEPORT: false + LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 + E2E_IPFSD_TYPE: go + dockerizer: + docker: + - image: cimg/go:1.16.7 + environment: + IMAGE_NAME: ipfs/go-ipfs + WIP_IMAGE_TAG: wip + +jobs: + gobuild: + executor: golang + steps: + - checkout + - *make_out_dirs + - *restore_gomod + - run: + command: make cmd/ipfs-try-build + environment: + TEST_NO_FUSE: 0 + - run: + command: make cmd/ipfs-try-build + environment: + TEST_NO_FUSE: 1 + - *store_gomod + golint: + executor: golang + steps: + - checkout + - *make_out_dirs + - *restore_gomod + - run: | + make -O test_go_lint + - *store_gomod + gotest: + executor: golang + steps: + - checkout + - *make_out_dirs + - *restore_gomod + + - run: | + make -j 1 test/unit/gotest.junit.xml \ + && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] + - run: + when: always + command: bash <(curl -s https://codecov.io/bash) -cF unittests -X search -f coverage/unit_tests.coverprofile + - run: + command: go test -v ./... + working_directory: ~/ipfs/go-ipfs/docs/examples/go-ipfs-as-a-library + + - run: + when: always + command: mv "test/unit/gotest.junit.xml" /tmp/circleci-test-results/unit + + - *store_gomod + + - store_test_results: + path: /tmp/circleci-test-results + # Save artifacts + - store_artifacts: + path: /tmp/circleci-artifacts + - store_artifacts: + path: /tmp/circleci-test-results + sharness: + machine: + image: ubuntu-2004:202010-01 + resource_class: << pipeline.parameters.resource_class >> + working_directory: ~/ipfs/go-ipfs + environment: + <<: *default_environment + TEST_NO_DOCKER: 1 + TEST_NO_FUSE: 1 + TEST_VERBOSE: 1 + steps: + - run: sudo apt update + - run: | + mkdir ~/localgo && cd ~/localgo + wget https://golang.org/dl/go1.16.7.linux-amd64.tar.gz + tar xfz go1.16.7.linux-amd64.tar.gz + echo "export PATH=$(pwd)/go/bin:\$PATH" >> ~/.bashrc + - run: go version + - run: sudo apt install socat net-tools + - checkout + + - run: + mkdir rb-pinning-service-api && + cd rb-pinning-service-api && + git init && + git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git && + git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 && + git checkout FETCH_HEAD + - run: + cd rb-pinning-service-api && + docker-compose pull && + docker-compose up -d + + - *make_out_dirs + - *restore_gomod + + - run: + name: Setup Environment Variables + # we need the docker host IP; all ports exported by child containers can be accessed there. + command: echo "export TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV + - run: + echo TEST_DOCKER_HOST=$TEST_DOCKER_HOST && + make -O -j << pipeline.parameters.make_jobs >> coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 TEST_DOCKER_HOST=$TEST_DOCKER_HOST + - run: + when: always + command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile + + - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness + # make sure we fail if there are test failures + - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0' + + - *store_gomod + + - store_test_results: + path: /tmp/circleci-test-results + # Save artifacts + - store_artifacts: + path: /tmp/circleci-artifacts + - store_artifacts: + path: /tmp/circleci-test-results + build: + executor: golang + steps: + - checkout + - *make_out_dirs + - *restore_gomod + - run: + name: Building + command: make build + - run: + name: Storing + command: | + mkdir -p /tmp/circleci-workspace/bin + cp cmd/ipfs/ipfs /tmp/circleci-workspace/bin + - persist_to_workspace: + root: /tmp/circleci-workspace + paths: + - bin/ipfs + - *store_gomod + interop: + executor: node + parallelism: 4 + steps: + - *make_out_dirs + - attach_workspace: + at: /tmp/circleci-workspace + - run: + name: Installing dependencies + command: | + npm init -y + npm install ipfs@^0.52.2 + npm install ipfs-interop@^4.0.0 + npm install mocha-circleci-reporter@0.0.3 + working_directory: ~/ipfs/go-ipfs/interop + - run: + name: Running tests + command: | + mkdir -p /tmp/test-results/interop/ + export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)" + npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter + working_directory: ~/ipfs/go-ipfs/interop + environment: + IPFS_REUSEPORT: false + LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 + IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs + - store_test_results: + path: /tmp/test-results + go-ipfs-api: + executor: golang + steps: + - *make_out_dirs + - attach_workspace: + at: /tmp/circleci-workspace + - run: + name: Cloning + command: | + git clone https://github.com/ipfs/go-ipfs-api.git + git -C go-ipfs-api log -1 + - run: + name: Starting the daemon + command: /tmp/circleci-workspace/bin/ipfs daemon --init --enable-namesys-pubsub + background: true + - run: + name: Waiting for the daemon + no_output_timeout: 30s + command: | + while ! /tmp/circleci-workspace/bin/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do + sleep 1 + done + - restore_cache: + keys: + - v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }} + - v1-go-api- + - run: + command: go test -v ./... + working_directory: ~/ipfs/go-ipfs/go-ipfs-api + - save_cache: + key: v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }} + paths: + - ~/go/pkg/mod + - ~/.cache/go-build/ + - run: + name: Stopping the daemon + command: /tmp/circleci-workspace/bin/ipfs shutdown + go-ipfs-http-client: + executor: golang + steps: + - *make_out_dirs + - attach_workspace: + at: /tmp/circleci-workspace + - run: + name: Cloning + command: | + git clone https://github.com/ipfs/go-ipfs-http-client.git + git -C go-ipfs-http-client log -1 + - restore_cache: + keys: + - v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }} + - v1-http-client- + - run: + name: go test -v ./... + command: | + export PATH=/tmp/circleci-workspace/bin:$PATH + go test -v ./... + working_directory: ~/ipfs/go-ipfs/go-ipfs-http-client + - save_cache: + key: v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }} + paths: + - ~/go/pkg/mod + - ~/.cache/go-build/ + ipfs-webui: + executor: node-browsers + steps: + - *make_out_dirs + - attach_workspace: + at: /tmp/circleci-workspace + - run: + name: Cloning + command: | + git clone https://github.com/ipfs-shipyard/ipfs-webui.git + git -C ipfs-webui log -1 + - restore_cache: + keys: + - v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }} + - v1-ipfs-webui- + - run: + name: Installing dependencies + command: | + npm install + working_directory: ~/ipfs/go-ipfs/ipfs-webui + - run: + name: Running upstream tests (finish early if they fail) + command: | + npm test || circleci-agent step halt + working_directory: ~/ipfs/go-ipfs/ipfs-webui + - run: + name: Running tests with go-ipfs built from current commit + command: npm test + working_directory: ~/ipfs/go-ipfs/ipfs-webui + environment: + IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs + - save_cache: + key: v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }} + paths: + - ~/ipfs/go-ipfs/ipfs-webui/node_modules + # We only run build as a test here. DockerHub images are built and published + # by Github Action now: https://github.com/ipfs/go-ipfs/pull/8467 + docker-build: + executor: dockerizer + steps: + - checkout + - setup_remote_docker: + version: "19.03.13" + - run: + name: Build Docker image + command: | + docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG . + +workflows: + version: 2 + + # Runs for all branches, but not on tags + # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag + test: + jobs: + - gobuild + - golint + - gotest + - sharness + - build + - interop: + requires: + - build + - go-ipfs-api: + requires: + - build + - go-ipfs-http-client: + requires: + - build + - ipfs-webui: + requires: + - build + - docker-build diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index 72d014c8fda..00000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: 'Bug Report' -about: 'Report a bug in go-ipfs.' -labels: kind/bug, need/triage ---- - - - -#### Version information: - - -#### Description: - diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000000..96b29dc8bac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,62 @@ +name: Bug Report +description: Report a bug in go-ipfs. +labels: + - kind/bug + - need/triage +body: + - type: markdown + attributes: + value: | + - Make sure you are running the [latest version of go-ipfs][releases] before reporting an issue. + - If you have an enhancement or feature request for go-ipfs, please select [a different option][issues]. + - Please report possible security issues by email to security@ipfs.io + + [issues]: https://github.com/ipfs/go-ipfs/issues/new/choose + [releases]: https://github.com/ipfs/go-ipfs/releases + - type: checkboxes + attributes: + label: Checklist + description: Please verify that you've followed these steps + options: + - label: This is a bug report, not a question. Ask questions on [discuss.ipfs.io](https://discuss.ipfs.io). + required: true + - label: I have searched on the [issue tracker](https://github.com/ipfs/go-ipfs/issues?q=is%3Aissue) for my bug. + required: true + - label: I am running the latest [go-ipfs version](https://dist.ipfs.io/#go-ipfs) or have an issue updating. + required: true + - type: dropdown + id: install + validations: + required: true + attributes: + label: Installation method + description: Please select your installation method + options: + - ipfs-desktop + - ipfs-update or dist.ipfs.io + - third-party binary + - built from source + - type: textarea + id: version + attributes: + label: Version + render: Text + description: | + Enter the output of `ipfs version --all`. If you can't run that command, please include a copy of your [gateway's version page](http://localhost:8080/api/v0/version?enc=text&all=true). + - type: textarea + id: config + attributes: + label: Config + render: json + description: | + Enter the output of `ipfs config show`. + - type: textarea + attributes: + label: Description + description: | + This is where you get to tell us what went wrong. When doing so, please make sure to include *all* relevant information. + + Please try to include: + * What you were doing when you experienced the bug. + * Any error messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). + * When possible, steps to reliably produce the bug. diff --git a/.github/ISSUE_TEMPLATE/doc.md b/.github/ISSUE_TEMPLATE/doc.md deleted file mode 100644 index 0a5432f5708..00000000000 --- a/.github/ISSUE_TEMPLATE/doc.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -name: 'Documentation Issue' -about: 'Report missing, erroneous docs, broken links or propose new go-ipfs docs' -labels: topic/docs-ipfs, need/triage ---- - - -#### Location - - - -#### Description - - diff --git a/.github/ISSUE_TEMPLATE/doc.yml b/.github/ISSUE_TEMPLATE/doc.yml new file mode 100644 index 00000000000..dc9034fae5a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/doc.yml @@ -0,0 +1,29 @@ +name: Documentation Issue +description: Report missing, erroneous docs, broken links or propose new go-ipfs docs. +labels: + - topic/docs-ipfs + - need/triage +body: + - type: markdown + attributes: + value: | + Problems with documentation on https://docs.ipfs.io should be reported to https://github.com/ipfs/ipfs-docs + - type: checkboxes + attributes: + label: Checklist + description: Please verify the following. + options: + - label: I am reporting a documentation issue in this repo, not https://docs.ipfs.io. + required: true + - label: I have searched on the [issue tracker](https://github.com/ipfs/go-ipfs/issues?q=is%3Aissue) for my issue. + required: true + - type: input + attributes: + label: Location + description: | + If possible, please provide a link to the documentation issue. + - type: textarea + attributes: + label: Description + description: | + Please describe your issue. diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md deleted file mode 100644 index 8c1c31c0d9d..00000000000 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: 'Enhancement' -about: 'Suggest an improvement to an existing go-ipfs feature.' -labels: kind/enhancement ---- - - diff --git a/.github/ISSUE_TEMPLATE/enhancement.yml b/.github/ISSUE_TEMPLATE/enhancement.yml new file mode 100644 index 00000000000..f1be06c358c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement.yml @@ -0,0 +1,33 @@ +name: Enhancement +description: Suggest an improvement to an existing go-ipfs feature. +labels: + - kind/enhancement +body: + - type: markdown + attributes: + value: | + Suggest an enhancement to go-ipfs (the program). If you'd like to suggest an improvement to the IPFS protocol, please discuss it on [the forum](https://discuss.ipfs.io). + + Issues in this repo must be specific, actionable, and well motivated. They should be starting points for _building_ new features, not brainstorming ideas. + + If you have an idea you'd like to discuss, please open a new thread on [the forum](https://discuss.ipfs.io). + + **Example:** + + > Reduce memory usage of `ipfs cat` (specific) by buffering less in ... (actionable). This would let me run go-ipfs on my Raspberry Pi (motivated). + - type: checkboxes + attributes: + label: Checklist + description: Please verify the following. + options: + - label: My issue is specific & actionable. + required: true + - label: I am not suggesting a protocol enhancement. + required: true + - label: I have searched on the [issue tracker](https://github.com/ipfs/go-ipfs/issues?q=is%3Aissue) for my issue. + required: true + - type: textarea + attributes: + label: Description + description: | + Please describe your idea. When requesting an enhancement, please be sure to include your motivation and try to be as specific as possible. diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md deleted file mode 100644 index 0cd141dc706..00000000000 --- a/.github/ISSUE_TEMPLATE/feature.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -name: 'Feature' -about: 'Suggest a new feature in go-ipfs.' -labels: kind/feature, need/triage ---- - - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000000..62414cabb02 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,34 @@ +name: Feature +description: Suggest a new feature in go-ipfs. +labels: + - kind/feature +body: + - type: markdown + attributes: + value: | + Suggest a new feature in go-ipfs (the program). If you'd like to suggest an improvement to the IPFS protocol, please discuss it on [the forum](https://discuss.ipfs.io). + + Issues in this repo must be specific, actionable, and well motivated. They should be starting points for _building_ new features, not brainstorming ideas. + + If you have an idea you'd like to discuss, please open a new thread on [the forum](https://discuss.ipfs.io). + + **Example:** + + > Add deduplication-optimized chunking of tar files in `ipfs add` (specific) by examining tar headers ... (actionable). This would let me efficiently store and update many versions of code archives (motivated). + + - type: checkboxes + attributes: + label: Checklist + description: Please verify the following. + options: + - label: My issue is specific & actionable. + required: true + - label: I am not suggesting a protocol enhancement. + required: true + - label: I have searched on the [issue tracker](https://github.com/ipfs/go-ipfs/issues?q=is%3Aissue) for my issue. + required: true + - type: textarea + attributes: + label: Description + description: | + Please describe your idea. When requesting a feature, please be sure to include your motivation and and a concrete description of how the feature should work. diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000000..8d2a95a9a60 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,34 @@ +name: Publish Docker image + +on: + workflow_dispatch: + push: + branches: + - 'master' + - 'bifrost-*' + tags: + - 'v*' + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + env: + IMAGE_NAME: ipfs/go-ipfs + WIP_IMAGE_TAG: wip + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Build wip Docker image + run: docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG . + + - name: Log in to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Publish Docker Image to Docker Hub + run: ./bin/push-docker-tags.sh $(date -u +%F) + diff --git a/CHANGELOG.md b/CHANGELOG.md index c1fac6a714f..07e8b6bcf2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,610 @@ # go-ipfs changelog +## v0.10.0 2021-09-30 + +We're happy to announce go-ipfs 0.10.0. This release brings some big changes to the IPLD internals of go-ipfs that make working with non-UnixFS DAGs easier than ever. There are also a variety of new commands and configuration options available. + +As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. + +### 🛠 TLDR: BREAKING CHANGES + +- `ipfs dag get` + - default output changed to [`dag-json`](https://ipld.io/specs/codecs/dag-json/spec/) + - dag-pb (e.g. unixfs) field names changed - impacts userland code that works with `dag-pb` objects returned by `dag get` + - no longer emits an additional new-line character at the end of the data output +- `ipfs dag put` + - defaults changed to reduce ambiguity and surprises: input is now assumed to be [`dag-json`](https://ipld.io/specs/codecs/dag-json/spec/), and data is serialized to [`dag-cbor`](https://ipld.io/specs/codecs/dag-cbor/spec/) at rest. + - `--format` and `--input-enc` were removed and replaced with `--store-codec` and `--input-codec` + - codec names now match the ones defined in the [multicodec table](https://github.com/multiformats/multicodec/blob/master/table.csv) + - dag-pb (e.g. unixfs) field names changed - impacts userland code that works with `dag-pb` objects stored via `dag put` + +Keep reading to learn more details. + +### 🔦 Highlights + +#### 🌲 IPLD Levels Up + +The handling of data serialization as well as many aspects of DAG traversal and pathing have been migrated from older libraries, including [go-merkledag](https://github.com/ipfs/go-merkledag) and [go-ipld-format](https://github.com/ipfs/go-ipld-format) to the new **[go-ipld-prime](https://github.com/ipld/go-ipld-prime)** library and its components. This allows us to use many of the newer tools afforded by go-ipld-prime, stricter and more uniform codec implementations, support for additional (pluggable) codecs, and some minor performance improvements. + +This is significant refactor of a core component that touches many parts of IPFS, and does come with some **breaking changes**: + +* **IPLD plugins**: + * The `PluginIPLD` interface has been changed to utilize go-ipld-prime. There is a demonstration of the change in the [bundled git plugin](./plugin/plugins/git/). +* **The semantics of `dag put` and `dag get` change**: + * `dag get` now takes the `output-codec` option which accepts a [multicodec](https://docs.ipfs.io/concepts/glossary/#multicodec) name used to encode the output. By default this is `dag-json`, which is a strict and deterministic subset of JSON created by the IPLD team. Users may notice differences from the previously plain Go JSON output, particularly where bytes are concerned which are now encoded using a form similar to CIDs: `{"/":{"bytes":"unpadded-base64-bytes"}}` rather than the previously Go-specific plain padded base64 string. See the [dag-json specification](https://ipld.io/specs/codecs/dag-json/spec/) for an explanation of these forms. + * `dag get` no longer prints an additional new-line character at the end of the encoded block output. This means that the output as presented by `dag get` are the exact bytes of the requested node. A round-trip of such bytes back in through `dag put` using the same codec should result in the same CID. + * `dag put` uses the `input-codec` option to specify the multicodec name of the format data is being provided in, and the `store-codec` option to specify the multicodec name of the format the data should be stored in at rest. These formerly defaulted to `json` and `cbor` respectively. They now default to `dag-json` and `dag-cbor` respectively but may be changed to any supported codec (bundled or loaded via plugin) by its [multicodec name](https://github.com/multiformats/multicodec/blob/master/table.csv). + * The `json` and `cbor` multicodec names (as used by `input-enc` and `format` options) are now no longer aliases for `dag-json` and `dag-cbor` respectively. Instead, they now refer to their proper [multicodec](https://github.com/multiformats/multicodec/blob/master/table.csv) types. `cbor` refers to a plain CBOR format, which will not encode CIDs and does not have strict deterministic encoding rules. `json` is a plain JSON format, which also won't encode CIDs and will encode bytes in the Go-specific padded base64 string format rather than the dag-json method of byte encoding. See https://ipld.io/specs/codecs/ for more information on IPLD codecs. + * `protobuf` is no longer used as the codec name for `dag-pb` + * The codec name `raw` is used to mean Bytes in the [IPLD Data Model](https://github.com/ipld/specs/blob/master/data-model-layer/data-model.md#bytes-kind) +* **UnixFS refactor**. The **dag-pb codec**, which is used to encode UnixFS data for IPFS, is now represented through the `dag` API in a form that mirrors the protobuf schema used to define the binary format. This unifies the implementations and specification of dag-pb across the IPLD and IPFS stacks. Previously, additional layers of code for file and directory handling within IPFS between protobuf serialization and UnixFS obscured the protobuf representation. Much of this code has now been replaced and there are fewer layers of transformation. This means that interacting with dag-pb data via the `dag` API will use different forms: + * Previously, using `dag get` on a dag-pb block would present the block serialized as JSON as `{"data":"padded-base64-bytes","links":[{"Name":"foo","Size":100,"Cid":{"/":"Qm..."}},...]}`. + * Now, the dag-pb data with dag-json codec for output will be serialized using the data model from the [dag-pb specification](https://ipld.io/specs/codecs/dag-pb/spec/): `{"Data":{"/":{"bytes":"unpadded-base64-bytes"}},"Links":[{"Name":"foo","Tsize":100,"Hash":{"/":"Qm..."}},...]}`. Aside from the change in byte formatting, most field names have changed: `data` → `Data`, `links` → `Links`, `Size` → `Tsize`, `Cid` → `Hash`. Note that this output can be changed now using the `output-codec` option to specify an alternative codec. + * Similarly, using `dag put` and a `store-codec` option of `dag-pb` now requires that the input conform to this dag-pb specified form. Previously, input using `{"data":"...","links":[...]}` was accepted, now it must be `{"Data":"...","Links":[...]}`. + * Previously it was not possible to use paths to navigate to any of these properties of a dag-pb node, the only possible paths were named links, e.g. `dag get QmFoo/NamedLink` where `NamedLink` was one of the links whose name was `NamedLink`. This functionality remains the same, but by prefixing the path with `/ipld/` we enter data model pathing semantics and can `dag get /ipld/QmFoo/Links/0/Hash` to navigate to links or `/ipld/QmFoo/Data` to simply retrieve the data section of the node, for example. + * ℹ See the [dag-pb specification](https://ipld.io/specs/codecs/dag-pb/) for details on the codec and its data model representation. + * ℹ See this [detailed write-up](https://github.com/ipld/ipld/blob/master/design/tricky-choices/dag-pb-forms-impl-and-use.md) for further background on these changes. + +#### Ⓜ Multibase Command + +go-ipfs now provides utility commands for working with [multibase](https://docs.ipfs.io/concepts/glossary/#multibase): + +```console +$ echo -n hello | ipfs multibase encode -b base16 > file-mbase16 +$ cat file-mbase16 +f68656c6c6f + +$ ipfs multibase decode file-mbase16 +hello + +$ cat file-mbase16 | ipfs multibase decode +hello + +$ ipfs multibase transcode -b base2 file-mbase16 +00110100001100101011011000110110001101111 +``` + +See `ipfs multibase --help` for more examples. + +#### 🔨 Bitswap now supports greater configurability + +This release adds an [`Internal` section](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#internal) to the configuration file that is designed to help advanced users optimize their setups without needing a custom binary. The `Internal` section is not guaranteed to be the same from release to release and may not be covered by migrations. If you use the `Internal` section you should be making sure to check the config documentation between releases for any changes. + +#### 🐚 Programmatic shell completions command + +`ipfs commands completion bash` will generate a bash completion script for go-ipfs commands + +#### 📜 Profile collection command + +Performance profiles can now be collected using `ipfs diag profile`. If you need to do some debugging or have an issue to submit the collected profiles are very useful to have around. + +#### 🍎 Mac OS notarized binaries + +The go-ipfs and related migration binaries (for both Intel and Apple Sillicon) are now signed and notarized to make Mac OS installation easier. + +#### 👨‍👩‍👦 Improved MDNS + +There is a completed implementation of the revised libp2p MDNS spec. This should result in better MDNS discovery and better local/offline operation as a result. + +#### 🚗 CAR import statistics + +`dag import` command now supports `--stats` option which will include the number of imported blocks and their total size in the output. + +#### 🕸 Peering command + +This release adds `swarm peering` command for easy management of the peering subsystem. Peer in the peering subsystem is maintained to be connected at all times, and gets reconnected on disconnect with a back-off. + +See `ipfs swarm peering --help` for more details. + +### Changelog + +- github.com/ipfs/go-ipfs: + - fuse: load unixfs adls as their dagpb substrates + - enable the legacy mDNS implementation + - test: add dag get --ouput-codec test + - change ipfs dag get flag name from format to output-codec + - test: check behavior of loading UnixFS sharded directories with missing shards + - remove dag put option shortcuts + - change names of ipfs dag put flags to make changes clearer + - feat: dag import --stats (#8237) ([ipfs/go-ipfs#8237](https://github.com/ipfs/go-ipfs/pull/8237)) + - feat: ipfs-webui v2.13.0 (#8430) ([ipfs/go-ipfs#8430](https://github.com/ipfs/go-ipfs/pull/8430)) + - feat(cli): add daemon option --agent-version-suffix (#8419) ([ipfs/go-ipfs#8419](https://github.com/ipfs/go-ipfs/pull/8419)) + - feat: multibase transcode command (#8403) ([ipfs/go-ipfs#8403](https://github.com/ipfs/go-ipfs/pull/8403)) + - fix: take the lock while listing peers + - feature: 'ipfs swarm peering' command (#8147) ([ipfs/go-ipfs#8147](https://github.com/ipfs/go-ipfs/pull/8147)) + - fix(sharness): add extra check in flush=false in files write + - chore: update IPFS Desktop testing steps (#8393) ([ipfs/go-ipfs#8393](https://github.com/ipfs/go-ipfs/pull/8393)) + - add more buttons; remove some sections covered in the docs; general cleanup + - Cosmetic fixups in examples (#8325) ([ipfs/go-ipfs#8325](https://github.com/ipfs/go-ipfs/pull/8325)) + - perf: use performance-enhancing FUSE mount options + - ci: publish Docker images for bifrost-* branches + - chore: add comments to peerlog plugin about being unsupported + - test: add unit tests for peerlog config parsing + - ci: preload peerlog plugin, disable by default + - fix(mkreleaselog): specify the parent commit when diffing + - update go-libp2p to v0.15.0-rc.1 ([ipfs/go-ipfs#8354](https://github.com/ipfs/go-ipfs/pull/8354)) + - feat: add 'ipfs multibase' commands (#8180) ([ipfs/go-ipfs#8180](https://github.com/ipfs/go-ipfs/pull/8180)) + - support bitswap configurability (#8268) ([ipfs/go-ipfs#8268](https://github.com/ipfs/go-ipfs/pull/8268)) + - IPLD Prime In IPFS: Target Merge Branch (#7976) ([ipfs/go-ipfs#7976](https://github.com/ipfs/go-ipfs/pull/7976)) + - ci: upgrade to Go 1.16.7 on CI ([ipfs/go-ipfs#8324](https://github.com/ipfs/go-ipfs/pull/8324)) + - Add flag to create parent directories in files cp command ([ipfs/go-ipfs#8340](https://github.com/ipfs/go-ipfs/pull/8340)) + - fix: avoid out of bounds error when rendering short hashes ([ipfs/go-ipfs#8318](https://github.com/ipfs/go-ipfs/pull/8318)) + - fix: remove some deprecated calls ([ipfs/go-ipfs#8296](https://github.com/ipfs/go-ipfs/pull/8296)) + - perf: set an appropriate capacity ([ipfs/go-ipfs#8244](https://github.com/ipfs/go-ipfs/pull/8244)) + - Fix: Use a pointer type on IpfsNode.Peering ([ipfs/go-ipfs#8331](https://github.com/ipfs/go-ipfs/pull/8331)) + - fix: macos notarized fs-repo-migrations (#8333) ([ipfs/go-ipfs#8333](https://github.com/ipfs/go-ipfs/pull/8333)) + - README.md: Add MacPorts to install section ([ipfs/go-ipfs#8220](https://github.com/ipfs/go-ipfs/pull/8220)) + - feat: register first block metric by default ([ipfs/go-ipfs#8332](https://github.com/ipfs/go-ipfs/pull/8332)) + - Build a go-ipfs:extras docker image ([ipfs/go-ipfs#8142](https://github.com/ipfs/go-ipfs/pull/8142)) + - fix/go-ipfs-as-a-library ([ipfs/go-ipfs#8266](https://github.com/ipfs/go-ipfs/pull/8266)) + - Expose additional migration APIs (#8153) ([ipfs/go-ipfs#8153](https://github.com/ipfs/go-ipfs/pull/8153)) + - point ipfs to pinner that syncs on every pin (#8231) ([ipfs/go-ipfs#8231](https://github.com/ipfs/go-ipfs/pull/8231)) + - docs: chocolatey package name + - Disambiguate online/offline naming in sharness tests ([ipfs/go-ipfs#8254](https://github.com/ipfs/go-ipfs/pull/8254)) + - Rename DOCKER_HOST to TEST_DOCKER_HOST to avoid conflicts ([ipfs/go-ipfs#8283](https://github.com/ipfs/go-ipfs/pull/8283)) + - feat: add an "ipfs diag profile" command ([ipfs/go-ipfs#8291](https://github.com/ipfs/go-ipfs/pull/8291)) + - Merge branch 'release' + - feat: improve mkreleaslog ([ipfs/go-ipfs#8290](https://github.com/ipfs/go-ipfs/pull/8290)) + - Add test with expected failure for #3503 ([ipfs/go-ipfs#8280](https://github.com/ipfs/go-ipfs/pull/8280)) + - Create PATCH_RELEASE_TEMPLATE.md + - fix document error ([ipfs/go-ipfs#8271](https://github.com/ipfs/go-ipfs/pull/8271)) + - feat: webui v2.12.4 + - programmatic shell completions ([ipfs/go-ipfs#8043](https://github.com/ipfs/go-ipfs/pull/8043)) + - test: gateway response for bafkqaaa + - doc(README): update chat links (and misc fixes) ([ipfs/go-ipfs#8222](https://github.com/ipfs/go-ipfs/pull/8222)) + - link to the actual doc (#8126) ([ipfs/go-ipfs#8126](https://github.com/ipfs/go-ipfs/pull/8126)) + - Improve peer hints for pin remote add (#8143) ([ipfs/go-ipfs#8143](https://github.com/ipfs/go-ipfs/pull/8143)) + - fix(mkreleaselog): support multiple commit authors ([ipfs/go-ipfs#8214](https://github.com/ipfs/go-ipfs/pull/8214)) + - fix(mkreleaselog): handle commit 0 ([ipfs/go-ipfs#8121](https://github.com/ipfs/go-ipfs/pull/8121)) + - bump snap to build with Go 1.16 + - chore: update CHANGELOG + - chore: switch tar-utils dep to ipfs org + - feat: print error on bootstrap failure ([ipfs/go-ipfs#8166](https://github.com/ipfs/go-ipfs/pull/8166)) + - fix: typo in migration error + - refactor: improved humanNumber and humanSI + - feat: humanized durations in stat provide + - feat: humanized numbers in stat provide + - feat: add a text output encoding for the stats provide command + - fix: webui-2.12.3 + - refactor(pinmfs): log error if pre-existing pin failed (#8056) ([ipfs/go-ipfs#8056](https://github.com/ipfs/go-ipfs/pull/8056)) + - config.md: fix typos/improve wording ([ipfs/go-ipfs#8031](https://github.com/ipfs/go-ipfs/pull/8031)) + - fix(peering_test) : Fix the peering_test to check the connection explicitly added ([ipfs/go-ipfs#8140](https://github.com/ipfs/go-ipfs/pull/8140)) + - build: ignore generated files in changelog ([ipfs/go-ipfs#7712](https://github.com/ipfs/go-ipfs/pull/7712)) + - update version to 0.10.0-dev ([ipfs/go-ipfs#8136](https://github.com/ipfs/go-ipfs/pull/8136)) +- github.com/ipfs/go-bitswap (v0.3.4 -> v0.4.0): + - More stats, knobs and tunings (#514) ([ipfs/go-bitswap#514](https://github.com/ipfs/go-bitswap/pull/514)) + - fix: fix a map access race condition in the want index ([ipfs/go-bitswap#523](https://github.com/ipfs/go-bitswap/pull/523)) + - fix: make blockstore cancel test less timing dependent ([ipfs/go-bitswap#507](https://github.com/ipfs/go-bitswap/pull/507)) + - fix(decision): fix a datarace on disconnect ([ipfs/go-bitswap#508](https://github.com/ipfs/go-bitswap/pull/508)) + - optimize the lookup which peers are waiting for a given block ([ipfs/go-bitswap#486](https://github.com/ipfs/go-bitswap/pull/486)) + - fix: hold the task worker lock when starting task workers ([ipfs/go-bitswap#504](https://github.com/ipfs/go-bitswap/pull/504)) + - fix: Nil dereference while using SetSendDontHaves ([ipfs/go-bitswap#488](https://github.com/ipfs/go-bitswap/pull/488)) + - Fix flaky tests in message queue ([ipfs/go-bitswap#497](https://github.com/ipfs/go-bitswap/pull/497)) + - Fix flaky DontHaveTimeoutManger tests ([ipfs/go-bitswap#495](https://github.com/ipfs/go-bitswap/pull/495)) + - sync: update CI config files ([ipfs/go-bitswap#485](https://github.com/ipfs/go-bitswap/pull/485)) +- github.com/ipfs/go-blockservice (v0.1.4 -> v0.1.7): + - update go-bitswap to v0.3.4 ([ipfs/go-blockservice#78](https://github.com/ipfs/go-blockservice/pull/78)) + - fix staticcheck ([ipfs/go-blockservice#75](https://github.com/ipfs/go-blockservice/pull/75)) + - fix: handle missing session exchange in Session ([ipfs/go-blockservice#73](https://github.com/ipfs/go-blockservice/pull/73)) +- github.com/ipfs/go-datastore (v0.4.5 -> v0.4.6): + - sync: update CI config files ([ipfs/go-datastore#175](https://github.com/ipfs/go-datastore/pull/175)) + - speedup tests ([ipfs/go-datastore#177](https://github.com/ipfs/go-datastore/pull/177)) + - test: reduce element count when the race detector is enabled ([ipfs/go-datastore#176](https://github.com/ipfs/go-datastore/pull/176)) + - fix staticcheck ([ipfs/go-datastore#173](https://github.com/ipfs/go-datastore/pull/173)) + - remove Makefile ([ipfs/go-datastore#172](https://github.com/ipfs/go-datastore/pull/172)) +- github.com/ipfs/go-ds-badger (v0.2.6 -> v0.2.7): + - Log start and end of GC rounds ([ipfs/go-ds-badger#115](https://github.com/ipfs/go-ds-badger/pull/115)) +- github.com/ipfs/go-fs-lock (v0.0.6 -> v0.0.7): + - chore: update log ([ipfs/go-fs-lock#24](https://github.com/ipfs/go-fs-lock/pull/24)) + - sync: update CI config files ([ipfs/go-fs-lock#21](https://github.com/ipfs/go-fs-lock/pull/21)) + - fix TestLockedByOthers on Windows ([ipfs/go-fs-lock#19](https://github.com/ipfs/go-fs-lock/pull/19)) +- github.com/ipfs/go-ipfs-config (v0.14.0 -> v0.16.0): + - feat: add Internal and Internal.Bitswap config options + - feat: add an OptionalInteger type + - fix: make sure the Priority type properly implements the JSON marshal/unmarshal interfaces + - fix: remove deprecated calls ([ipfs/go-ipfs-config#138](https://github.com/ipfs/go-ipfs-config/pull/138)) + - sync: update CI config files ([ipfs/go-ipfs-config#132](https://github.com/ipfs/go-ipfs-config/pull/132)) + - remove period, fix staticcheck ([ipfs/go-ipfs-config#131](https://github.com/ipfs/go-ipfs-config/pull/131)) +- github.com/ipfs/go-ipfs-pinner (v0.1.1 -> v0.1.2): + - Fix/minimize rebuild (#15) ([ipfs/go-ipfs-pinner#15](https://github.com/ipfs/go-ipfs-pinner/pull/15)) + - Define ErrNotPinned alongside the Pinner interface + - fix staticcheck ([ipfs/go-ipfs-pinner#11](https://github.com/ipfs/go-ipfs-pinner/pull/11)) + - fix: remove the rest of the pb backed pinner ([ipfs/go-ipfs-pinner#9](https://github.com/ipfs/go-ipfs-pinner/pull/9)) + - Remove old ipldpinner that has been replaced by dspinner ([ipfs/go-ipfs-pinner#7](https://github.com/ipfs/go-ipfs-pinner/pull/7)) + - optimize CheckIfPinned ([ipfs/go-ipfs-pinner#6](https://github.com/ipfs/go-ipfs-pinner/pull/6)) +- github.com/ipfs/go-ipfs-provider (v0.5.1 -> v0.6.1): + - Update to IPLD Prime (#32) ([ipfs/go-ipfs-provider#32](https://github.com/ipfs/go-ipfs-provider/pull/32)) +- github.com/ipfs/go-ipld-git (v0.0.4 -> v0.1.1): + - return ErrUnexpectedEOF when Decode input is too short + - Update go-ipld-git to a go-ipld-prime codec (#46) ([ipfs/go-ipld-git#46](https://github.com/ipfs/go-ipld-git/pull/46)) + - fix staticcheck ([ipfs/go-ipld-git#49](https://github.com/ipfs/go-ipld-git/pull/49)) + - change WriteTo to the standard signature ([ipfs/go-ipld-git#47](https://github.com/ipfs/go-ipld-git/pull/47)) + - don't copy mutexes ([ipfs/go-ipld-git#48](https://github.com/ipfs/go-ipld-git/pull/48)) +- github.com/ipfs/go-ipns (v0.1.0 -> v0.1.2): + - fix: remove deprecated calls ([ipfs/go-ipns#30](https://github.com/ipfs/go-ipns/pull/30)) + - remove Makefile ([ipfs/go-ipns#27](https://github.com/ipfs/go-ipns/pull/27)) +- github.com/ipfs/go-log/v2 (v2.1.3 -> v2.3.0): + - Stop defaulting to color output on non-TTY ([ipfs/go-log#116](https://github.com/ipfs/go-log/pull/116)) + - feat: add ability to use custom zap core ([ipfs/go-log#114](https://github.com/ipfs/go-log/pull/114)) + - fix staticcheck ([ipfs/go-log#112](https://github.com/ipfs/go-log/pull/112)) + - test: fix flaky label test ([ipfs/go-log#111](https://github.com/ipfs/go-log/pull/111)) + - per-subsystem log-levels ([ipfs/go-log#109](https://github.com/ipfs/go-log/pull/109)) + - fix: don't panic on invalid log labels ([ipfs/go-log#110](https://github.com/ipfs/go-log/pull/110)) +- github.com/ipfs/go-merkledag (v0.3.2 -> v0.4.0): + - Use IPLD-prime: target merge branch ([ipfs/go-merkledag#67](https://github.com/ipfs/go-merkledag/pull/67)) + - sync: update CI config files ([ipfs/go-merkledag#70](https://github.com/ipfs/go-merkledag/pull/70)) + - staticcheck ([ipfs/go-merkledag#69](https://github.com/ipfs/go-merkledag/pull/69)) + - Fix bug in dagutils MergeDiffs. (#59) ([ipfs/go-merkledag#59](https://github.com/ipfs/go-merkledag/pull/59)) + - chore: add tests to verify allowable data layouts ([ipfs/go-merkledag#58](https://github.com/ipfs/go-merkledag/pull/58)) +- github.com/ipfs/go-namesys (v0.3.0 -> v0.3.1): + - fix: remove deprecated call to pk.Bytes ([ipfs/go-namesys#19](https://github.com/ipfs/go-namesys/pull/19)) +- github.com/ipfs/go-path (v0.0.9 -> v0.1.2): + - fix: give one minute timeouts to function calls instead of block retrievals ([ipfs/go-path#44](https://github.com/ipfs/go-path/pull/44)) + - IPLD Prime In IPFS: Target Merge Branch (#36) ([ipfs/go-path#36](https://github.com/ipfs/go-path/pull/36)) + - remove Makefile ([ipfs/go-path#40](https://github.com/ipfs/go-path/pull/40)) + - sync: update CI config files ([ipfs/go-path#39](https://github.com/ipfs/go-path/pull/39)) +- github.com/ipfs/go-peertaskqueue (v0.2.0 -> v0.4.0): + - add stats + - Have a configurable maximum active work per peer ([ipfs/go-peertaskqueue#10](https://github.com/ipfs/go-peertaskqueue/pull/10)) + - sync: update CI config files ([ipfs/go-peertaskqueue#13](https://github.com/ipfs/go-peertaskqueue/pull/13)) + - fix staticcheck ([ipfs/go-peertaskqueue#12](https://github.com/ipfs/go-peertaskqueue/pull/12)) + - fix go vet ([ipfs/go-peertaskqueue#11](https://github.com/ipfs/go-peertaskqueue/pull/11)) +- github.com/ipfs/go-unixfsnode (null -> v1.1.3): + - make UnixFSHAMTShard implement the ADL interface (#11) ([ipfs/go-unixfsnode#11](https://github.com/ipfs/go-unixfsnode/pull/11)) +- github.com/ipfs/interface-go-ipfs-core (v0.4.0 -> v0.5.1): + - IPLD In IPFS: Target Merge Branch (#67) ([ipfs/interface-go-ipfs-core#67](https://github.com/ipfs/interface-go-ipfs-core/pull/67)) + - fix staticcheck ([ipfs/interface-go-ipfs-core#72](https://github.com/ipfs/interface-go-ipfs-core/pull/72)) + - remove Makefile ([ipfs/interface-go-ipfs-core#70](https://github.com/ipfs/interface-go-ipfs-core/pull/70)) +- github.com/ipld/go-codec-dagpb (v1.2.0 -> v1.3.0): + - fix staticcheck warnings ([ipld/go-codec-dagpb#29](https://github.com/ipld/go-codec-dagpb/pull/29)) + - update go-ipld-prime, use go:generate + - allow decoding PBNode fields in any order + - expose APIs without Reader/Writer overhead + - preallocate 1KiB on the stack for marshals + - encode directly with a []byte + - decode directly with a []byte + - remove unnecessary xerrors dep +- github.com/ipld/go-ipld-prime (v0.9.1-0.20210324083106-dc342a9917db -> v0.12.2): + - Printer feature ([ipld/go-ipld-prime#238](https://github.com/ipld/go-ipld-prime/pull/238)) + - schema: keep TypeSystem names ordered + - schema/dmt: redesign with bindnode and add Compile + - codec: make cbor and json codecs use ErrUnexpectedEOF + - bindnode: fix for stringjoin struct emission when first field is the empty string ([ipld/go-ipld-prime#239](https://github.com/ipld/go-ipld-prime/pull/239)) + - schema: typekind names are not capitalized. + - Bindnode fixes continued ([ipld/go-ipld-prime#233](https://github.com/ipld/go-ipld-prime/pull/233)) + - helper methods for encoding and decoding ([ipld/go-ipld-prime#232](https://github.com/ipld/go-ipld-prime/pull/232)) + - mark v0.12.0 + - Major refactor: extract datamodel package. + ([ipld/go-ipld-prime#228](https://github.com/ipld/go-ipld-prime/pull/228)) + - Fix ExploreRecursive stopAt condition, add tests, add error return to Explore (#229) ([ipld/go-ipld-prime#229](https://github.com/ipld/go-ipld-prime/pull/229)) + - selector: add tests which are driven by language-agnostic spec fixtures. ([ipld/go-ipld-prime#231](https://github.com/ipld/go-ipld-prime/pull/231)) + - selector: Improve docs for implementors. (#227) ([ipld/go-ipld-prime#227](https://github.com/ipld/go-ipld-prime/pull/227)) + - Bindnode fixes of opportunity ([ipld/go-ipld-prime#226](https://github.com/ipld/go-ipld-prime/pull/226)) + - node/bindnode: redesign the shape of unions in Go ([ipld/go-ipld-prime#223](https://github.com/ipld/go-ipld-prime/pull/223)) + - summary of the v0.11.0 changelog should holler even more about how cool bindnode is. + - mark v0.11.0 + - node/bindnode: mark as experimental in its godoc. + - codecs: more docs, a terminology guide, consistency in options. ([ipld/go-ipld-prime#221](https://github.com/ipld/go-ipld-prime/pull/221)) + - Changelog backfill. + - selectors: docs enhancements, new construction helpers. ([ipld/go-ipld-prime#199](https://github.com/ipld/go-ipld-prime/pull/199)) + - Changelog backfill. + - Allow parsing of single Null tokens from refmt + - Add link conditions for 'stop-at' expression in ExploreRecursive selector ([ipld/go-ipld-prime#214](https://github.com/ipld/go-ipld-prime/pull/214)) + - Remove base64 padding for dag-json bytes as per spec + - node/bindnode: temporarily skip Links schema test + - test: add test for traversal of typed node links + - fix: typed links LinkTargetNodePrototype should return ReferencedType + - Make `go vet` happy + - Add MapSortMode to MarshalOptions + - Add {Unm,M}arshalOptions for explicit mode switching for cbor vs dagcbor + - Sort map entries marshalling dag-cbor + - node/bindnode: first pass at inferring IPLD schemas + - Add {Unm,M}arshalOptions for explicit mode switching for json vs dagjson + - Make tests pass with sorted dag-json output + - Sort map entries marshalling dag-json + - Simplify refmt usage + - Fix failing test using dagjson encoding + - Fix some failing tests using dagjson + - Remove pretty-printing + - Update readme linking to specs and meta repo. + - Fix example names so they render on go.pkg.dev. + - fluent/quip: remove in favor of qp + - node/basic: add Chooser + - schema: add TypedPrototype + - node/bindnode: rethink and better document APIs + - node/tests: cover yet more interface methods + - node/tests: cover more error cases for scalar kinds + - node/tests: add more extensive scalar kind tests + - node/bindnode: start running all schema tests + - mark v0.10.0 + - More changelog grooming. + - Changelog grooming. + - node/tests: put most of the schema test cases here + - Add more explicit discussion of indicies to ListIterator. + - node/bindnode: start of a reflect-based Node implementation + - add DeepEqual and start using it in tests + - Add enumerate methods to the multicodec registries. ([ipld/go-ipld-prime#176](https://github.com/ipld/go-ipld-prime/pull/176)) + - Make a multicodec.Registry type available. ([ipld/go-ipld-prime#172](https://github.com/ipld/go-ipld-prime/pull/172)) + - fluent/qp: don't panic on string panics + - Allow emitting & parsing of bytes per dagjson codec spec ([ipld/go-ipld-prime#166](https://github.com/ipld/go-ipld-prime/pull/166)) + - Package docs for dag-cbor. + - Update package docs. + - schema/gen/go: apply gofmt automatically ([ipld/go-ipld-prime#163](https://github.com/ipld/go-ipld-prime/pull/163)) + - schema/gen/go: fix remaining vet warnings on generated code + - schema/gen/go: batch file writes via a bytes.Buffer ([ipld/go-ipld-prime#161](https://github.com/ipld/go-ipld-prime/pull/161)) + - schema/gen/go: avoid Maybe pointers for small types + - fix readme formatting typo + - feat(linksystem): add reification to LinkSystem ([ipld/go-ipld-prime#158](https://github.com/ipld/go-ipld-prime/pull/158)) +- github.com/libp2p/go-addr-util (v0.0.2 -> v0.1.0): + - stop using the deprecated go-multiaddr-net package ([libp2p/go-addr-util#34](https://github.com/libp2p/go-addr-util/pull/34)) + - Remove `IsFDCostlyTransport` ([libp2p/go-addr-util#31](https://github.com/libp2p/go-addr-util/pull/31)) +- github.com/libp2p/go-libp2p (v0.14.3 -> v0.15.0): + - chore: update go-tcp-transport to v0.2.8 + - implement the new mDNS spec, move the old mDNS implementation (#1161) ([libp2p/go-libp2p#1161](https://github.com/libp2p/go-libp2p/pull/1161)) + - remove deprecated basichost.New constructor ([libp2p/go-libp2p#1156](https://github.com/libp2p/go-libp2p/pull/1156)) + - Make BasicHost.evtLocalAddrsUpdated event emitter stateful. ([libp2p/go-libp2p#1147](https://github.com/libp2p/go-libp2p/pull/1147)) + - fix: deflake multipro echo test ([libp2p/go-libp2p#1149](https://github.com/libp2p/go-libp2p/pull/1149)) + - fix(basic_host): stream not closed when context done ([libp2p/go-libp2p#1148](https://github.com/libp2p/go-libp2p/pull/1148)) + - chore: update deps ([libp2p/go-libp2p#1141](https://github.com/libp2p/go-libp2p/pull/1141)) + - remove secio from examples ([libp2p/go-libp2p#1143](https://github.com/libp2p/go-libp2p/pull/1143)) + - remove deprecated Filter option ([libp2p/go-libp2p#1132](https://github.com/libp2p/go-libp2p/pull/1132)) + - fix: remove deprecated call ([libp2p/go-libp2p#1136](https://github.com/libp2p/go-libp2p/pull/1136)) + - test: fix flaky example test ([libp2p/go-libp2p#1135](https://github.com/libp2p/go-libp2p/pull/1135)) + - remove deprecated identify.ClientVersion ([libp2p/go-libp2p#1133](https://github.com/libp2p/go-libp2p/pull/1133)) + - remove Go version requirement and note about Go modules from README ([libp2p/go-libp2p#1126](https://github.com/libp2p/go-libp2p/pull/1126)) + - Error assignment fix ([libp2p/go-libp2p#1124](https://github.com/libp2p/go-libp2p/pull/1124)) + - perf/basic_host: Don't handle address change if we hasn't anyone ([libp2p/go-libp2p#1115](https://github.com/libp2p/go-libp2p/pull/1115)) +- github.com/libp2p/go-libp2p-core (v0.8.5 -> v0.9.0): + - feat: remove unused metrics (#208) ([libp2p/go-libp2p-core#208](https://github.com/libp2p/go-libp2p-core/pull/208)) + - feat: keep addresses for longer (#207) ([libp2p/go-libp2p-core#207](https://github.com/libp2p/go-libp2p-core/pull/207)) + - remove deprecated key stretching struct / function (#203) ([libp2p/go-libp2p-core#203](https://github.com/libp2p/go-libp2p-core/pull/203)) + - remove deprecated Bytes method from the Key interface (#204) ([libp2p/go-libp2p-core#204](https://github.com/libp2p/go-libp2p-core/pull/204)) + - remove deprecated functions in the peer package (#205) ([libp2p/go-libp2p-core#205](https://github.com/libp2p/go-libp2p-core/pull/205)) + - remove deprecated constructor for the insecure transport (#206) ([libp2p/go-libp2p-core#206](https://github.com/libp2p/go-libp2p-core/pull/206)) + - feat: add helper functions for working with addr infos (#202) ([libp2p/go-libp2p-core#202](https://github.com/libp2p/go-libp2p-core/pull/202)) + - fix: make timestamps strictly increasing (#201) ([libp2p/go-libp2p-core#201](https://github.com/libp2p/go-libp2p-core/pull/201)) + - ci: use github-actions for compatibility testing (#200) ([libp2p/go-libp2p-core#200](https://github.com/libp2p/go-libp2p-core/pull/200)) + - sync: update CI config files (#189) ([libp2p/go-libp2p-core#189](https://github.com/libp2p/go-libp2p-core/pull/189)) + - remove minimum Go version from README (#199) ([libp2p/go-libp2p-core#199](https://github.com/libp2p/go-libp2p-core/pull/199)) + - remove flaky tests (#194) ([libp2p/go-libp2p-core#194](https://github.com/libp2p/go-libp2p-core/pull/194)) + - reduce default timeouts to 15s (#192) ([libp2p/go-libp2p-core#192](https://github.com/libp2p/go-libp2p-core/pull/192)) + - fix benchmark of key verifications (#190) ([libp2p/go-libp2p-core#190](https://github.com/libp2p/go-libp2p-core/pull/190)) + - fix staticcheck errors (#191) ([libp2p/go-libp2p-core#191](https://github.com/libp2p/go-libp2p-core/pull/191)) + - doc: document Close on Transport (#188) ([libp2p/go-libp2p-core#188](https://github.com/libp2p/go-libp2p-core/pull/188)) + - add a helper function to go directly from a string to an AddrInfo (#184) ([libp2p/go-libp2p-core#184](https://github.com/libp2p/go-libp2p-core/pull/184)) +- github.com/libp2p/go-libp2p-http (v0.2.0 -> v0.2.1): + - remove Makefile ([libp2p/go-libp2p-http#70](https://github.com/libp2p/go-libp2p-http/pull/70)) + - fix staticcheck ([libp2p/go-libp2p-http#67](https://github.com/libp2p/go-libp2p-http/pull/67)) + - Revert "increase buffer size" + - Increase read buffer size to reduce poll system calls ([libp2p/go-libp2p-http#66](https://github.com/libp2p/go-libp2p-http/pull/66)) +- github.com/libp2p/go-libp2p-kad-dht (v0.12.2 -> v0.13.1): + - Extract validation from ProtocolMessenger ([libp2p/go-libp2p-kad-dht#741](https://github.com/libp2p/go-libp2p-kad-dht/pull/741)) + - remove codecov.yml ([libp2p/go-libp2p-kad-dht#742](https://github.com/libp2p/go-libp2p-kad-dht/pull/742)) + - integrate some basic opentelemetry tracing ([libp2p/go-libp2p-kad-dht#734](https://github.com/libp2p/go-libp2p-kad-dht/pull/734)) + - feat: delete GetValues ([libp2p/go-libp2p-kad-dht#728](https://github.com/libp2p/go-libp2p-kad-dht/pull/728)) + - chore: skip flaky test when race detector is enabled ([libp2p/go-libp2p-kad-dht#731](https://github.com/libp2p/go-libp2p-kad-dht/pull/731)) + - Dont count connection times in usefulness ([libp2p/go-libp2p-kad-dht#660](https://github.com/libp2p/go-libp2p-kad-dht/pull/660)) + - Routing table refresh should NOT block ([libp2p/go-libp2p-kad-dht#705](https://github.com/libp2p/go-libp2p-kad-dht/pull/705)) + - update bootstrapPeers to be func() []peer.AddrInfo (#716) ([libp2p/go-libp2p-kad-dht#716](https://github.com/libp2p/go-libp2p-kad-dht/pull/716)) +- github.com/libp2p/go-libp2p-noise (v0.2.0 -> v0.2.2): + - remove note about go modules in README ([libp2p/go-libp2p-noise#100](https://github.com/libp2p/go-libp2p-noise/pull/100)) + - fix: remove deprecated call to pk.Bytes ([libp2p/go-libp2p-noise#99](https://github.com/libp2p/go-libp2p-noise/pull/99)) +- github.com/libp2p/go-libp2p-peerstore (v0.2.7 -> v0.2.8): + - Fix perfomance issue in updating addr book ([libp2p/go-libp2p-peerstore#141](https://github.com/libp2p/go-libp2p-peerstore/pull/141)) + - Fix test flakes ([libp2p/go-libp2p-peerstore#164](https://github.com/libp2p/go-libp2p-peerstore/pull/164)) + - Only remove records during GC ([libp2p/go-libp2p-peerstore#135](https://github.com/libp2p/go-libp2p-peerstore/pull/135)) + - sync: update CI config files ([libp2p/go-libp2p-peerstore#160](https://github.com/libp2p/go-libp2p-peerstore/pull/160)) + - fix: fix some race conditions in the ds address book ([libp2p/go-libp2p-peerstore#161](https://github.com/libp2p/go-libp2p-peerstore/pull/161)) + - address lints and test failures ([libp2p/go-libp2p-peerstore#159](https://github.com/libp2p/go-libp2p-peerstore/pull/159)) + - stop using the deprecated go-multiaddr-net package ([libp2p/go-libp2p-peerstore#158](https://github.com/libp2p/go-libp2p-peerstore/pull/158)) +- github.com/libp2p/go-libp2p-pubsub (v0.4.2 -> v0.5.4): + - make slowness a warning, with a user configurable threshold + - reduce log spam from empty heartbeat messages + - fix: code review + - add support for custom protocol matching function + - fix: remove deprecated Bytes call (#436) ([libp2p/go-libp2p-pubsub#436](https://github.com/libp2p/go-libp2p-pubsub/pull/436)) + - cleanup: fix vet and staticcheck failures (#435) ([libp2p/go-libp2p-pubsub#435](https://github.com/libp2p/go-libp2p-pubsub/pull/435)) + - Revert noisy newline changes + - fix: avoid panic when peer is blacklisted after connection + - release priority locks early when handling batches + - don't respawn writer if we fail to open a stream; declare it a peer error + - batch process dead peer notifications + - use a priority lock instead of a semaphore + - do the notification in a goroutine + - emit new peer notification without holding the semaphore + - use a semaphore for new peer notifications so that we don't block the event loop + - don't accumulate pending goroutines from new connections + - rename RawTracer's DroppedInSubscribe into UndeliverableMessage + - add a new RawTracer event to track messages dropped in Subscribe + - add an option to configure the Subscription output queue length + - fix some comments + - expose more events for RawTracer + - Make close concurrent safe + - Fix close of closed channel + - Update README to point to correct example directory (#424) ([libp2p/go-libp2p-pubsub#424](https://github.com/libp2p/go-libp2p-pubsub/pull/424)) + - fix: remove deprecated and never used topic descriptors (#423) ([libp2p/go-libp2p-pubsub#423](https://github.com/libp2p/go-libp2p-pubsub/pull/423)) + - Refactor Gossipsub Parameters To Make Them More Configurable (#421) ([libp2p/go-libp2p-pubsub#421](https://github.com/libp2p/go-libp2p-pubsub/pull/421)) + - add tests for gs features and custom protocols + - add support for custom gossipsub protocols and feature tests + - RIP travis, Long Live CircleCI (#414) ([libp2p/go-libp2p-pubsub#414](https://github.com/libp2p/go-libp2p-pubsub/pull/414)) + - Ignore transient connections (#412) ([libp2p/go-libp2p-pubsub#412](https://github.com/libp2p/go-libp2p-pubsub/pull/412)) + - demote log spam to debug + - fix bug + - add last amount of validation + - add threshold validation + - strengthen validation + - rename checkSignature to checkSigningPolicy + - rename validation.Publish to PushLocal + - fix TestValidate, add TestValidate2 + - skip flaky test until we can fix it + - implement synchronous validation for locally published messages + - expose internalTracer as RawTracer + - export rejection named string constants + - more intelligent handling of ip whitelist check + - remove obsolete explicit IP whitelisting in favor of subnets + - add subnet whitelisting for IPColocation +- github.com/libp2p/go-libp2p-quic-transport (v0.11.2 -> v0.12.0): + - sync: update CI config files (#228) ([libp2p/go-libp2p-quic-transport#228](https://github.com/libp2p/go-libp2p-quic-transport/pull/228)) + - fix closing of streams in example ([libp2p/go-libp2p-quic-transport#221](https://github.com/libp2p/go-libp2p-quic-transport/pull/221)) + - close all UDP connections when the reuse is closed ([libp2p/go-libp2p-quic-transport#216](https://github.com/libp2p/go-libp2p-quic-transport/pull/216)) + - fix staticcheck ([libp2p/go-libp2p-quic-transport#217](https://github.com/libp2p/go-libp2p-quic-transport/pull/217)) + - sync: update CI config files (#214) ([libp2p/go-libp2p-quic-transport#214](https://github.com/libp2p/go-libp2p-quic-transport/pull/214)) + - implement a Transport.Close that waits for the reuse's GC to finish ([libp2p/go-libp2p-quic-transport#211](https://github.com/libp2p/go-libp2p-quic-transport/pull/211)) + - don't compare peer IDs when hole punching ([libp2p/go-libp2p-quic-transport#210](https://github.com/libp2p/go-libp2p-quic-transport/pull/210)) + - add hole punching support (#194) ([libp2p/go-libp2p-quic-transport#194](https://github.com/libp2p/go-libp2p-quic-transport/pull/194)) +- github.com/libp2p/go-libp2p-swarm (v0.5.0 -> v0.5.3): + - sync: update CI config files ([libp2p/go-libp2p-swarm#263](https://github.com/libp2p/go-libp2p-swarm/pull/263)) + - remove incorrect call to InterceptAddrDial ([libp2p/go-libp2p-swarm#260](https://github.com/libp2p/go-libp2p-swarm/pull/260)) + - speed up the TestFDLimitUnderflow test ([libp2p/go-libp2p-swarm#262](https://github.com/libp2p/go-libp2p-swarm/pull/262)) + - sync: update CI config files (#248) ([libp2p/go-libp2p-swarm#248](https://github.com/libp2p/go-libp2p-swarm/pull/248)) +- github.com/libp2p/go-libp2p-testing (v0.4.0 -> v0.4.2): + - fix deadlock in the transport's serve function ([libp2p/go-libp2p-testing#35](https://github.com/libp2p/go-libp2p-testing/pull/35)) + - fix: cleanup transport suite ([libp2p/go-libp2p-testing#34](https://github.com/libp2p/go-libp2p-testing/pull/34)) + - Address `go vet` and `saticcheck` issues ([libp2p/go-libp2p-testing#33](https://github.com/libp2p/go-libp2p-testing/pull/33)) + - Defer closing stream for reading ([libp2p/go-libp2p-testing#32](https://github.com/libp2p/go-libp2p-testing/pull/32)) +- github.com/libp2p/go-libp2p-tls (v0.1.3 -> v0.2.0): + - fix: don't fail the handshake when the libp2p extension is critical ([libp2p/go-libp2p-tls#88](https://github.com/libp2p/go-libp2p-tls/pull/88)) + - fix deprecated call to key.Bytes ([libp2p/go-libp2p-tls#86](https://github.com/libp2p/go-libp2p-tls/pull/86)) + - fix usage of deprecated peer.IDB58Decode ([libp2p/go-libp2p-tls#77](https://github.com/libp2p/go-libp2p-tls/pull/77)) + - remove setting of the TLS 1.3 GODEBUG flag ([libp2p/go-libp2p-tls#68](https://github.com/libp2p/go-libp2p-tls/pull/68)) + - improve the error message returned when peer verification fails ([libp2p/go-libp2p-tls#57](https://github.com/libp2p/go-libp2p-tls/pull/57)) + - update to Go 1.14 ([libp2p/go-libp2p-tls#54](https://github.com/libp2p/go-libp2p-tls/pull/54)) + - Update deps and fix tests ([libp2p/go-libp2p-tls#43](https://github.com/libp2p/go-libp2p-tls/pull/43)) +- github.com/libp2p/go-libp2p-transport-upgrader (v0.4.2 -> v0.4.6): + - chore: update deps ([libp2p/go-libp2p-transport-upgrader#78](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/78)) + - fix typo in error message ([libp2p/go-libp2p-transport-upgrader#77](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/77)) + - fix staticcheck ([libp2p/go-libp2p-transport-upgrader#74](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/74)) + - don't listen on all interfaces in tests ([libp2p/go-libp2p-transport-upgrader#73](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/73)) + - stop using the deprecated go-multiaddr-net ([libp2p/go-libp2p-transport-upgrader#72](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/72)) +- github.com/libp2p/go-libp2p-xor (v0.0.0-20200501025846-71e284145d58 -> v0.0.0-20210714161855-5c005aca55db): + - Add immutable remove operation ([libp2p/go-libp2p-xor#14](https://github.com/libp2p/go-libp2p-xor/pull/14)) + - fix go vet and staticcheck ([libp2p/go-libp2p-xor#11](https://github.com/libp2p/go-libp2p-xor/pull/11)) +- github.com/libp2p/go-reuseport-transport (v0.0.4 -> v0.0.5): + - remove note about Go modules in README ([libp2p/go-reuseport-transport#32](https://github.com/libp2p/go-reuseport-transport/pull/32)) + - stop using the deprecated go-multiaddr-net package ([libp2p/go-reuseport-transport#30](https://github.com/libp2p/go-reuseport-transport/pull/30)) +- github.com/libp2p/go-socket-activation (v0.0.2 -> v0.1.0): + - chore: stop using the deprecated go-multiaddr-net package ([libp2p/go-socket-activation#16](https://github.com/libp2p/go-socket-activation/pull/16)) + - fix staticcheck ([libp2p/go-socket-activation#13](https://github.com/libp2p/go-socket-activation/pull/13)) +- github.com/libp2p/go-tcp-transport (v0.2.4 -> v0.2.8): + - disable metrics collection on Windows ([libp2p/go-tcp-transport#93](https://github.com/libp2p/go-tcp-transport/pull/93)) + - sync: update CI config files (#90) ([libp2p/go-tcp-transport#90](https://github.com/libp2p/go-tcp-transport/pull/90)) + - chore: update go-libp2p-transport-upgrader and go-reuseport-transport ([libp2p/go-tcp-transport#84](https://github.com/libp2p/go-tcp-transport/pull/84)) +- github.com/libp2p/go-ws-transport (v0.4.0 -> v0.5.0): + - chore: update go-libp2p-transport-upgrader and go-libp2p-core ([libp2p/go-ws-transport#103](https://github.com/libp2p/go-ws-transport/pull/103)) + - remove deprecated type ([libp2p/go-ws-transport#102](https://github.com/libp2p/go-ws-transport/pull/102)) + - sync: update CI config files ([libp2p/go-ws-transport#101](https://github.com/libp2p/go-ws-transport/pull/101)) + - chore: various cleanups required to get vet/staticcheck/test to pass ([libp2p/go-ws-transport#100](https://github.com/libp2p/go-ws-transport/pull/100)) +- github.com/lucas-clemente/quic-go (v0.21.2 -> v0.23.0): + - update to Go 1.17.x ([lucas-clemente/quic-go#3258](https://github.com/lucas-clemente/quic-go/pull/3258)) + - quicvarint: export Min and Max (#3253) ([lucas-clemente/quic-go#3253](https://github.com/lucas-clemente/quic-go/pull/3253)) + - drop support for Go 1.15 ([lucas-clemente/quic-go#3247](https://github.com/lucas-clemente/quic-go/pull/3247)) + - quicvarint: add Reader and Writer interfaces (#3233) ([lucas-clemente/quic-go#3233](https://github.com/lucas-clemente/quic-go/pull/3233)) + - fix race when stream.Read and CancelRead are called concurrently ([lucas-clemente/quic-go#3241](https://github.com/lucas-clemente/quic-go/pull/3241)) + - also count coalesced 0-RTT packets in the integration tests ([lucas-clemente/quic-go#3251](https://github.com/lucas-clemente/quic-go/pull/3251)) + - remove draft versions 32 and 34 from README (#3244) ([lucas-clemente/quic-go#3244](https://github.com/lucas-clemente/quic-go/pull/3244)) + - update Changelog ([lucas-clemente/quic-go#3245](https://github.com/lucas-clemente/quic-go/pull/3245)) + - optimize hasOutstandingCryptoPackets in sentPacketHandler ([lucas-clemente/quic-go#3230](https://github.com/lucas-clemente/quic-go/pull/3230)) + - permit underlying conn to implement batch interface directly ([lucas-clemente/quic-go#3237](https://github.com/lucas-clemente/quic-go/pull/3237)) + - cancel the PTO timer when all Handshake packets are acknowledged ([lucas-clemente/quic-go#3231](https://github.com/lucas-clemente/quic-go/pull/3231)) + - fix flaky INVALID_TOKEN server test ([lucas-clemente/quic-go#3223](https://github.com/lucas-clemente/quic-go/pull/3223)) + - drop support for QUIC draft version 32 and 34 ([lucas-clemente/quic-go#3217](https://github.com/lucas-clemente/quic-go/pull/3217)) + - fix flaky 0-RTT integration test ([lucas-clemente/quic-go#3224](https://github.com/lucas-clemente/quic-go/pull/3224)) + - use batched reads ([lucas-clemente/quic-go#3142](https://github.com/lucas-clemente/quic-go/pull/3142)) + - add a config option to disable sending of Version Negotiation packets ([lucas-clemente/quic-go#3216](https://github.com/lucas-clemente/quic-go/pull/3216)) + - remove the RetireBugBackwardsCompatibilityMode ([lucas-clemente/quic-go#3213](https://github.com/lucas-clemente/quic-go/pull/3213)) + - remove outdated ackhandler test case ([lucas-clemente/quic-go#3212](https://github.com/lucas-clemente/quic-go/pull/3212)) + - remove unused StripGreasedVersions function ([lucas-clemente/quic-go#3214](https://github.com/lucas-clemente/quic-go/pull/3214)) + - fix incorrect usage of errors.Is ([lucas-clemente/quic-go#3215](https://github.com/lucas-clemente/quic-go/pull/3215)) + - return error on SendMessage when session is closed ([lucas-clemente/quic-go#3218](https://github.com/lucas-clemente/quic-go/pull/3218)) + - remove a redundant error check ([lucas-clemente/quic-go#3210](https://github.com/lucas-clemente/quic-go/pull/3210)) + - update golangci-lint to v1.41.1 ([lucas-clemente/quic-go#3205](https://github.com/lucas-clemente/quic-go/pull/3205)) + - Update doc for dialer in http3.RoundTripper ([lucas-clemente/quic-go#3208](https://github.com/lucas-clemente/quic-go/pull/3208)) +- github.com/multiformats/go-multiaddr (v0.3.3 -> v0.4.0): + - remove forced dependency on deprecated go-maddr-filter ([multiformats/go-multiaddr#162](https://github.com/multiformats/go-multiaddr/pull/162)) + - remove deprecated SwapToP2pMultiaddrs ([multiformats/go-multiaddr#161](https://github.com/multiformats/go-multiaddr/pull/161)) + - remove Makefile ([multiformats/go-multiaddr#163](https://github.com/multiformats/go-multiaddr/pull/163)) + - remove deprecated filter functions ([multiformats/go-multiaddr#157](https://github.com/multiformats/go-multiaddr/pull/157)) + - remove deprecated NetCodec ([multiformats/go-multiaddr#159](https://github.com/multiformats/go-multiaddr/pull/159)) + - add Noise ([multiformats/go-multiaddr#156](https://github.com/multiformats/go-multiaddr/pull/156)) + - Add TLS protocol ([multiformats/go-multiaddr#153](https://github.com/multiformats/go-multiaddr/pull/153)) +- github.com/multiformats/go-multicodec (v0.2.0 -> v0.3.0): + - Export reserved range constants (#53) ([multiformats/go-multicodec#53](https://github.com/multiformats/go-multicodec/pull/53)) + - make Code.Set accept valid code numbers + - replace Of with Code.Set, implementing flag.Value + - add multiformats/multicodec as a git submodule + - update the generator with the "status" CSV column + - Run `go generate` to generate the latest codecs + - Add lookup for multicodec code by string name ([multiformats/go-multicodec#40](https://github.com/multiformats/go-multicodec/pull/40)) + +### Contributors + +| Contributor | Commits | Lines ± | Files Changed | +|-------------|---------|---------|---------------| +| Daniel Martí | 42 | +8549/-6587 | 170 | +| Eric Myhre | 55 | +5883/-6715 | 395 | +| Marten Seemann | 100 | +1814/-2028 | 275 | +| Steven Allen | 80 | +1573/-1998 | 127 | +| hannahhoward | 18 | +1721/-671 | 53 | +| Will | 2 | +1114/-1217 | 18 | +| Andrew Gillis | 2 | +1220/-720 | 14 | +| gammazero | 3 | +43/-1856 | 10 | +| Masih H. Derkani | 3 | +960/-896 | 8 | +| Adin Schmahmann | 25 | +1458/-313 | 44 | +| vyzo | 27 | +986/-353 | 60 | +| Will Scott | 6 | +852/-424 | 16 | +| Rod Vagg | 19 | +983/-255 | 66 | +| Petar Maymounkov | 6 | +463/-179 | 22 | +| web3-bot | 10 | +211/-195 | 24 | +| adlrocha | 1 | +330/-75 | 15 | +| RubenKelevra | 2 | +128/-210 | 2 | +| Ian Davis | 3 | +200/-109 | 17 | +| Cory Schwartz | 3 | +231/-33 | 7 | +| Keenan Nemetz | 1 | +184/-71 | 2 | +| Randy Reddig | 2 | +187/-53 | 8 | +| Takashi Matsuda | 3 | +201/-2 | 7 | +| guseggert | 4 | +161/-20 | 9 | +| Lucas Molas | 5 | +114/-47 | 27 | +| nisdas | 4 | +115/-45 | 7 | +| Michael Muré | 6 | +107/-33 | 24 | +| Richard Ramos | 2 | +113/-9 | 3 | +| Marcin Rataj | 12 | +88/-24 | 13 | +| Ondrej Prazak | 2 | +104/-6 | 4 | +| Michal Dobaczewski | 2 | +77/-28 | 3 | +| Jorropo | 3 | +9/-75 | 4 | +| Andey Robins | 1 | +70/-3 | 3 | +| Gus Eggert | 10 | +34/-31 | 12 | +| noot | 1 | +54/-9 | 5 | +| Maxim Merzhanov | 1 | +29/-24 | 1 | +| Adrian Lanzafame | 1 | +30/-13 | 2 | +| Bogdan Stirbat | 1 | +22/-16 | 2 | +| Shad Sterling | 1 | +28/-3 | 1 | +| Jesse Bouwman | 5 | +30/-0 | 5 | +| Pavel Karpy | 1 | +19/-7 | 2 | +| lasiar | 5 | +14/-10 | 5 | +| Dennis Trautwein | 1 | +20/-4 | 2 | +| Louis Thibault | 1 | +22/-1 | 2 | +| whyrusleeping | 2 | +21/-1 | 2 | +| aarshkshah1992 | 3 | +12/-8 | 3 | +| Peter Rabbitson | 2 | +20/-0 | 2 | +| bt90 | 2 | +17/-2 | 2 | +| Dominic Della Valle | 1 | +13/-1 | 2 | +| Audrius Butkevicius | 1 | +12/-1 | 1 | +| Brian Strauch | 1 | +9/-3 | 1 | +| Aarsh Shah | 2 | +1/-11 | 2 | +| Whyrusleeping | 1 | +11/-0 | 1 | +| Max | 1 | +7/-3 | 1 | +| vallder | 1 | +3/-5 | 1 | +| Michael Burns | 3 | +2/-6 | 3 | +| Lasse Johnsen | 1 | +4/-4 | 2 | +| snyh | 1 | +5/-2 | 1 | +| Hector Sanjuan | 2 | +3/-2 | 2 | +| 市川恭佑 (ebi) | 1 | +1/-3 | 1 | +| godcong | 2 | +2/-1 | 2 | +| Mathis Engelbart | 1 | +1/-2 | 1 | +| folbrich | 1 | +1/-1 | 1 | +| Med Mouine | 1 | +1/-1 | 1 | + + ## v0.9.1 2021-07-20 This is a small bug fix release resolving the following issues: diff --git a/Dockerfile b/Dockerfile index 113c6599db1..9322e891b3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Note: when updating the go minor version here, also update the go-channel in snap/snapcraft.yml -FROM golang:1.15.2-buster +FROM golang:1.16.7-buster LABEL maintainer="Steven Allen " # Install deps diff --git a/README.md b/README.md index 67f229ee041..753a911a09b 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,9 @@ ![banner](https://ipfs.io/ipfs/bafykbzacecaesuqmivkauix25v6i6xxxsvsrtxknhgb5zak3xxsg2nb4dhs2u/ipfs.go.png) -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![Matrix](https://img.shields.io/badge/matrix-%23ipfs%3Amatrix.org-blue.svg?style=flat-square)](https://matrix.to/#/room/#ipfs:matrix.org) -[![IRC](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) -[![Discord](https://img.shields.io/discord/475789330380488707?color=blueviolet&label=discord&style=flat-square)](https://discord.gg/24fmuwR) -[![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](https://godoc.org/github.com/ipfs/go-ipfs) -[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) -[![CircleCI](https://img.shields.io/circleci/build/github/ipfs/go-ipfs?style=flat-square)](https://circleci.com/gh/ipfs/go-ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square&cacheSeconds=3600)](https://protocol.ai) +[![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square&cacheSeconds=3600)](https://godoc.org/github.com/ipfs/go-ipfs) +[![CircleCI](https://img.shields.io/circleci/build/github/ipfs/go-ipfs?style=flat-square&cacheSeconds=3600)](https://circleci.com/gh/ipfs/go-ipfs) ## What is IPFS? @@ -22,29 +18,60 @@ Before opening an issue, consider using one of the following locations to ensure - IPFS _design_ in [ipfs/specs issues](https://github.com/ipfs/specs/issues). - Exploration of new ideas in [ipfs/notes issues](https://github.com/ipfs/notes/issues). - Ask questions and meet the rest of the community at the [IPFS Forum](https://discuss.ipfs.io). + - Or [chat with us](https://docs.ipfs.io/community/chat/). + +[![YouTube Channel Subscribers](https://img.shields.io/youtube/channel/subscribers/UCdjsUXJ3QawK4O5L1kqqsew?label=Subscribe%20IPFS&style=social&cacheSeconds=3600)](https://www.youtube.com/channel/UCdjsUXJ3QawK4O5L1kqqsew) [![Follow @IPFS on Twitter](https://img.shields.io/twitter/follow/IPFS?style=social&cacheSeconds=3600)](https://twitter.com/IPFS) + +## Next milestones + +[Milestones on Github](https://github.com/ipfs/go-ipfs/milestones) + + ## Table of Contents - [Security Issues](#security-issues) - [Install](#install) - [System Requirements](#system-requirements) - - [Install prebuilt packages](#install-prebuilt-packages) - - [From Linux package managers](#from-linux-package-managers) + - [Docker](#docker) + - [Native Linux package managers](#native-linux-package-managers) + - [ArchLinux](#archlinux) + - [Nix](#nix-linux) + - [Solus](#solus) + - [openSUSE](#opensuse) + - [Other package managers](#other-package-managers) + - [Guix](#guix) + - [Snap](#snap) + - [macOS package managers](#macos-package-managers) + - [MacPorts](#MacPorts) + - [Nix](#nix-macos) + - [Windows package managers](#windows-package-managers) + - [Chocolatey](#chocolatey) + - [Scoop](#scoop) + - [Install prebuilt binaries](#install-prebuilt-binaries) - [Build from Source](#build-from-source) - [Install Go](#install-go) - [Download and Compile IPFS](#download-and-compile-ipfs) + - [Cross Compiling](#cross-compiling) + - [OpenSSL](#openssl) - [Troubleshooting](#troubleshooting) - [Updating go-ipfs](#updating-go-ipfs) - [Getting Started](#getting-started) - [Some things to try](#some-things-to-try) - [Usage](#usage) - - [Running IPFS inside Docker](#running-ipfs-inside-docker) - [Troubleshooting](#troubleshooting-1) - [Packages](#packages) - [Development](#development) + - [Map of go-ipfs Subsystems](#map-of-go-ipfs-subsystems) - [CLI, HTTP-API, Architecture Diagram](#cli-http-api-architecture-diagram) - [Testing](#testing) - [Development Dependencies](#development-dependencies) + - [Developer Notes](#developer-notes) - [Contributing](#contributing) - [License](#license) @@ -69,40 +96,32 @@ If your system is resource-constrained, we recommend: 1. Installing OpenSSL and rebuilding go-ipfs manually with `make build GOTAGS=openssl`. See the [download and compile](#download-and-compile-ipfs) section for more information on compiling go-ipfs. 2. Initializing your daemon with `ipfs init --profile=lowpower` -### Install prebuilt packages +### Docker -We host prebuilt binaries over at our [distributions page](https://ipfs.io/ipns/dist.ipfs.io#go-ipfs). - -From there: -- Click the blue "Download go-ipfs" on the right side of the page. -- Open/extract the archive. -- Move `ipfs` to your path (`install.sh` can do it for you). +[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/ipfs/go-ipfs?color=blue&label=go-ipfs%20docker%20image&logo=docker&sort=semver&style=flat-square&cacheSeconds=3600)](https://hub.docker.com/r/ipfs/go-ipfs/) -You can also download go-ipfs from this project's GitHub releases page if you are unable to access ipfs.io. +More info on how to run go-ipfs inside docker can be found [here](https://docs.ipfs.io/how-to/run-ipfs-inside-docker/). -### From Linux package managers +### Native Linux package managers - [Arch Linux](#arch-linux) -- [Nix](#nix) +- [Nix](#nix-linux) - [Solus](#solus) -- [Snap](#snap) +- [openSUSE](#openSUSE) -#### Arch Linux +#### ArchLinux -In Arch Linux go-ipfs is available as -[go-ipfs](https://www.archlinux.org/packages/community/x86_64/go-ipfs/) package. +[![go-ipfs via Community Repo](https://img.shields.io/archlinux/v/community/x86_64/go-ipfs?color=1793d1&label=go-ipfs&logo=arch-linux&style=flat-square&cacheSeconds=3600)](https://wiki.archlinux.org/title/IPFS) -``` -$ sudo pacman -S go-ipfs +```bash +# pacman -Syu go-ipfs ``` -Development version of go-ipfs is also on AUR under -[go-ipfs-git](https://aur.archlinux.org/packages/go-ipfs-git/). -You can install it using your favorite AUR Helper or manually from AUR. +[![go-ipfs-git via AUR](https://img.shields.io/static/v1?label=go-ipfs-git&message=latest%40master&color=1793d1&logo=arch-linux&style=flat-square&cacheSeconds=3600)](https://aur.archlinux.org/packages/go-ipfs-git/) -#### Nix +#### Nix -For Linux and MacOSX you can use the purely functional package manager [Nix](https://nixos.org/nix/): +With the purely functional package manager [Nix](https://nixos.org/nix/) you can install go-ipfs like this: ``` $ nix-env -i ipfs @@ -110,14 +129,6 @@ $ nix-env -i ipfs You can also install the Package by using its attribute name, which is also `ipfs`. -#### Guix - -GNU's functional package manager, [Guix](https://www.gnu.org/software/guix/), also provides a go-ipfs package: - -``` -$ guix package -i go-ipfs -``` - #### Solus In solus, go-ipfs is available in the main repository as @@ -129,6 +140,23 @@ $ sudo eopkg install go-ipfs You can also install it through the Solus software center. +#### openSUSE + +[Community Package for go-ipfs](https://software.opensuse.org/package/go-ipfs) + +### Other package managers + +- [Guix](#guix) +- [Snap](#snap) + +#### Guix + +GNU's functional package manager, [Guix](https://www.gnu.org/software/guix/), also provides a go-ipfs package: + +``` +$ guix package -i go-ipfs +``` + #### Snap With snap, in any of the [supported Linux distributions](https://snapcraft.io/docs/core/install): @@ -137,14 +165,37 @@ With snap, in any of the [supported Linux distributions](https://snapcraft.io/do $ sudo snap install ipfs ``` -### From Windows package managers +#### macOS package managers + +- [MacPorts](#macports) +- [Nix](#nix-macos) + +#### MacPorts + +The package [ipfs](https://ports.macports.org/port/ipfs) currently points to go-ipfs and is being maintained. + +``` +$ sudo port install ipfs +``` + +#### Nix + +In macOS you can use the purely functional package manager [Nix](https://nixos.org/nix/): + +``` +$ nix-env -i ipfs +``` + +You can also install the Package by using its attribute name, which is also `ipfs`. + +### Windows package managers - [Chocolatey](#chocolatey) - [Scoop](#scoop) #### Chocolatey -The package [ipfs](https://chocolatey.org/packages/ipfs) currently points to go-ipfs and is being maintained. +[![Chocolatey Version](https://img.shields.io/chocolatey/v/go-ipfs?color=00a4ef&label=go-ipfs&logo=windows&style=flat-square&cacheSeconds=3600)](https://chocolatey.org/packages/go-ipfs) ```Powershell PS> choco install ipfs @@ -158,9 +209,24 @@ PS> scoop bucket add extras PS> scoop install go-ipfs ``` +### Install prebuilt binaries + +[![dist.ipfs.io Downloads](https://img.shields.io/github/v/release/ipfs/go-ipfs?label=dist.ipfs.io&logo=ipfs&style=flat-square&cacheSeconds=3600)](https://ipfs.io/ipns/dist.ipfs.io#go-ipfs) + +From there: +- Click the blue "Download go-ipfs" on the right side of the page. +- Open/extract the archive. +- Move `ipfs` to your path (`install.sh` can do it for you). + +You can also download go-ipfs from this project's GitHub releases page if you are unable to access [dist.ipfs.io](https://ipfs.io/ipns/dist.ipfs.io#go-ipfs): + +[GitHub releases](https://github.com/ipfs/go-ipfs/releases) + ### Build from Source -go-ipfs's build system requires Go 1.15.2 and some standard POSIX build tools: +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/ipfs/go-ipfs?label=Requires%20Go&logo=go&style=flat-square&cacheSeconds=3600) + +go-ipfs's build system requires Go and some standard POSIX build tools: * GNU make * Git @@ -170,7 +236,9 @@ To build without GCC, build with `CGO_ENABLED=0` (e.g., `make build CGO_ENABLED= #### Install Go -The build process for ipfs requires Go 1.15.2 or higher. If you don't have it: [Download Go 1.15+](https://golang.org/dl/). +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/ipfs/go-ipfs?label=Requires%20Go&logo=go&style=flat-square&cacheSeconds=3600) + +If you need to update: [Download latest version of Go](https://golang.org/dl/). You'll need to add Go's bin directories to your `$PATH` environment variable e.g., by adding these lines to your `/etc/profile` (for a system-wide installation) or `$HOME/.profile`: @@ -222,8 +290,8 @@ dependencies as well. - _WARNING_: Older versions of OSX FUSE (for Mac OS X) can cause kernel panics when mounting!- We strongly recommend you use the [latest version of OSX FUSE](http://osxfuse.github.io/). (See https://github.com/ipfs/go-ipfs/issues/177) -- For more details on setting up FUSE (so that you can mount the filesystem), see the docs folder. -- Shell command completion is available in `misc/completion/ipfs-completion.bash`. Read [docs/command-completion.md](docs/command-completion.md) to learn how to install it. +- Read [docs/fuse.md](docs/fuse.md) for more details on setting up FUSE (so that you can mount the filesystem). +- Shell command completions can be generated with one of the `ipfs commands completion` subcommands. Read [docs/command-completion.md](docs/command-completion.md) to learn more. - See the [misc folder](https://github.com/ipfs/go-ipfs/tree/master/misc) for how to connect IPFS to systemd or whatever init system your distro uses. ### Updating go-ipfs @@ -262,7 +330,10 @@ $ ipfs get /ipns/dist.ipfs.io/go-ipfs/$VERSION/go-ipfs_$VERSION_windows-amd64.zi ## Getting Started -See also: https://docs.ipfs.io/introduction/usage/ +### Usage + +[![docs: Command-line quick start](https://img.shields.io/static/v1?label=docs&message=Command-line%20quick%20start&color=blue&style=flat-square&cacheSeconds=3600)](https://docs.ipfs.io/how-to/command-line-quick-start/) +[![docs: Command-line reference](https://img.shields.io/static/v1?label=docs&message=Command-line%20reference&color=blue&style=flat-square&cacheSeconds=3600)](https://docs.ipfs.io/reference/cli/) To start using IPFS, you must first initialize IPFS's config files on your system, this is done with `ipfs init`. See `ipfs init --help` for information on @@ -279,136 +350,6 @@ Basic proof of 'ipfs working' locally: # QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o ipfs cat -### Usage - -``` - ipfs - Global p2p merkle-dag filesystem. - - ipfs [] [] ... - -SUBCOMMANDS - BASIC COMMANDS - init Initialize local IPFS configuration - add Add a file to IPFS - cat Show IPFS object data - get Download IPFS objects - ls List links from an object - refs List hashes of links from an object - - DATA STRUCTURE COMMANDS - dag Interact with IPLD DAG nodes - files Interact with files as if they were a unix filesystem - block Interact with raw blocks in the datastore - - ADVANCED COMMANDS - daemon Start a long-running daemon process - mount Mount an IPFS read-only mount point - resolve Resolve any type of name - name Publish and resolve IPNS names - key Create and list IPNS name keypairs - dns Resolve DNS links - pin Pin objects to local storage - repo Manipulate the IPFS repository - stats Various operational stats - p2p Libp2p stream mounting - filestore Manage the filestore (experimental) - - NETWORK COMMANDS - id Show info about IPFS peers - bootstrap Add or remove bootstrap peers - swarm Manage connections to the p2p network - dht Query the DHT for values or peers - ping Measure the latency of a connection - diag Print diagnostics - - TOOL COMMANDS - config Manage configuration - version Show IPFS version information - update Download and apply go-ipfs updates - commands List all available commands - cid Convert and discover properties of CIDs - log Manage and show logs of running daemon - - Use 'ipfs --help' to learn more about each command. - - ipfs uses a repository in the local file system. By default, the repo is located at - ~/.ipfs. To change the repo location, set the $IPFS_PATH environment variable: - - export IPFS_PATH=/path/to/ipfsrepo -``` - -### Running IPFS inside Docker - -An IPFS docker image is hosted at [hub.docker.com/r/ipfs/go-ipfs](https://hub.docker.com/r/ipfs/go-ipfs/). -To make files visible inside the container you need to mount a host directory -with the `-v` option to docker. Choose a directory that you want to use to -import/export files from IPFS. You should also choose a directory to store -IPFS files that will persist when you restart the container. - - export ipfs_staging= - export ipfs_data= - -Start a container running ipfs and expose ports 4001, 5001 and 8080: - - docker run -d --name ipfs_host -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest - -Watch the ipfs log: - - docker logs -f ipfs_host - -Wait for ipfs to start. ipfs is running when you see: - - Gateway (readonly) server - listening on /ip4/0.0.0.0/tcp/8080 - -You can now stop watching the log. - -Run ipfs commands: - - docker exec ipfs_host ipfs - -For example: connect to peers - - docker exec ipfs_host ipfs swarm peers - -Add files: - - cp -r $ipfs_staging - docker exec ipfs_host ipfs add -r /export/ - -Stop the running container: - - docker stop ipfs_host - -When starting a container running ipfs for the first time with an empty data directory, it will call `ipfs init` to initialize configuration files and generate a new keypair. At this time, you can choose which profile to apply using the `IPFS_PROFILE` environment variable: - - docker run -d --name ipfs_host -e IPFS_PROFILE=server -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest - -#### Private swarms inside Docker - -It is possible to initialize the container with a swarm key file (`/data/ipfs/swarm.key`) using the variables `IPFS_SWARM_KEY` and `IPFS_SWARM_KEY_FILE`. The `IPFS_SWARM_KEY` creates `swarm.key` with the contents of the variable itself, whilst `IPFS_SWARM_KEY_FILE` copies the key from a path stored in the variable. The `IPFS_SWARM_KEY_FILE` **overwrites** the key generated by `IPFS_SWARM_KEY`. - - docker run -d --name ipfs_host -e IPFS_SWARM_KEY= -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest - -The swarm key initialization can also be done using docker secrets **(requires docker swarm or docker-compose)**: - - cat your_swarm.key | docker secret create swarm_key_secret - - docker run -d --name ipfs_host --secret swarm_key_secret -e IPFS_SWARM_KEY_FILE=/run/secrets/swarm_key_secret -v $ipfs_staging:/export -v $ipfs_data:/data/ipfs -p 4001:4001 -p 4001:4001/udp -p 127.0.0.1:8080:8080 -p 127.0.0.1:5001:5001 ipfs/go-ipfs:latest - -#### Key rotation inside Docker - -If needed, it is possible to do key rotation in an ephemeral container that is temporarily executing against a volume that is mounted under `/data/ipfs`: - -```sh -# given container named 'ipfs-test' that persists repo at /path/to/persisted/.ipfs -$ docker run -d --name ipfs-test -v /path/to/persisted/.ipfs:/data/ipfs ipfs/go-ipfs:v0.7.0 -$ docker stop ipfs-test - -# key rotation works like this (old key saved under 'old-self') -$ docker run --rm -it -v /path/to/persisted/.ipfs:/data/ipfs ipfs/go-ipfs:v0.7.0 key rotate -o old-self -t ed25519 -$ docker start ipfs-test # will start with the new key -``` - ### Troubleshooting If you have previously installed IPFS before and you are running into problems getting a newer version to work, try deleting (or backing up somewhere else) your IPFS config directory (~/.ipfs by default) and rerunning `ipfs init`. This will reinitialize the config file to its defaults and clear out the local datastore of any bad entries. @@ -532,8 +473,7 @@ We ❤️ all [our contributors](docs/AUTHORS); this project wouldn’t be what This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). -You can contact us on the freenode #ipfs-dev channel or attend one of our -[weekly calls](https://github.com/ipfs/team-mgmt/issues/674). +Please reach out to us in one [chat](https://docs.ipfs.io/community/chat/) rooms. ## License diff --git a/bin/mkreleaselog b/bin/mkreleaselog index bcc4f325513..d9f91833707 100755 --- a/bin/mkreleaselog +++ b/bin/mkreleaselog @@ -1,40 +1,78 @@ #!/bin/zsh -#set -x +# +# Invocation: mkreleaselog [FIRST_REF [LAST_REF]] + set -euo pipefail export GO111MODULE=on export GOPATH="$(go env GOPATH)" -alias jq="jq --unbuffered" - -AUTHORS=( +# List of PCRE regular expressions to match "included" modules. +INCLUDE_MODULES=( # orgs - ipfs - ipld - libp2p - multiformats - filecoin-project - ipfs-shipyard - - # Authors of personal repos used by go-ipfs that should be mentioned in the + "^github.com/ipfs/" + "^github.com/ipld/" + "^github.com/libp2p/" + "^github.com/multiformats/" + "^github.com/filecoin-project/" + "^github.com/ipfs-shipyard/" + + # Authors of personal modules used by go-ipfs that should be mentioned in the # release notes. - whyrusleeping - Kubuxu - jbenet - Stebalien - marten-seemann - hsanjuan - lucas-clemente - warpfork + "^github.com/whyrusleeping/" + "^github.com/Kubuxu/" + "^github.com/jbenet/" + "^github.com/Stebalien/" + "^github.com/marten-seemann/" + "^github.com/hsanjuan/" + "^github.com/lucas-clemente/" + "^github.com/warpfork/" +) + +# List of PCRE regular expressions to match "excluded" modules. Applied after includes. +EXCLUDE_MODULES=( + "^github.com/marten-seemann/qtls" ) -[[ -n "${REPO_FILTER+x}" ]] || REPO_FILTER="github.com/(${$(printf "|%s" "${AUTHORS[@]}"):1})" +# Ignored files as git pathspecs. These patters will match any full path component. +IGNORE_FILES=( + ".gx" + "package.json" + ".travis.yml" + "go.mod" + "go.sum" + ".github" + ".circleci" + "*.pb.go" + "cbor_gen.go" + "ipldsch_*.go" +) + +########################################################################################## + +if [[ ${#INCLUDE_MODULES[@]} -gt 0 ]]; then + INCLUDE_REGEX="(${$(printf "|%s" "${INCLUDE_MODULES[@]}"):1})" +else + INCLUDE_REGEX="" # "match anything" +fi + +if [[ ${#EXCLUDE_MODULES[@]} -gt 0 ]]; then + EXCLUDE_REGEX="(${$(printf "|%s" "${EXCLUDE_MODULES[@]}"):1})" +else + EXCLUDE_REGEX='$^' # "match nothing" +fi + +IGNORE_FILES_PATHSPEC=() +for f in "${IGNORE_FILES[@]}"; do + IGNORE_FILES_PATHSPEC+=(":^:**/$f" ":^:$f") # Prepend the magic "ignore this" sequence. +done -[[ -n "${IGNORED_FILES+x}" ]] || IGNORED_FILES='^\(\.gx\|package\.json\|\.travis\.yml\|go.mod\|go\.sum|\.github|\.circleci\)$' NL=$'\n' ROOT_DIR="$(git rev-parse --show-toplevel)" +alias jq="jq --unbuffered" + msg() { echo "$*" >&2 } @@ -49,13 +87,15 @@ statlog() { mailmap_file="$ROOT_DIR/.mailmap" fi - git -C "$rpath" -c mailmap.file="$mailmap_file" log --use-mailmap --shortstat --no-merges --pretty="tformat:%H%n%aN%n%aE" "$start..$end" | while - read hash - read name - read email - read _ # empty line - read changes - do + local stack=() + git -C "$rpath" -c mailmap.file="$mailmap_file" log --use-mailmap --shortstat --no-merges --pretty="tformat:%H%x09%aN%x09%aE" "$start..$end" -- . "${IGNORE_FILES_PATHSPEC[@]}" | while read -r line; do + if [[ -n "$line" ]]; then + stack+=("$line") + continue + fi + + read -r changes + changed=0 insertions=0 deletions=0 @@ -72,14 +112,18 @@ statlog() { fi done<<<"${changes//,/$NL}" - jq -n \ - --arg "hash" "$hash" \ - --arg "name" "$name" \ - --arg "email" "$email" \ - --argjson "changed" "$changed" \ - --argjson "insertions" "$insertions" \ - --argjson "deletions" "$deletions" \ - '{Commit: $hash, Author: $name, Email: $email, Files: $changed, Insertions: $insertions, Deletions: $deletions}' + for author in "${stack[@]}"; do + IFS=$'\t' read -r hash name email <<<"$author" + jq -n \ + --arg "hash" "$hash" \ + --arg "name" "$name" \ + --arg "email" "$email" \ + --argjson "changed" "$changed" \ + --argjson "insertions" "$insertions" \ + --argjson "deletions" "$deletions" \ + '{Commit: $hash, Author: $name, Email: $email, Files: $changed, Insertions: $insertions, Deletions: $deletions}' + done + stack=() done } @@ -103,6 +147,16 @@ pr_link() { printf -- "[%s#%s](https://%s/pull/%s)" "$ghname" "$prnum" "$repo" "$prnum" } +ignored_commit() { + local commit="$1" + local matches + + # Check to see if this commit includes any non-ignored files. + matches=$(git -C "$dir" diff-tree --no-commit-id --name-only -r "$commit^" "$commit" \ + -- "${IGNORE_FILES_PATHSPEC[@]}" | wc -l) + [[ "$matches" -eq 0 ]] +} + # Generate a release log for a range of commits in a single repo. release_log() { setopt local_options BASH_REMATCH @@ -119,9 +173,10 @@ release_log() { --first-parent \ "$start..$end" | while read commit subject; do - # Skip gx-only PRs. - git -C "$dir" diff-tree --no-commit-id --name-only "$commit^" "$commit" | - grep -v "${IGNORED_FILES}" >/dev/null || continue + # Skip commits that only touch ignored files. + if ignored_commit "$commit"; then + continue + fi if [[ "$subject" =~ '^Merge pull request #([0-9]+) from' ]]; then local prnum="${BASH_REMATCH[2]}" @@ -207,11 +262,11 @@ recursive_release_log() { printf -- "- %s:\n" "$module" release_log "$module" "$start" "$end" | indent - statlog "$module" "$start" "$end" > statlog.json dep_changes old_deps.json new_deps.json | - jq --arg filter "$REPO_FILTER" 'select(.Path | match($filter))' | + jq --arg inc "$INCLUDE_REGEX" --arg exc "$EXCLUDE_REGEX" \ + 'select(.Path | test($inc)) | select(.Path | test($exc) | not)' | # Compute changelogs jq -r '"\(.Path) \(.New.Version) \(.New.Ref) \(.Old.Version) \(.Old.Ref // "")"' | while read module new new_ref old old_ref; do diff --git a/bin/push-docker-tags.sh b/bin/push-docker-tags.sh index 464a0f8942b..00d314c07a8 100755 --- a/bin/push-docker-tags.sh +++ b/bin/push-docker-tags.sh @@ -29,18 +29,18 @@ # set -euo pipefail -if [[ $# -lt 3 ]] ; then - echo 'At least 3 args required. Pass 5 args for a dry run.' +if [[ $# -lt 1 ]] ; then + echo 'At least 1 arg required. Pass 5 args for a dry run.' echo 'Usage:' - echo './push-docker-tags.sh [git tag name] [dry run]' + echo './push-docker-tags.sh [git commit sha1] [git branch name] [git tag name] [dry run]' exit 1 fi BUILD_NUM=$1 -GIT_SHA1=$2 +GIT_SHA1=${2:-$(git rev-parse HEAD)} GIT_SHA1_SHORT=$(echo "$GIT_SHA1" | cut -c 1-7) -GIT_BRANCH=$3 -GIT_TAG=${4:-""} +GIT_BRANCH=${3:-$(git symbolic-ref -q --short HEAD || echo "unknown")} +GIT_TAG=${4:-$(git describe --tags --exact-match || echo "")} DRY_RUN=${5:-false} WIP_IMAGE_TAG=${WIP_IMAGE_TAG:-wip} @@ -67,9 +67,10 @@ elif [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then pushTag "latest" pushTag "release" # see: https://github.com/ipfs/go-ipfs/issues/3999#issuecomment-742228981 -elif [ "$GIT_BRANCH" = "feat/stabilize-dht" ]; then - pushTag "bifrost-${BUILD_NUM}-${GIT_SHA1_SHORT}" - pushTag "bifrost-latest" +elif [[ $GIT_BRANCH =~ ^bifrost-.* ]]; then + # sanitize the branch name since docker tags have stricter char limits than git branch names + branch=$(echo "$GIT_BRANCH" | tr '/' '-' | tr --delete --complement '[:alnum:]-') + pushTag "${branch}-${BUILD_NUM}-${GIT_SHA1_SHORT}" elif [ "$GIT_BRANCH" = "master" ]; then pushTag "master-${BUILD_NUM}-${GIT_SHA1_SHORT}" diff --git a/cmd/ipfs/migration.go b/cmd/ipfs/add_migrations.go similarity index 52% rename from cmd/ipfs/migration.go rename to cmd/ipfs/add_migrations.go index d4e3e15b9d0..b5b6c31a137 100644 --- a/cmd/ipfs/migration.go +++ b/cmd/ipfs/add_migrations.go @@ -2,17 +2,13 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "io" "io/ioutil" - "net/url" "os" "path/filepath" - "strings" - config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreapi" @@ -24,140 +20,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -// readMigrationConfig reads the migration config out of the config, avoiding -// reading anything other than the migration section. That way, we're free to -// make arbitrary changes to all _other_ sections in migrations. -func readMigrationConfig(repoRoot string) (*config.Migration, error) { - var cfg struct { - Migration config.Migration - } - - cfgPath, err := config.Filename(repoRoot) - if err != nil { - return nil, err - } - - cfgFile, err := os.Open(cfgPath) - if err != nil { - return nil, err - } - defer cfgFile.Close() - - err = json.NewDecoder(cfgFile).Decode(&cfg) - if err != nil { - return nil, err - } - - switch cfg.Migration.Keep { - case "": - cfg.Migration.Keep = config.DefaultMigrationKeep - case "discard", "cache", "keep": - default: - return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'") - } - - if len(cfg.Migration.DownloadSources) == 0 { - cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources - } - - return &cfg.Migration, nil -} - -func readIpfsConfig(repoRoot *string) (bootstrap []string, peers []peer.AddrInfo) { - if repoRoot == nil { - return - } - - cfgPath, err := config.Filename(*repoRoot) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - - cfgFile, err := os.Open(cfgPath) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - defer cfgFile.Close() - - // Attempt to read bootstrap addresses - var bootstrapCfg struct { - Bootstrap []string - } - err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg) - if err != nil { - fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config") - } else { - bootstrap = bootstrapCfg.Bootstrap - } - - if _, err = cfgFile.Seek(0, 0); err != nil { - fmt.Fprintln(os.Stderr, err) - } - - // Attempt to read peers - var peeringCfg struct { - Peering config.Peering - } - err = json.NewDecoder(cfgFile).Decode(&peeringCfg) - if err != nil { - fmt.Fprintln(os.Stderr, "cannot read peering from config") - } else { - peers = peeringCfg.Peering.Peers - } - - return -} - -// getMigrationFetcher creates one or more fetchers according to -// config.Migration.DownloadSources. If an IpfsFetcher is required, then -// bootstrap and peer information in read from the config file in repoRoot, -// unless repoRoot is nil. -func getMigrationFetcher(cfg *config.Migration, repoRoot *string) (migrations.Fetcher, error) { - const httpUserAgent = "go-ipfs" - - // Fetch migrations from current distribution, or location from environ - fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist) - - var fetchers []migrations.Fetcher - for _, src := range cfg.DownloadSources { - src := strings.TrimSpace(src) - switch src { - case "IPFS", "ipfs": - bootstrap, peers := readIpfsConfig(repoRoot) - fetchers = append(fetchers, ipfsfetcher.NewIpfsFetcher(fetchDistPath, 0, bootstrap, peers)) - case "HTTPS", "https", "HTTP", "http": - fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, "", httpUserAgent, 0)) - default: - u, err := url.Parse(src) - if err != nil { - return nil, fmt.Errorf("bad gateway address: %s", err) - } - switch u.Scheme { - case "": - u.Scheme = "https" - case "https", "http": - default: - return nil, errors.New("bad gateway address: url scheme must be http or https") - } - fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, u.String(), httpUserAgent, 0)) - case "": - // Ignore empty string - } - } - if len(fetchers) == 0 { - return nil, errors.New("no sources specified") - } - - if len(fetchers) == 1 { - return fetchers[0], nil - } - - // Wrap fetchers in a MultiFetcher to try them in order - return migrations.NewMultiFetcher(fetchers...), nil -} - +// addMigrations adds any migration downloaded by the fetcher to the IPFS node func addMigrations(ctx context.Context, node *core.IpfsNode, fetcher migrations.Fetcher, pin bool) error { var fetchers []migrations.Fetcher if mf, ok := fetcher.(*migrations.MultiFetcher); ok { diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index f93152b316f..c2cefbafead 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -30,6 +30,7 @@ import ( nodeMount "github.com/ipfs/go-ipfs/fuse/node" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" + "github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher" sockets "github.com/libp2p/go-socket-activation" cmds "github.com/ipfs/go-ipfs-cmds" @@ -66,6 +67,7 @@ const ( enablePubSubKwd = "enable-pubsub-experiment" enableIPNSPubSubKwd = "enable-namesys-pubsub" enableMultiplexKwd = "enable-mplex-experiment" + agentVersionSuffix = "agent-version-suffix" // apiAddrKwd = "address-api" // swarmAddrKwd = "address-swarm" ) @@ -179,6 +181,7 @@ Headers. cmds.BoolOption(enablePubSubKwd, "Instantiate the ipfs daemon with the experimental pubsub feature enabled."), cmds.BoolOption(enableIPNSPubSubKwd, "Enable IPNS record distribution through pubsub; enables pubsub."), cmds.BoolOption(enableMultiplexKwd, "DEPRECATED"), + cmds.StringOption(agentVersionSuffix, "Optional suffix to the AgentVersion presented by `ipfs id` and also advertised through BitSwap."), // TODO: add way to override addresses. tricky part: updating the config if also --init. // cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"), @@ -294,12 +297,26 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("fs-repo requires migration") } - migrationCfg, err := readMigrationConfig(cctx.ConfigRoot) + // Read Migration section of IPFS config + migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot) if err != nil { return err } - fetcher, err = getMigrationFetcher(migrationCfg, &cctx.ConfigRoot) + // Define function to create IPFS fetcher. Do not supply an + // already-constructed IPFS fetcher, because this may be expensive and + // not needed according to migration config. Instead, supply a function + // to construct the particular IPFS fetcher implementation used here, + // which is called only if an IPFS fetcher is needed. + newIpfsFetcher := func(distPath string) migrations.Fetcher { + return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot) + } + + // Fetch migrations from current distribution, or location from environ + fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist) + + // Create fetchers according to migrationCfg.DownloadSources + fetcher, err = migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher) if err != nil { return err } @@ -395,6 +412,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("unrecognized routing option: %s", routingOption) } + agentVersionSuffixString, _ := req.Options[agentVersionSuffix].(string) + if agentVersionSuffixString != "" { + version.SetUserAgentSuffix(agentVersionSuffixString) + } + node, err := core.NewNode(req.Context, ncfg) if err != nil { log.Error("error from node construction: ", err) diff --git a/cmd/ipfs/migration_test.go b/cmd/ipfs/migration_test.go deleted file mode 100644 index da35bf3029c..00000000000 --- a/cmd/ipfs/migration_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package main - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - config "github.com/ipfs/go-ipfs-config" - "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" - "github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher" -) - -var testConfig = ` -{ - "Bootstrap": [ - "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", - "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" - ], - "Migration": { - "DownloadSources": ["IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"], - "Keep": "cache" - }, - "Peering": { - "Peers": [ - { - "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", - "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] - } - ] - } -} -` - -func TestReadMigrationConfigDefaults(t *testing.T) { - tmpDir := makeConfig("{}") - defer os.RemoveAll(tmpDir) - - cfg, err := readMigrationConfig(tmpDir) - if err != nil { - t.Fatal(err) - } - - if cfg.Keep != config.DefaultMigrationKeep { - t.Error("expected default value for Keep") - } - - if len(cfg.DownloadSources) != len(config.DefaultMigrationDownloadSources) { - t.Fatal("expected default number of download sources") - } - for i, src := range config.DefaultMigrationDownloadSources { - if cfg.DownloadSources[i] != src { - t.Errorf("wrong DownloadSource: %s", cfg.DownloadSources[i]) - } - } -} - -func TestReadMigrationConfigErrors(t *testing.T) { - tmpDir := makeConfig(`{"Migration": {"Keep": "badvalue"}}`) - defer os.RemoveAll(tmpDir) - - _, err := readMigrationConfig(tmpDir) - if err == nil { - t.Fatal("expected error") - } - if !strings.HasPrefix(err.Error(), "unknown") { - t.Fatal("did not get expected error:", err) - } - - os.RemoveAll(tmpDir) - _, err = readMigrationConfig(tmpDir) - if err == nil { - t.Fatal("expected error") - } - - bootstrap, peers := readIpfsConfig(&tmpDir) - if bootstrap != nil { - t.Error("expected nil bootstrap") - } - if peers != nil { - t.Error("expected nil peers") - } - - tmpDir = makeConfig(`}{`) - defer os.RemoveAll(tmpDir) - _, err = readMigrationConfig(tmpDir) - if err == nil { - t.Fatal("expected error") - } -} - -func TestReadMigrationConfig(t *testing.T) { - tmpDir := makeConfig(testConfig) - defer os.RemoveAll(tmpDir) - - cfg, err := readMigrationConfig(tmpDir) - if err != nil { - t.Fatal(err) - } - - if len(cfg.DownloadSources) != 4 { - t.Fatal("wrong number of DownloadSources") - } - expect := []string{"IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"} - for i := range expect { - if cfg.DownloadSources[i] != expect[i] { - t.Errorf("wrong DownloadSource at %d", i) - } - } - - if cfg.Keep != "cache" { - t.Error("wrong value for Keep") - } -} - -func TestReadIpfsConfig(t *testing.T) { - tmpDir := makeConfig(testConfig) - defer os.RemoveAll(tmpDir) - - bootstrap, peers := readIpfsConfig(nil) - if bootstrap != nil || peers != nil { - t.Fatal("expected nil ipfs config items") - } - - bootstrap, peers = readIpfsConfig(&tmpDir) - if len(bootstrap) != 2 { - t.Fatal("wrong number of bootstrap addresses") - } - if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" { - t.Fatal("wrong bootstrap address") - } - - if len(peers) != 1 { - t.Fatal("wrong number of peers") - } - - peer := peers[0] - if peer.ID.String() != "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5" { - t.Errorf("wrong ID for first peer") - } - if len(peer.Addrs) != 2 { - t.Error("wrong number of addrs for first peer") - } -} - -func TestReadPartialIpfsConfig(t *testing.T) { - const ( - configBadBootstrap = ` -{ - "Bootstrap": "unreadable", - "Migration": { - "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], - "Keep": "cache" - }, - "Peering": { - "Peers": [ - { - "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", - "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] - } - ] - } -} -` - configBadPeers = ` -{ - "Bootstrap": [ - "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", - "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" - ], - "Migration": { - "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], - "Keep": "cache" - }, - "Peering": "Unreadable-data" -} -` - ) - - tmpDir := makeConfig(configBadBootstrap) - defer os.RemoveAll(tmpDir) - - bootstrap, peers := readIpfsConfig(&tmpDir) - if bootstrap != nil { - t.Fatal("expected nil bootstrap") - } - if len(peers) != 1 { - t.Fatal("wrong number of peers") - } - if len(peers[0].Addrs) != 2 { - t.Error("wrong number of addrs for first peer") - } - os.RemoveAll(tmpDir) - - tmpDir = makeConfig(configBadPeers) - defer os.RemoveAll(tmpDir) - - bootstrap, peers = readIpfsConfig(&tmpDir) - if peers != nil { - t.Fatal("expected nil peers") - } - if len(bootstrap) != 2 { - t.Fatal("wrong number of bootstrap addresses") - } - if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" { - t.Fatal("wrong bootstrap address") - } -} - -func makeConfig(configData string) string { - tmpDir, err := ioutil.TempDir("", "migration_test") - if err != nil { - panic(err) - } - - cfgFile, err := os.Create(filepath.Join(tmpDir, "config")) - if err != nil { - panic(err) - } - if _, err = cfgFile.Write([]byte(configData)); err != nil { - panic(err) - } - if err = cfgFile.Close(); err != nil { - panic(err) - } - return tmpDir -} - -func TestGetMigrationFetcher(t *testing.T) { - var f migrations.Fetcher - var err error - - cfg := &config.Migration{} - - cfg.DownloadSources = []string{"ftp://bad.gateway.io"} - _, err = getMigrationFetcher(cfg, nil) - if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") { - t.Fatal("Expected bad gateway address error, got:", err) - } - - cfg.DownloadSources = []string{"::bad.gateway.io"} - _, err = getMigrationFetcher(cfg, nil) - if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") { - t.Fatal("Expected bad gateway address error, got:", err) - } - - cfg.DownloadSources = []string{"http://localhost"} - f, err = getMigrationFetcher(cfg, nil) - if err != nil { - t.Fatal(err) - } - if _, ok := f.(*migrations.HttpFetcher); !ok { - t.Fatal("expected HttpFetcher") - } - - cfg.DownloadSources = []string{"ipfs"} - f, err = getMigrationFetcher(cfg, nil) - if err != nil { - t.Fatal(err) - } - if _, ok := f.(*ipfsfetcher.IpfsFetcher); !ok { - t.Fatal("expected IpfsFetcher") - } - - cfg.DownloadSources = []string{"http"} - f, err = getMigrationFetcher(cfg, nil) - if err != nil { - t.Fatal(err) - } - if _, ok := f.(*migrations.HttpFetcher); !ok { - t.Fatal("expected HttpFetcher") - } - - cfg.DownloadSources = []string{"IPFS", "HTTPS"} - f, err = getMigrationFetcher(cfg, nil) - if err != nil { - t.Fatal(err) - } - mf, ok := f.(*migrations.MultiFetcher) - if !ok { - t.Fatal("expected MultiFetcher") - } - if mf.Len() != 2 { - t.Fatal("expected 2 fetchers in MultiFetcher") - } - - cfg.DownloadSources = []string{"ipfs", "https", "some.domain.io"} - f, err = getMigrationFetcher(cfg, nil) - if err != nil { - t.Fatal(err) - } - mf, ok = f.(*migrations.MultiFetcher) - if !ok { - t.Fatal("expected MultiFetcher") - } - if mf.Len() != 3 { - t.Fatal("expected 3 fetchers in MultiFetcher") - } - - cfg.DownloadSources = nil - _, err = getMigrationFetcher(cfg, nil) - if err == nil { - t.Fatal("expected error when no sources specified") - } - - cfg.DownloadSources = []string{"", ""} - _, err = getMigrationFetcher(cfg, nil) - if err == nil { - t.Fatal("expected error when empty string fetchers specified") - } -} diff --git a/core/commands/commands.go b/core/commands/commands.go index b45ac54c638..27be029c015 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -6,6 +6,7 @@ package commands import ( + "bytes" "fmt" "io" "os" @@ -63,6 +64,9 @@ func CommandsCmd(root *cmds.Command) *cmds.Command { Tagline: "List all available commands.", ShortDescription: `Lists all available commands (and subcommands) and exits.`, }, + Subcommands: map[string]*cmds.Command{ + "completion": CompletionCmd(root), + }, Options: []cmds.Option{ cmds.BoolOption(flagsOptionName, "f", "Show command flags"), }, @@ -131,6 +135,44 @@ func cmdPathStrings(cmd *Command, showOptions bool) []string { return cmds } +func CompletionCmd(root *cmds.Command) *cmds.Command { + return &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Generate shell completions.", + }, + NoRemote: true, + Subcommands: map[string]*cmds.Command{ + "bash": { + Helptext: cmds.HelpText{ + Tagline: "Generate bash shell completions.", + ShortDescription: "Generates command completions for the bash shell.", + LongDescription: ` +Generates command completions for the bash shell. + +The simplest way to see it working is write the completions +to a file and then source it: + + > ipfs commands completion bash > ipfs-completion.bash + > source ./ipfs-completion.bash + +To install the completions permanently, they can be moved to +/etc/bash_completion.d or sourced from your ~/.bashrc file. +`, + }, + NoRemote: true, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + var buf bytes.Buffer + if err := writeBashCompletions(root, &buf); err != nil { + return err + } + res.SetLength(uint64(buf.Len())) + return res.Emit(&buf) + }, + }, + }, + } +} + type nonFatalError string // streamResult is a helper function to stream results that possibly diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 81f07c01bf4..964baad9299 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -22,6 +22,8 @@ func TestROCommands(t *testing.T) { "/block/stat", "/cat", "/commands", + "/commands/completion", + "/commands/completion/bash", "/dag", "/dag/get", "/dag/resolve", @@ -88,18 +90,26 @@ func TestCommands(t *testing.T) { "/bootstrap/rm", "/bootstrap/rm/all", "/cat", + "/cid", + "/cid/base32", + "/cid/bases", + "/cid/codecs", + "/cid/format", + "/cid/hashes", "/commands", + "/commands/completion", + "/commands/completion/bash", "/config", "/config/edit", - "/config/replace", - "/config/show", "/config/profile", "/config/profile/apply", + "/config/replace", + "/config/show", "/dag", - "/dag/get", "/dag/export", - "/dag/put", + "/dag/get", "/dag/import", + "/dag/put", "/dag/resolve", "/dag/stat", "/dht", @@ -113,6 +123,7 @@ func TestCommands(t *testing.T) { "/diag/cmds", "/diag/cmds/clear", "/diag/cmds/set-time", + "/diag/profile", "/diag/sys", "/dns", "/file", @@ -127,16 +138,16 @@ func TestCommands(t *testing.T) { "/files/read", "/files/rm", "/files/stat", + "/files/write", "/filestore", "/filestore/dups", "/filestore/ls", "/filestore/verify", - "/files/write", "/get", "/id", "/key", - "/key/gen", "/key/export", + "/key/gen", "/key/import", "/key/list", "/key/rename", @@ -148,12 +159,17 @@ func TestCommands(t *testing.T) { "/log/tail", "/ls", "/mount", + "/multibase", + "/multibase/decode", + "/multibase/encode", + "/multibase/transcode", + "/multibase/list", "/name", "/name/publish", "/name/pubsub", + "/name/pubsub/cancel", "/name/pubsub/state", "/name/pubsub/subs", - "/name/pubsub/cancel", "/name/resolve", "/object", "/object/data", @@ -222,6 +238,10 @@ func TestCommands(t *testing.T) { "/swarm/filters/add", "/swarm/filters/rm", "/swarm/peers", + "/swarm/peering", + "/swarm/peering/add", + "/swarm/peering/ls", + "/swarm/peering/rm", "/tar", "/tar/add", "/tar/cat", @@ -230,12 +250,6 @@ func TestCommands(t *testing.T) { "/urlstore/add", "/version", "/version/deps", - "/cid", - "/cid/format", - "/cid/base32", - "/cid/codecs", - "/cid/bases", - "/cid/hashes", } cmdSet := make(map[string]struct{}) diff --git a/core/commands/completion.go b/core/commands/completion.go new file mode 100644 index 00000000000..ed1865d2677 --- /dev/null +++ b/core/commands/completion.go @@ -0,0 +1,142 @@ +package commands + +import ( + "io" + "sort" + "text/template" + + cmds "github.com/ipfs/go-ipfs-cmds" +) + +type completionCommand struct { + Name string + Subcommands []*completionCommand + ShortFlags []string + ShortOptions []string + LongFlags []string + LongOptions []string +} + +func commandToCompletions(name string, cmd *cmds.Command) *completionCommand { + parsed := &completionCommand{ + Name: name, + } + for name, subCmd := range cmd.Subcommands { + parsed.Subcommands = append(parsed.Subcommands, commandToCompletions(name, subCmd)) + } + sort.Slice(parsed.Subcommands, func(i, j int) bool { + return parsed.Subcommands[i].Name < parsed.Subcommands[j].Name + }) + + for _, opt := range cmd.Options { + if opt.Type() == cmds.Bool { + parsed.LongFlags = append(parsed.LongFlags, opt.Name()) + for _, name := range opt.Names() { + if len(name) == 1 { + parsed.ShortFlags = append(parsed.ShortFlags, name) + break + } + } + } else { + parsed.LongOptions = append(parsed.LongOptions, opt.Name()) + for _, name := range opt.Names() { + if len(name) == 1 { + parsed.ShortOptions = append(parsed.ShortOptions, name) + break + } + } + } + } + sort.Slice(parsed.LongFlags, func(i, j int) bool { + return parsed.LongFlags[i] < parsed.LongFlags[j] + }) + sort.Slice(parsed.ShortFlags, func(i, j int) bool { + return parsed.ShortFlags[i] < parsed.ShortFlags[j] + }) + sort.Slice(parsed.LongOptions, func(i, j int) bool { + return parsed.LongOptions[i] < parsed.LongOptions[j] + }) + sort.Slice(parsed.ShortOptions, func(i, j int) bool { + return parsed.ShortOptions[i] < parsed.ShortOptions[j] + }) + return parsed +} + +var bashCompletionTemplate *template.Template + +func init() { + commandTemplate := template.Must(template.New("command").Parse(` +while [[ ${index} -lt ${COMP_CWORD} ]]; do + case "${COMP_WORDS[index]}" in + -*) + let index++ + continue + ;; + {{ range .Subcommands }} + "{{ .Name }}") + let index++ + {{ template "command" . }} + return 0 + ;; + {{ end }} + esac + break +done + +if [[ "${word}" == -* ]]; then +{{ if .ShortFlags -}} + _ipfs_compgen -W $'{{ range .ShortFlags }}-{{.}} \n{{end}}' -- "${word}" +{{ end -}} +{{- if .ShortOptions -}} + _ipfs_compgen -S = -W $'{{ range .ShortOptions }}-{{.}}\n{{end}}' -- "${word}" +{{ end -}} +{{- if .LongFlags -}} + _ipfs_compgen -W $'{{ range .LongFlags }}--{{.}} \n{{end}}' -- "${word}" +{{ end -}} +{{- if .LongOptions -}} + _ipfs_compgen -S = -W $'{{ range .LongOptions }}--{{.}}\n{{end}}' -- "${word}" +{{ end -}} + return 0 +fi + +while [[ ${index} -lt ${COMP_CWORD} ]]; do + if [[ "${COMP_WORDS[index]}" != -* ]]; then + let argidx++ + fi + let index++ +done + +{{- if .Subcommands }} +if [[ "${argidx}" -eq 0 ]]; then + _ipfs_compgen -W $'{{ range .Subcommands }}{{.Name}} \n{{end}}' -- "${word}" +fi +{{ end -}} +`)) + + bashCompletionTemplate = template.Must(commandTemplate.New("root").Parse(`#!/bin/bash + +_ipfs_compgen() { + local oldifs="$IFS" + IFS=$'\n' + while read -r line; do + COMPREPLY+=("$line") + done < <(compgen "$@") + IFS="$oldifs" +} + +_ipfs() { + COMPREPLY=() + local index=1 + local argidx=0 + local word="${COMP_WORDS[COMP_CWORD]}" + {{ template "command" . }} +} +complete -o nosort -o nospace -o default -F _ipfs ipfs +`)) +} + +// writeBashCompletions generates a bash completion script for the given command tree. +func writeBashCompletions(cmd *cmds.Command, out io.Writer) error { + cmds := commandToCompletions("ipfs", cmd) + return bashCompletionTemplate.Execute(out, cmds) +} diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index cdfa8ce9661..cbaf7f93fe1 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -16,9 +16,10 @@ import ( ) const ( + pinRootsOptionName = "pin-roots" progressOptionName = "progress" silentOptionName = "silent" - pinRootsOptionName = "pin-roots" + statsOptionName = "stats" ) // DagCmd provides a subset of commands for interacting with ipld dag objects @@ -53,9 +54,15 @@ type ResolveOutput struct { RemPath string } +type CarImportStats struct { + BlockCount uint64 + BlockBytesCount uint64 +} + // CarImportOutput is the output type of the 'dag import' commands type CarImportOutput struct { - Root RootMeta + Root *RootMeta `json:",omitempty"` + Stats *CarImportStats `json:",omitempty"` } // RootMeta is the metadata for a root pinning response @@ -77,10 +84,10 @@ into an object of the specified format. cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"), - cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"), + cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"), + cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), - cmds.StringOption("hash", "Hash function to use").WithDefault(""), + cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), }, Run: dagPut, Type: OutputObject{}, @@ -108,6 +115,9 @@ format. Arguments: []cmds.Argument{ cmds.StringArg("ref", true, false, "The object to get").EnableStdin(), }, + Options: []cmds.Option{ + cmds.StringOption("output-codec", "Format that the object will be encoded as.").WithDefault("dag-json"), + }, Run: dagGet, } @@ -157,8 +167,10 @@ var DagResolveCmd = &cmds.Command{ } type importResult struct { - roots map[cid.Cid]struct{} - err error + blockCount uint64 + blockBytesCount uint64 + roots map[cid.Cid]struct{} + err error } // DagImportCmd is a command for importing a car to ipfs @@ -190,8 +202,9 @@ Maximum supported CAR version: 1 cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(silentOptionName, "No output."), + cmds.BoolOption(statsOptionName, "Output stats."), }, Type: CarImportOutput{}, Run: dagImport, @@ -203,6 +216,22 @@ Maximum supported CAR version: 1 return nil } + // event should have only one of `Root` or `Stats` set, not both + if event.Root == nil { + if event.Stats == nil { + return fmt.Errorf("Unexpected message from DAG import") + } + stats, _ := req.Options[statsOptionName].(bool) + if stats { + fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount) + } + return nil + } + + if event.Stats != nil { + return fmt.Errorf("Unexpected message from DAG import") + } + enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err diff --git a/core/commands/dag/get.go b/core/commands/dag/get.go index a5f92273ae3..546ba4e5d19 100644 --- a/core/commands/dag/get.go +++ b/core/commands/dag/get.go @@ -1,11 +1,18 @@ package dagcmd import ( - "strings" + "fmt" + "io" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/traversal" + mc "github.com/multiformats/go-multicodec" + cmds "github.com/ipfs/go-ipfs-cmds" ) @@ -15,6 +22,12 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } + codecStr, _ := req.Options["output-codec"].(string) + var codec mc.Code + if err := codec.Set(codecStr); err != nil { + return err + } + rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0])) if err != nil { return err @@ -25,14 +38,34 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } - var out interface{} = obj + universal, ok := obj.(ipldlegacy.UniversalNode) + if !ok { + return fmt.Errorf("%T is not a valid IPLD node", obj) + } + + finalNode := universal.(ipld.Node) + if len(rp.Remainder()) > 0 { - rem := strings.Split(rp.Remainder(), "/") - final, _, err := obj.Resolve(rem) + remainderPath := ipld.ParsePath(rp.Remainder()) + + finalNode, err = traversal.Get(finalNode, remainderPath) if err != nil { return err } - out = final } - return cmds.EmitOnce(res, &out) + + encoder, err := multicodec.LookupEncoder(uint64(codec)) + if err != nil { + return fmt.Errorf("invalid encoding: %s - %s", codec, err) + } + + r, w := io.Pipe() + go func() { + defer w.Close() + if err := encoder(finalNode, w); err != nil { + _ = w.CloseWithError(err) + } + }() + + return res.Emit(r) } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index b83af8b911d..244260601bf 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment failedPins++ } - if err := res.Emit(&CarImportOutput{Root: ret}); err != nil { + if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil { return err } } @@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } } + stats, _ := req.Options[statsOptionName].(bool) + if stats { + err = res.Emit(&CarImportOutput{ + Stats: &CarImportStats{ + BlockCount: done.blockCount, + BlockBytesCount: done.blockBytesCount, + }, + }) + if err != nil { + return err + } + } + return nil } @@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, batch := ipld.NewBatch(req.Context, api.Dag()) roots := make(map[cid.Cid]struct{}) + var blockCount, blockBytesCount uint64 it := req.Files.Entries() for it.Next() { @@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, if err := batch.Add(req.Context, nd); err != nil { return err } + blockCount++ + blockBytesCount += uint64(len(block.RawData())) } return nil @@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return } - ret <- importResult{roots: roots} + ret <- importResult{ + blockCount: blockCount, + blockBytesCount: blockBytesCount, + roots: roots} } diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 7f6e744c872..0bb7fd2ae50 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -1,16 +1,28 @@ package dagcmd import ( + "bytes" "fmt" - "math" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/commands/cmdenv" - "github.com/ipfs/go-ipfs/core/coredag" + ipldlegacy "github.com/ipfs/go-ipld-legacy" + "github.com/ipld/go-ipld-prime/multicodec" + basicnode "github.com/ipld/go-ipld-prime/node/basic" cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" + mc "github.com/multiformats/go-multicodec" + + // Expected minimal set of available format/ienc codecs. + _ "github.com/ipld/go-codec-dagpb" + _ "github.com/ipld/go-ipld-prime/codec/cbor" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/dagjson" + _ "github.com/ipld/go-ipld-prime/codec/json" + _ "github.com/ipld/go-ipld-prime/codec/raw" ) func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -19,21 +31,38 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } - ienc, _ := req.Options["input-enc"].(string) - format, _ := req.Options["format"].(string) + inputCodec, _ := req.Options["input-codec"].(string) + storeCodec, _ := req.Options["store-codec"].(string) hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) - // mhType tells inputParser which hash should be used. MaxUint64 means 'use - // default hash' (sha256 for cbor, sha1 for git..) - mhType := uint64(math.MaxUint64) + var icodec mc.Code + if err := icodec.Set(inputCodec); err != nil { + return err + } + var scodec mc.Code + if err := scodec.Set(storeCodec); err != nil { + return err + } + var mhType mc.Code + if err := mhType.Set(hash); err != nil { + return err + } - if hash != "" { - var ok bool - mhType, ok = mh.Names[hash] - if !ok { - return fmt.Errorf("%s in not a valid multihash name", hash) - } + cidPrefix := cid.Prefix{ + Version: 1, + Codec: uint64(scodec), + MhType: uint64(mhType), + MhLength: -1, + } + + decoder, err := multicodec.LookupDecoder(uint64(icodec)) + if err != nil { + return err + } + encoder, err := multicodec.LookupEncoder(uint64(scodec)) + if err != nil { + return err } var adder ipld.NodeAdder = api.Dag() @@ -48,22 +77,36 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e if file == nil { return fmt.Errorf("expected a regular file") } - nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1) + + node := basicnode.Prototype.Any.NewBuilder() + if err := decoder(node, file); err != nil { + return err + } + n := node.Build() + + bd := bytes.NewBuffer([]byte{}) + if err := encoder(n, bd); err != nil { + return err + } + + blockCid, err := cidPrefix.Sum(bd.Bytes()) + if err != nil { + return err + } + blk, err := blocks.NewBlockWithCid(bd.Bytes(), blockCid) if err != nil { return err } - if len(nds) == 0 { - return fmt.Errorf("no node returned from ParseInputs") + ln := ipldlegacy.LegacyNode{ + Block: blk, + Node: n, } - for _, nd := range nds { - err := b.Add(req.Context, nd) - if err != nil { - return err - } + if err := b.Add(req.Context, &ln); err != nil { + return err } - cid := nds[0].Cid() + cid := ln.Cid() if err := res.Emit(&OutputObject{Cid: cid}); err != nil { return err } diff --git a/core/commands/diag.go b/core/commands/diag.go index d7ddb6dfd13..89b46381f1e 100644 --- a/core/commands/diag.go +++ b/core/commands/diag.go @@ -10,7 +10,8 @@ var DiagCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "sys": sysDiagCmd, - "cmds": ActiveReqsCmd, + "sys": sysDiagCmd, + "cmds": ActiveReqsCmd, + "profile": sysProfileCmd, }, } diff --git a/core/commands/files.go b/core/commands/files.go index 927ce51e2cf..f081af799fb 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -356,12 +356,21 @@ GC'ed. cmds.StringArg("source", true, false, "Source IPFS or MFS path to copy."), cmds.StringArg("dest", true, false, "Destination within MFS."), }, + Options: []cmds.Option{ + cmds.BoolOption(filesParentsOptionName, "p", "Make parent directories as needed."), + }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + mkParents, _ := req.Options[filesParentsOptionName].(bool) nd, err := cmdenv.GetNode(env) if err != nil { return err } + prefix, err := getPrefixNew(req) + if err != nil { + return err + } + api, err := cmdenv.GetApi(env, req) if err != nil { return err @@ -389,6 +398,13 @@ GC'ed. return fmt.Errorf("cp: cannot get node from path %s: %s", src, err) } + if mkParents { + err := ensureContainingDirectoryExists(nd.FilesRoot, dst, prefix) + if err != nil { + return err + } + } + err = mfs.PutNode(nd.FilesRoot, dst, node) if err != nil { return fmt.Errorf("cp: cannot put node in path %s: %s", dst, err) diff --git a/core/commands/id.go b/core/commands/id.go index b77ad7ddbe9..06a70bf35b5 100644 --- a/core/commands/id.go +++ b/core/commands/id.go @@ -223,6 +223,6 @@ func printSelf(keyEnc ke.KeyEncoder, node *core.IpfsNode) (interface{}, error) { sort.Strings(info.Protocols) } info.ProtocolVersion = identify.LibP2PVersion - info.AgentVersion = version.UserAgent + info.AgentVersion = version.GetUserAgentVersion() return info, nil } diff --git a/core/commands/keyencode/keyencode.go b/core/commands/keyencode/keyencode.go index 1db095d6d83..bdffac6ada3 100644 --- a/core/commands/keyencode/keyencode.go +++ b/core/commands/keyencode/keyencode.go @@ -29,8 +29,7 @@ func KeyEncoderFromString(formatLabel string) (KeyEncoder, error) { func (enc KeyEncoder) FormatID(id peer.ID) string { if enc.baseEnc == nil { - //nolint deprecated - return peer.IDB58Encode(id) + return peer.Encode(id) } if s, err := peer.ToCid(id).StringOfBase(enc.baseEnc.Encoding()); err != nil { panic(err) diff --git a/core/commands/multibase.go b/core/commands/multibase.go new file mode 100644 index 00000000000..83f3ac2b4c6 --- /dev/null +++ b/core/commands/multibase.go @@ -0,0 +1,171 @@ +package commands + +import ( + "bytes" + "fmt" + "io/ioutil" + "strings" + + cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/go-ipfs/core/commands/cmdenv" + mbase "github.com/multiformats/go-multibase" +) + +var MbaseCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Encode and decode files or stdin with multibase format", + }, + Subcommands: map[string]*cmds.Command{ + "encode": mbaseEncodeCmd, + "decode": mbaseDecodeCmd, + "transcode": mbaseTranscodeCmd, + "list": basesCmd, + }, + Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), +} + +const ( + mbaseOptionName = "b" +) + +var mbaseEncodeCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Encode data into multibase string", + LongDescription: ` +This command expects a file name or data provided via stdin. + +By default it will use URL-safe base64url encoding, +but one can customize used base with -b: + + > echo hello | ipfs multibase encode -b base16 > output_file + > cat output_file + f68656c6c6f0a + + > echo hello > input_file + > ipfs multibase encode -b base16 input_file + f68656c6c6f0a + `, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("file", true, false, "data to encode").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(mbaseOptionName, "multibase encoding").WithDefault("base64url"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + if err := req.ParseBodyArgs(); err != nil { + return err + } + encoderName, _ := req.Options[mbaseOptionName].(string) + encoder, err := mbase.EncoderByName(encoderName) + if err != nil { + return err + } + files := req.Files.Entries() + file, err := cmdenv.GetFileArg(files) + if err != nil { + return fmt.Errorf("failed to access file: %w", err) + } + buf, err := ioutil.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + encoded := encoder.Encode(buf) + reader := strings.NewReader(encoded) + return resp.Emit(reader) + }, +} + +var mbaseDecodeCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Decode multibase string", + LongDescription: ` +This command expects multibase inside of a file or via stdin: + + > echo -n hello | ipfs multibase encode -b base16 > file + > cat file + f68656c6c6f + + > ipfs multibase decode file + hello + + > cat file | ipfs multibase decode + hello +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("encoded_file", true, false, "encoded data to decode").EnableStdin(), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + if err := req.ParseBodyArgs(); err != nil { + return err + } + files := req.Files.Entries() + file, err := cmdenv.GetFileArg(files) + if err != nil { + return fmt.Errorf("failed to access file: %w", err) + } + encoded_data, err := ioutil.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + _, data, err := mbase.Decode(string(encoded_data)) + if err != nil { + return fmt.Errorf("failed to decode multibase: %w", err) + } + reader := bytes.NewReader(data) + return resp.Emit(reader) + }, +} + +var mbaseTranscodeCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Transcode multibase string between bases", + LongDescription: ` +This command expects multibase inside of a file or via stdin. + +By default it will use URL-safe base64url encoding, +but one can customize used base with -b: + + > echo -n hello | ipfs multibase encode > file + > cat file + uaGVsbG8 + + > ipfs multibase transcode file -b base16 > transcoded_file + > cat transcoded_file + f68656c6c6f +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("encoded_file", true, false, "encoded data to decode").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(mbaseOptionName, "multibase encoding").WithDefault("base64url"), + }, + Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { + if err := req.ParseBodyArgs(); err != nil { + return err + } + encoderName, _ := req.Options[mbaseOptionName].(string) + encoder, err := mbase.EncoderByName(encoderName) + if err != nil { + return err + } + files := req.Files.Entries() + file, err := cmdenv.GetFileArg(files) + if err != nil { + return fmt.Errorf("failed to access file: %w", err) + } + encoded_data, err := ioutil.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file contents: %w", err) + } + _, data, err := mbase.Decode(string(encoded_data)) + if err != nil { + return fmt.Errorf("failed to decode multibase: %w", err) + } + encoded := encoder.Encode(data) + reader := strings.NewReader(encoded) + return resp.Emit(reader) + }, +} diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index cd5e0c4288c..6548f29c350 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -167,13 +167,19 @@ NOTE: a comma-separated notation is supported in CLI for convenience: } // Prepare Pin.origins - // Add own multiaddrs to the 'origins' array, so Pinning Service can - // use that as a hint and connect back to us (if possible) + // If CID in blockstore, add own multiaddrs to the 'origins' array + // so pinning service can use that as a hint and connect back to us. node, err := cmdenv.GetNode(env) if err != nil { return err } - if node.PeerHost != nil { + + isInBlockstore, err := node.Blockstore.Has(rp.Cid()) + if err != nil { + return err + } + + if isInBlockstore && node.PeerHost != nil { addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost)) if err != nil { return err diff --git a/core/commands/profile.go b/core/commands/profile.go new file mode 100644 index 00000000000..46c3ee19b8c --- /dev/null +++ b/core/commands/profile.go @@ -0,0 +1,237 @@ +package commands + +import ( + "archive/zip" + "context" + "encoding/json" + "fmt" + "io" + "os" + "runtime" + "runtime/pprof" + "strings" + "time" + + cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/go-ipfs/core/commands/e" +) + +// time format that works in filenames on windows. +var timeFormat = strings.ReplaceAll(time.RFC3339, ":", "_") + +type profileResult struct { + File string +} + +const cpuProfileTimeOption = "cpu-profile-time" + +var sysProfileCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Collect a performance profile for debugging.", + ShortDescription: ` +Collects cpu, heap, and goroutine profiles from a running go-ipfs daemon +into a single zip file. To aid in debugging, this command also attempts to +include a copy of the running go-ipfs binary. +`, + LongDescription: ` +Collects cpu, heap, and goroutine profiles from a running go-ipfs daemon +into a single zipfile. To aid in debugging, this command also attempts to +include a copy of the running go-ipfs binary. + +Profile's can be examined using 'go tool pprof', some tips can be found at +https://github.com/ipfs/go-ipfs/blob/master/docs/debug-guide.md. + +Privacy Notice: + +The output file includes: + +- A list of running goroutines. +- A CPU profile. +- A heap profile. +- Your copy of go-ipfs. +- The output of 'ipfs version --all'. + +It does not include: + +- Any of your IPFS data or metadata. +- Your config or private key. +- Your IP address. +- The contents of your computer's memory, filesystem, etc. + +However, it could reveal: + +- Your build path, if you built go-ipfs yourself. +- If and how a command/feature is being used (inferred from running functions). +- Memory offsets of various data structures. +- Any modifications you've made to go-ipfs. +`, + }, + NoLocal: true, + Options: []cmds.Option{ + cmds.StringOption(outputOptionName, "o", "The path where the output should be stored."), + cmds.StringOption(cpuProfileTimeOption, "The amount of time spent profiling CPU usage.").WithDefault("30s"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cpuProfileTimeStr, _ := req.Options[cpuProfileTimeOption].(string) + cpuProfileTime, err := time.ParseDuration(cpuProfileTimeStr) + if err != nil { + return fmt.Errorf("failed to parse CPU profile duration %q: %w", cpuProfileTimeStr, err) + } + + r, w := io.Pipe() + go func() { + _ = w.CloseWithError(writeProfiles(req.Context, cpuProfileTime, w)) + }() + return res.Emit(r) + }, + PostRun: cmds.PostRunMap{ + cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + v, err := res.Next() + if err != nil { + return err + } + + outReader, ok := v.(io.Reader) + if !ok { + return e.New(e.TypeErr(outReader, v)) + } + + outPath, _ := res.Request().Options[outputOptionName].(string) + if outPath == "" { + outPath = "ipfs-profile-" + time.Now().Format(timeFormat) + ".zip" + } + fi, err := os.Create(outPath) + if err != nil { + return err + } + defer fi.Close() + + _, err = io.Copy(fi, outReader) + if err != nil { + return err + } + return re.Emit(&profileResult{File: outPath}) + }, + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *profileResult) error { + fmt.Fprintf(w, "Wrote profiles to: %s\n", out.File) + return nil + }), + }, +} + +func writeProfiles(ctx context.Context, cpuProfileTime time.Duration, w io.Writer) error { + archive := zip.NewWriter(w) + + // Take some profiles. + type profile struct { + name string + file string + debug int + } + + profiles := []profile{{ + name: "goroutine", + file: "goroutines.stacks", + debug: 2, + }, { + name: "goroutine", + file: "goroutines.pprof", + }, { + name: "heap", + file: "heap.pprof", + }} + + for _, profile := range profiles { + prof := pprof.Lookup(profile.name) + out, err := archive.Create(profile.file) + if err != nil { + return err + } + err = prof.WriteTo(out, profile.debug) + if err != nil { + return err + } + } + + // Take a CPU profile. + if cpuProfileTime != 0 { + out, err := archive.Create("cpu.pprof") + if err != nil { + return err + } + + err = writeCPUProfile(ctx, cpuProfileTime, out) + if err != nil { + return err + } + } + + // Collect version info + // I'd use diag sysinfo, but that includes some more sensitive information + // (GOPATH, etc.). + { + out, err := archive.Create("version.json") + if err != nil { + return err + } + + err = json.NewEncoder(out).Encode(getVersionInfo()) + if err != nil { + return err + } + } + + // Collect binary + if fi, err := openIPFSBinary(); err == nil { + fname := "ipfs" + if runtime.GOOS == "windows" { + fname += ".exe" + } + + out, err := archive.Create(fname) + if err != nil { + return err + } + + _, err = io.Copy(out, fi) + _ = fi.Close() + if err != nil { + return err + } + } + return archive.Close() +} + +func writeCPUProfile(ctx context.Context, d time.Duration, w io.Writer) error { + if err := pprof.StartCPUProfile(w); err != nil { + return err + } + defer pprof.StopCPUProfile() + + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-timer.C: + case <-ctx.Done(): + return ctx.Err() + } + return nil +} + +func openIPFSBinary() (*os.File, error) { + if runtime.GOOS == "linux" { + pid := os.Getpid() + fi, err := os.Open(fmt.Sprintf("/proc/%d/exe", pid)) + if err == nil { + return fi, nil + } + } + path, err := os.Executable() + if err != nil { + return nil, err + } + return os.Open(path) +} diff --git a/core/commands/root.go b/core/commands/root.go index fbe2006e542..64b9b42fbf0 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -43,7 +43,10 @@ DATA STRUCTURE COMMANDS dag Interact with IPLD DAG nodes files Interact with files as if they were a unix filesystem block Interact with raw blocks in the datastore + +TEXT ENCODING COMMANDS cid Convert and discover properties of CIDs + multibase Encode and decode data with Multibase format ADVANCED COMMANDS daemon Start a long-running daemon process @@ -151,6 +154,7 @@ var rootSubcommands = map[string]*cmds.Command{ "version": VersionCmd, "shutdown": daemonShutdownCmd, "cid": CidCmd, + "multibase": MbaseCmd, } // RootRO is the readonly version of Root diff --git a/core/commands/swarm.go b/core/commands/swarm.go index b82a95630dc..7cde3f38d43 100644 --- a/core/commands/swarm.go +++ b/core/commands/swarm.go @@ -51,6 +51,7 @@ ipfs peers in the internet. "disconnect": swarmDisconnectCmd, "filters": swarmFiltersCmd, "peers": swarmPeersCmd, + "peering": swarmPeeringCmd, }, } @@ -61,6 +62,149 @@ const ( swarmDirectionOptionName = "direction" ) +type peeringResult struct { + ID peer.ID + Status string +} + +var swarmPeeringCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Modify the peering subsystem.", + ShortDescription: ` +'ipfs swarm peering' manages the peering subsystem. +Peers in the peering subsystem is maintained to be connected, reconnected +on disconnect with a back-off. +The changes are not saved to the config. +`, + }, + Subcommands: map[string]*cmds.Command{ + "add": swarmPeeringAddCmd, + "ls": swarmPeeringLsCmd, + "rm": swarmPeeringRmCmd, + }, +} + +var swarmPeeringAddCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Add peers into the peering subsystem.", + ShortDescription: ` +'ipfs swarm peering add' will add the new address to the peering subsystem as one that should always be connected to. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("address", true, true, "address of peer to add into the peering subsystem"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + addrs := make([]ma.Multiaddr, len(req.Arguments)) + + for i, arg := range req.Arguments { + addr, err := ma.NewMultiaddr(arg) + if err != nil { + return err + } + + addrs[i] = addr + } + + addInfos, err := peer.AddrInfosFromP2pAddrs(addrs...) + if err != nil { + return err + } + + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + for _, addrinfo := range addInfos { + node.Peering.AddPeer(addrinfo) + err = res.Emit(peeringResult{addrinfo.ID, "success"}) + if err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, pr *peeringResult) error { + fmt.Fprintf(w, "add %s %s\n", pr.ID.String(), pr.Status) + return nil + }), + }, + Type: peeringResult{}, +} + +var swarmPeeringLsCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List peers registered in the peering subsystem.", + ShortDescription: ` +'ipfs swarm peering ls' lists the peers that are registered in the peering subsystem and to which the daemon is always connected. +`, + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + peers := node.Peering.ListPeers() + return cmds.EmitOnce(res, addrInfos{Peers: peers}) + }, + Type: addrInfos{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ai *addrInfos) error { + for _, info := range ai.Peers { + fmt.Fprintf(w, "%s\n", info.ID) + for _, addr := range info.Addrs { + fmt.Fprintf(w, "\t%s\n", addr) + } + } + return nil + }), + }, +} + +type addrInfos struct { + Peers []peer.AddrInfo +} + +var swarmPeeringRmCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove a peer from the peering subsystem.", + ShortDescription: ` +'ipfs swarm peering rm' will remove the given ID from the peering subsystem and remove it from the always-on connection. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ID", true, true, "ID of peer to remove from the peering subsystem"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + for _, arg := range req.Arguments { + id, err := peer.Decode(arg) + if err != nil { + return err + } + + node.Peering.RemovePeer(id) + if err = res.Emit(peeringResult{id, "success"}); err != nil { + return err + } + } + return nil + }, + Type: peeringResult{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, pr *peeringResult) error { + fmt.Fprintf(w, "add %s %s\n", pr.ID.String(), pr.Status) + return nil + }), + }, +} + var swarmPeersCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List peers with open connections.", diff --git a/core/commands/sysdiag.go b/core/commands/sysdiag.go index 050b61c1373..9ab2578b25f 100644 --- a/core/commands/sysdiag.go +++ b/core/commands/sysdiag.go @@ -6,6 +6,7 @@ import ( "runtime" version "github.com/ipfs/go-ipfs" + "github.com/ipfs/go-ipfs/core" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" cmds "github.com/ipfs/go-ipfs-cmds" @@ -21,40 +22,49 @@ Prints out information about your computer to aid in easier debugging. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - info := make(map[string]interface{}) - err := runtimeInfo(info) + nd, err := cmdenv.GetNode(env) if err != nil { return err } - err = envVarInfo(info) + info, err := getInfo(nd) if err != nil { return err } + return cmds.EmitOnce(res, info) + }, +} - err = diskSpaceInfo(info) - if err != nil { - return err - } +func getInfo(nd *core.IpfsNode) (map[string]interface{}, error) { + info := make(map[string]interface{}) + err := runtimeInfo(info) + if err != nil { + return nil, err + } - err = memInfo(info) - if err != nil { - return err - } - nd, err := cmdenv.GetNode(env) - if err != nil { - return err - } + err = envVarInfo(info) + if err != nil { + return nil, err + } - err = netInfo(nd.IsOnline, info) - if err != nil { - return err - } + err = diskSpaceInfo(info) + if err != nil { + return nil, err + } - info["ipfs_version"] = version.CurrentVersionNumber - info["ipfs_commit"] = version.CurrentCommit - return cmds.EmitOnce(res, info) - }, + err = memInfo(info) + if err != nil { + return nil, err + } + + err = netInfo(nd.IsOnline, info) + if err != nil { + return nil, err + } + + info["ipfs_version"] = version.CurrentVersionNumber + info["ipfs_commit"] = version.CurrentCommit + return info, nil } func runtimeInfo(out map[string]interface{}) error { diff --git a/core/commands/version.go b/core/commands/version.go index a3693545281..ecdda594b2d 100644 --- a/core/commands/version.go +++ b/core/commands/version.go @@ -28,6 +28,16 @@ const ( versionAllOptionName = "all" ) +func getVersionInfo() *VersionOutput { + return &VersionOutput{ + Version: version.CurrentVersionNumber, + Commit: version.CurrentCommit, + Repo: fmt.Sprint(fsrepo.RepoVersion), + System: runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here + Golang: runtime.Version(), + } +} + var VersionCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show IPFS version information.", @@ -46,13 +56,7 @@ var VersionCmd = &cmds.Command{ // must be permitted to run before init Extra: CreateCmdExtras(SetDoesNotUseRepo(true), SetDoesNotUseConfigAsInput(true)), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - return cmds.EmitOnce(res, &VersionOutput{ - Version: version.CurrentVersionNumber, - Commit: version.CurrentCommit, - Repo: fmt.Sprint(fsrepo.RepoVersion), - System: runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here - Golang: runtime.Version(), - }) + return cmds.EmitOnce(res, getVersionInfo()) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, version *VersionOutput) error { diff --git a/core/core.go b/core/core.go index 49f1185646b..888d3d78013 100644 --- a/core/core.go +++ b/core/core.go @@ -17,6 +17,7 @@ import ( "github.com/ipfs/go-ipfs-pinner" bserv "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-fetcher" "github.com/ipfs/go-graphsync" bstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" @@ -24,7 +25,6 @@ import ( ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" mfs "github.com/ipfs/go-mfs" - resolver "github.com/ipfs/go-path/resolver" goprocess "github.com/jbenet/goprocess" connmgr "github.com/libp2p/go-libp2p-core/connmgr" ic "github.com/libp2p/go-libp2p-core/crypto" @@ -37,7 +37,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" psrouter "github.com/libp2p/go-libp2p-pubsub-router" record "github.com/libp2p/go-libp2p-record" - "github.com/libp2p/go-libp2p/p2p/discovery" + "github.com/libp2p/go-libp2p/p2p/discovery/mdns" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" @@ -70,22 +70,23 @@ type IpfsNode struct { PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network // Services - Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances - Blockstore bstore.GCBlockstore // the block store (lower level) - Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore - BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping - GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc - Blocks bserv.BlockService // the block service, get/add blocks. - DAG ipld.DAGService // the merkle dag service, get/add objects. - Resolver *resolver.Resolver // the path resolution system - Reporter *metrics.BandwidthCounter `optional:"true"` - Discovery discovery.Service `optional:"true"` - FilesRoot *mfs.Root - RecordValidator record.Validator + Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances + Blockstore bstore.GCBlockstore // the block store (lower level) + Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore + BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping + GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc + Blocks bserv.BlockService // the block service, get/add blocks. + DAG ipld.DAGService // the merkle dag service, get/add objects. + IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model + UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data + Reporter *metrics.BandwidthCounter `optional:"true"` + Discovery mdns.Service `optional:"true"` + FilesRoot *mfs.Root + RecordValidator record.Validator // Online PeerHost p2phost.Host `optional:"true"` // the network host (server+client) - Peering peering.PeeringService `optional:"true"` + Peering *peering.PeeringService `optional:"true"` Filters *ma.Filters `optional:"true"` Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper Routing routing.Routing `optional:"true"` // the routing system. recommend ipfs-dht diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index f8c165a0cd3..3d31abaabc2 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -19,11 +19,12 @@ import ( "fmt" bserv "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-ipfs-blockstore" - "github.com/ipfs/go-ipfs-exchange-interface" + "github.com/ipfs/go-fetcher" + blockstore "github.com/ipfs/go-ipfs-blockstore" + exchange "github.com/ipfs/go-ipfs-exchange-interface" offlinexch "github.com/ipfs/go-ipfs-exchange-offline" - "github.com/ipfs/go-ipfs-pinner" - "github.com/ipfs/go-ipfs-provider" + pin "github.com/ipfs/go-ipfs-pinner" + provider "github.com/ipfs/go-ipfs-provider" offlineroute "github.com/ipfs/go-ipfs-routing/offline" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" @@ -55,13 +56,14 @@ type CoreAPI struct { baseBlocks blockstore.Blockstore pinning pin.Pinner - blocks bserv.BlockService - dag ipld.DAGService - - peerstore pstore.Peerstore - peerHost p2phost.Host - recordValidator record.Validator - exchange exchange.Interface + blocks bserv.BlockService + dag ipld.DAGService + ipldFetcherFactory fetcher.Factory + unixFSFetcherFactory fetcher.Factory + peerstore pstore.Peerstore + peerHost p2phost.Host + recordValidator record.Validator + exchange exchange.Interface namesys namesys.NameSystem routing routing.Routing @@ -167,8 +169,10 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e baseBlocks: n.BaseBlocks, pinning: n.Pinning, - blocks: n.Blocks, - dag: n.DAG, + blocks: n.Blocks, + dag: n.DAG, + ipldFetcherFactory: n.IPLDFetcherFactory, + unixFSFetcherFactory: n.UnixFSFetcherFactory, peerstore: n.Peerstore, peerHost: n.PeerHost, diff --git a/core/coreapi/path.go b/core/coreapi/path.go index 598f5d11581..b9bf83e0df6 100644 --- a/core/coreapi/path.go +++ b/core/coreapi/path.go @@ -8,10 +8,10 @@ import ( "github.com/ipfs/go-namesys/resolve" "github.com/ipfs/go-cid" + "github.com/ipfs/go-fetcher" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - uio "github.com/ipfs/go-unixfs/io" + ipfspathresolver "github.com/ipfs/go-path/resolver" coreiface "github.com/ipfs/interface-go-ipfs-core" path "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -49,23 +49,19 @@ func (api *CoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.Resolved return nil, err } - var resolveOnce resolver.ResolveOnce - - switch ipath.Segments()[0] { - case "ipfs": - resolveOnce = uio.ResolveUnixfsOnce - case "ipld": - resolveOnce = resolver.ResolveSingle - default: + if ipath.Segments()[0] != "ipfs" && ipath.Segments()[0] != "ipld" { return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) } - r := &resolver.Resolver{ - DAG: api.dag, - ResolveOnce: resolveOnce, + var dataFetcher fetcher.Factory + if ipath.Segments()[0] == "ipld" { + dataFetcher = api.ipldFetcherFactory + } else { + dataFetcher = api.unixFSFetcherFactory } + resolver := ipfspathresolver.NewBasicResolver(dataFetcher) - node, rest, err := r.ResolveToLastNode(ctx, ipath) + node, rest, err := resolver.ResolveToLastNode(ctx, ipath) if err != nil { return nil, err } diff --git a/core/coreapi/swarm.go b/core/coreapi/swarm.go index 641b4c526f2..3c3c40ddbdc 100644 --- a/core/coreapi/swarm.go +++ b/core/coreapi/swarm.go @@ -116,18 +116,16 @@ func (api *SwarmAPI) Peers(context.Context) ([]coreiface.ConnectionInfo, error) conns := api.peerHost.Network().Conns() - var out []coreiface.ConnectionInfo + out := make([]coreiface.ConnectionInfo, 0, len(conns)) for _, c := range conns { - pid := c.RemotePeer() - addr := c.RemoteMultiaddr() ci := &connInfo{ peerstore: api.peerstore, conn: c, dir: c.Stat().Direction, - addr: addr, - peer: pid, + addr: c.RemoteMultiaddr(), + peer: c.RemotePeer(), } /* diff --git a/core/coreapi/test/api_test.go b/core/coreapi/test/api_test.go index e5ccb62dd6f..54f3b48ca43 100644 --- a/core/coreapi/test/api_test.go +++ b/core/coreapi/test/api_test.go @@ -22,7 +22,7 @@ import ( "github.com/ipfs/go-ipfs-config" coreiface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" - ci "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/crypto" peer "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p/p2p/net/mock" ) @@ -40,7 +40,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) for i := 0; i < n; i++ { var ident config.Identity if fullIdentity { - sk, pk, err := ci.GenerateKeyPair(ci.RSA, 2048) + sk, pk, err := crypto.GenerateKeyPair(crypto.RSA, 2048) if err != nil { return nil, err } @@ -50,7 +50,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - kbytes, err := sk.Bytes() + kbytes, err := crypto.MarshalPrivateKey(sk) if err != nil { return nil, err } diff --git a/core/coreapi/test/path_test.go b/core/coreapi/test/path_test.go new file mode 100644 index 00000000000..cc7ba2efaef --- /dev/null +++ b/core/coreapi/test/path_test.go @@ -0,0 +1,79 @@ +package test + +import ( + "context" + "strconv" + "testing" + "time" + + files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-merkledag" + uio "github.com/ipfs/go-unixfs/io" + "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-ipld-prime" +) + + +func TestPathUnixFSHAMTPartial(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create a node + apis, err := NodeProvider{}.MakeAPISwarm(ctx, true, 1) + if err != nil { + t.Fatal(err) + } + a := apis[0] + + // Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config + prevVal := uio.UseHAMTSharding + uio.UseHAMTSharding = true + defer func() { + uio.UseHAMTSharding = prevVal + }() + + // Create and add a sharded directory + dir := make(map[string]files.Node) + // Make sure we have at least two levels of sharding + for i := 0; i < uio.DefaultShardWidth + 1; i++ { + dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i))) + } + + r, err := a.Unixfs().Add(ctx, files.NewMapDirectory(dir), options.Unixfs.Pin(false)) + if err != nil { + t.Fatal(err) + } + + // Get the root of the directory + nd, err := a.Dag().Get(ctx, r.Cid()) + if err != nil { + t.Fatal(err) + } + + // Make sure the root is a DagPB node (this API might change in the future to account for ADLs) + _ = nd.(ipld.Node) + pbNode := nd.(*merkledag.ProtoNode) + + // Remove one of the sharded directory blocks + if err := a.Block().Rm(ctx, path.IpfsPath(pbNode.Links()[0].Cid)); err != nil { + t.Fatal(err) + } + + // Try and resolve each of the entries in the sharded directory which will result in pathing over the missing block + // + // Note: we could just check a particular path here, but it would require either greater use of the HAMT internals + // or some hard coded values in the test both of which would be a pain to follow. + for k := range dir { + // The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring + // because we exceeded the timeout on our query + timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second * 1) + _, err := a.ResolveNode(timeoutCtx, path.Join(r, k)) + if err != nil { + if timeoutCtx.Err() == nil { + t.Fatal(err) + } + } + timeoutCancel() + } +} diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index f400c515b97..fb1524da529 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -104,7 +104,7 @@ func VersionOption() ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Commit: %s\n", version.CurrentCommit) - fmt.Fprintf(w, "Client Version: %s\n", version.UserAgent) + fmt.Fprintf(w, "Client Version: %s\n", version.GetUserAgentVersion()) fmt.Fprintf(w, "Protocol Version: %s\n", id.LibP2PVersion) }) return mux, nil diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index ccec95b0151..f5ee54d8cfd 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -28,6 +28,7 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p-core/routing" + prometheus "github.com/prometheus/client_golang/prometheus" ) const ( @@ -62,6 +63,8 @@ type redirectTemplateData struct { type gatewayHandler struct { config GatewayConfig api coreiface.CoreAPI + + unixfsGetMetric *prometheus.SummaryVec } // StatusResponseWriter enables us to override HTTP Status Code passed to @@ -84,9 +87,27 @@ func (sw *statusResponseWriter) WriteHeader(code int) { } func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler { + unixfsGetMetric := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "ipfs", + Subsystem: "http", + Name: "unixfs_get_latency_seconds", + Help: "The time till the first block is received when 'getting' a file from the gateway.", + }, + []string{"gateway"}, + ) + if err := prometheus.Register(unixfsGetMetric); err != nil { + if are, ok := err.(prometheus.AlreadyRegisteredError); ok { + unixfsGetMetric = are.ExistingCollector.(*prometheus.SummaryVec) + } else { + log.Errorf("failed to register unixfsGetMetric: %v", err) + } + } + i := &gatewayHandler{ - config: c, - api: api, + config: c, + api: api, + unixfsGetMetric: unixfsGetMetric, } return i } @@ -271,7 +292,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return } - unixfsGetMetric.WithLabelValues(parsedPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsGetMetric.WithLabelValues(parsedPath.Namespace()).Observe(time.Since(begin).Seconds()) defer dr.Close() @@ -391,11 +412,12 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request size = humanize.Bytes(uint64(s)) } - hash := "" - if r, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name())); err == nil { - // Path may not be resolved. Continue anyways. - hash = r.Cid().String() + resolved, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name())) + if err != nil { + internalWebError(w, err) + return } + hash := resolved.Cid().String() // See comment above where originalUrlPath is declared. di := directoryItem{ diff --git a/core/corehttp/gateway_indexPage.go b/core/corehttp/gateway_indexPage.go index 6e172211638..3bee4822bb4 100644 --- a/core/corehttp/gateway_indexPage.go +++ b/core/corehttp/gateway_indexPage.go @@ -75,6 +75,9 @@ func breadcrumbs(urlPath string, dnslinkOrigin bool) []breadcrumb { } func shortHash(hash string) string { + if len(hash) <= 8 { + return hash + } return (hash[0:4] + "\u2026" + hash[len(hash)-4:]) } diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 053c22f9a8d..48c604e4a77 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -732,7 +732,7 @@ func TestVersion(t *testing.T) { t.Fatalf("response doesn't contain commit:\n%s", s) } - if !strings.Contains(s, "Client Version: "+version.UserAgent) { + if !strings.Contains(s, "Client Version: "+version.GetUserAgentVersion()) { t.Fatalf("response doesn't contain client version:\n%s", s) } diff --git a/core/corehttp/metrics.go b/core/corehttp/metrics.go index f34881f41a5..e7f36113bda 100644 --- a/core/corehttp/metrics.go +++ b/core/corehttp/metrics.go @@ -14,7 +14,7 @@ import ( promhttp "github.com/prometheus/client_golang/prometheus/promhttp" ) -// This adds the scraping endpoint which Prometheus uses to fetch metrics. +// MetricsScrapingOption adds the scraping endpoint which Prometheus uses to fetch metrics. func MetricsScrapingOption(path string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.Handle(path, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) @@ -51,7 +51,7 @@ func MetricsOpenCensusCollectionOption() ServeOption { } } -// This adds collection of net/http-related metrics +// MetricsCollectionOption adds collection of net/http-related metrics. func MetricsCollectionOption(handlerName string) ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { // Adapted from github.com/prometheus/client_golang/prometheus/http.go @@ -130,14 +130,10 @@ func MetricsCollectionOption(handlerName string) ServeOption { var ( peersTotalMetric = prometheus.NewDesc( prometheus.BuildFQName("ipfs", "p2p", "peers_total"), - "Number of connected peers", []string{"transport"}, nil) - - unixfsGetMetric = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "ipfs", - Subsystem: "http", - Name: "unixfs_get_latency_seconds", - Help: "The time till the first block is received when 'getting' a file from the gateway.", - }, []string{"namespace"}) + "Number of connected peers", + []string{"transport"}, + nil, + ) ) type IpfsNodeCollector struct { diff --git a/core/corehttp/metrics_test.go b/core/corehttp/metrics_test.go index 847d681a58c..f9fa10f679c 100644 --- a/core/corehttp/metrics_test.go +++ b/core/corehttp/metrics_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - core "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core" inet "github.com/libp2p/go-libp2p-core/network" swarmt "github.com/libp2p/go-libp2p-swarm/testing" @@ -20,7 +20,11 @@ func TestPeersTotal(t *testing.T) { hosts := make([]*bhost.BasicHost, 4) for i := 0; i < 4; i++ { - hosts[i] = bhost.New(swarmt.GenSwarm(t, ctx)) + var err error + hosts[i], err = bhost.NewHost(ctx, swarmt.GenSwarm(t, ctx), nil) + if err != nil { + t.Fatal(err) + } } dial := func(a, b inet.Network) { diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index 298163e3a6f..72656751a95 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeiflkjt66aetfgcrgvv75izymd5kc47g6luepqmfq6zsf5w6ueth6y" // v2.12.4 +const WebUIPath = "/ipfs/bafybeihcyruaeza7uyjd6ugicbcrqumejf6uf353e5etdkhotqffwtguva" // v2.13.0 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeiflkjt66aetfgcrgvv75izymd5kc47g6luepqmfq6zsf5w6ueth6y", "/ipfs/bafybeid26vjplsejg7t3nrh7mxmiaaxriebbm4xxrxxdunlk7o337m5sqq", "/ipfs/bafybeif4zkmu7qdhkpf3pnhwxipylqleof7rl6ojbe7mq3fzogz6m4xk3i", "/ipfs/bafybeianwe4vy7sprht5sm3hshvxjeqhwcmvbzq73u55sdhqngmohkjgs4", diff --git a/core/node/bitswap.go b/core/node/bitswap.go new file mode 100644 index 00000000000..a2548ab3ce6 --- /dev/null +++ b/core/node/bitswap.go @@ -0,0 +1,52 @@ +package node + +import ( + "context" + + "github.com/ipfs/go-bitswap" + "github.com/ipfs/go-bitswap/network" + blockstore "github.com/ipfs/go-ipfs-blockstore" + config "github.com/ipfs/go-ipfs-config" + exchange "github.com/ipfs/go-ipfs-exchange-interface" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" + "go.uber.org/fx" + + "github.com/ipfs/go-ipfs/core/node/helpers" +) + +const ( + // Docs: https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#internalbitswap + DefaultEngineBlockstoreWorkerCount = 128 + DefaultTaskWorkerCount = 8 + DefaultEngineTaskWorkerCount = 8 + DefaultMaxOutstandingBytesPerPeer = 1 << 20 +) + +// OnlineExchange creates new LibP2P backed block exchange (BitSwap) +func OnlineExchange(cfg *config.Config, provide bool) interface{} { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt routing.Routing, bs blockstore.GCBlockstore) exchange.Interface { + bitswapNetwork := network.NewFromIpfsHost(host, rt) + + var internalBsCfg config.InternalBitswap + if cfg.Internal.Bitswap != nil { + internalBsCfg = *cfg.Internal.Bitswap + } + + opts := []bitswap.Option{ + bitswap.ProvideEnabled(provide), + bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))), + bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))), + bitswap.EngineTaskWorkerCount(int(internalBsCfg.EngineTaskWorkerCount.WithDefault(DefaultEngineTaskWorkerCount))), + bitswap.MaxOutstandingBytesPerPeer(int(internalBsCfg.MaxOutstandingBytesPerPeer.WithDefault(DefaultMaxOutstandingBytesPerPeer))), + } + exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs, opts...) + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return exch.Close() + }, + }) + return exch + + } +} diff --git a/core/node/builder.go b/core/node/builder.go index 22ab500e150..803caf51885 100644 --- a/core/node/builder.go +++ b/core/node/builder.go @@ -15,7 +15,7 @@ import ( ds "github.com/ipfs/go-datastore" dsync "github.com/ipfs/go-datastore/sync" cfg "github.com/ipfs/go-ipfs-config" - ci "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/crypto" peer "github.com/libp2p/go-libp2p-core/peer" ) @@ -124,7 +124,7 @@ func (cfg *BuildCfg) options(ctx context.Context) (fx.Option, *cfg.Config) { func defaultRepo(dstore repo.Datastore) (repo.Repo, error) { c := cfg.Config{} - priv, pub, err := ci.GenerateKeyPairWithReader(ci.RSA, 2048, rand.Reader) + priv, pub, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, rand.Reader) if err != nil { return nil, err } @@ -134,7 +134,7 @@ func defaultRepo(dstore repo.Datastore) (repo.Repo, error) { return nil, err } - privkeyb, err := priv.Bytes() + privkeyb, err := crypto.MarshalPrivateKey(priv) if err != nil { return nil, err } diff --git a/core/node/core.go b/core/node/core.go index cb757d81968..cdb790bd22f 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -3,24 +3,26 @@ package node import ( "context" "fmt" - "time" - "github.com/ipfs/go-bitswap" - "github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-fetcher" + bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" "github.com/ipfs/go-filestore" - "github.com/ipfs/go-ipfs-blockstore" - "github.com/ipfs/go-ipfs-exchange-interface" - "github.com/ipfs/go-ipfs-pinner" + blockstore "github.com/ipfs/go-ipfs-blockstore" + exchange "github.com/ipfs/go-ipfs-exchange-interface" + pin "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipfs-pinner/dspinner" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" "github.com/ipfs/go-mfs" "github.com/ipfs/go-unixfs" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/ipfs/go-unixfsnode" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/schema" "go.uber.org/fx" "github.com/ipfs/go-ipfs/core/node/helpers" @@ -52,8 +54,7 @@ func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo) } syncDs := &syncDagService{ds, syncFn} - ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Minute) - defer cancel() + ctx := context.TODO() pinning, err := dspinner.New(ctx, rootDS, syncDs) if err != nil { @@ -82,24 +83,29 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter { return merkledag.NewSession(ctx, s.DAGService) } -// Dag creates new DAGService -func Dag(bs blockservice.BlockService) format.DAGService { - return merkledag.NewDAGService(bs) +type fetchersOut struct { + fx.Out + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` } -// OnlineExchange creates new LibP2P backed block exchange (BitSwap) -func OnlineExchange(provide bool) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt routing.Routing, bs blockstore.GCBlockstore) exchange.Interface { - bitswapNetwork := network.NewFromIpfsHost(host, rt) - exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs, bitswap.ProvideEnabled(provide)) - lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { - return exch.Close() - }, - }) - return exch +// FetcherConfig returns a fetcher config that can build new fetcher instances +func FetcherConfig(bs blockservice.BlockService) fetchersOut { + ipldFetcher := bsfetcher.NewFetcherConfig(bs) + ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) - } + unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify) + return fetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher} +} + +// Dag creates new DAGService +func Dag(bs blockservice.BlockService) format.DAGService { + return merkledag.NewDAGService(bs) } // Files loads persisted MFS root diff --git a/core/node/groups.go b/core/node/groups.go index 8d37f84ce13..930a2151bef 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -18,7 +18,6 @@ import ( offline "github.com/ipfs/go-ipfs-exchange-offline" offroute "github.com/ipfs/go-ipfs-routing/offline" - "github.com/ipfs/go-path/resolver" uio "github.com/ipfs/go-unixfs/io" "go.uber.org/fx" ) @@ -263,7 +262,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config) fx.Option { shouldBitswapProvide := !cfg.Experimental.StrategicProviding return fx.Options( - fx.Provide(OnlineExchange(shouldBitswapProvide)), + fx.Provide(OnlineExchange(cfg, shouldBitswapProvide)), maybeProvide(Graphsync, cfg.Experimental.GraphsyncEnabled), fx.Provide(DNSResolver), fx.Provide(Namesys(ipnsCacheSize)), @@ -294,7 +293,7 @@ func Offline(cfg *config.Config) fx.Option { var Core = fx.Options( fx.Provide(BlockService), fx.Provide(Dag), - fx.Provide(resolver.NewBasicResolver), + fx.Provide(FetcherConfig), fx.Provide(Pinning), fx.Provide(Files), ) diff --git a/core/node/libp2p/addrs.go b/core/node/libp2p/addrs.go index f64947130d6..818385194a7 100644 --- a/core/node/libp2p/addrs.go +++ b/core/node/libp2p/addrs.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/libp2p/go-libp2p" - host "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/host" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" @@ -13,13 +13,13 @@ import ( func AddrFilters(filters []string) func() (*ma.Filters, Libp2pOpts, error) { return func() (filter *ma.Filters, opts Libp2pOpts, err error) { filter = ma.NewFilters() - opts.Opts = append(opts.Opts, libp2p.Filters(filter)) //nolint + opts.Opts = append(opts.Opts, libp2p.ConnectionGater((*filtersConnectionGater)(filter))) for _, s := range filters { f, err := mamask.NewMask(s) if err != nil { return filter, opts, fmt.Errorf("incorrectly formatted address filter in config: %s", s) } - opts.Opts = append(opts.Opts, libp2p.FilterAddresses(f)) //nolint + filter.AddFilter(*f, ma.ActionDeny) } return filter, opts, nil } diff --git a/core/node/libp2p/discovery.go b/core/node/libp2p/discovery.go index 9ffe46180ec..f29f523cee1 100644 --- a/core/node/libp2p/discovery.go +++ b/core/node/libp2p/discovery.go @@ -6,7 +6,9 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p/p2p/discovery" + "github.com/libp2p/go-libp2p/p2p/discovery/mdns" + legacymdns "github.com/libp2p/go-libp2p/p2p/discovery/mdns_legacy" + "go.uber.org/fx" "github.com/ipfs/go-ipfs/core/node/helpers" @@ -35,18 +37,21 @@ func DiscoveryHandler(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host) } } -func SetupDiscovery(mdns bool, mdnsInterval int) func(helpers.MetricsCtx, fx.Lifecycle, host.Host, *discoveryHandler) error { +func SetupDiscovery(useMdns bool, mdnsInterval int) func(helpers.MetricsCtx, fx.Lifecycle, host.Host, *discoveryHandler) error { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, handler *discoveryHandler) error { - if mdns { + if useMdns { + service := mdns.NewMdnsService(host, mdns.ServiceName) + service.RegisterNotifee(handler) + if mdnsInterval == 0 { mdnsInterval = 5 } - service, err := discovery.NewMdnsService(helpers.LifecycleCtx(mctx, lc), host, time.Duration(mdnsInterval)*time.Second, discovery.ServiceTag) + legacyService, err := legacymdns.NewMdnsService(mctx, host, time.Duration(mdnsInterval)*time.Second, legacymdns.ServiceTag) if err != nil { log.Error("mdns error: ", err) return nil } - service.RegisterNotifee(handler) + legacyService.RegisterNotifee(handler) } return nil } diff --git a/core/node/libp2p/filters.go b/core/node/libp2p/filters.go new file mode 100644 index 00000000000..c6b8175d8eb --- /dev/null +++ b/core/node/libp2p/filters.go @@ -0,0 +1,36 @@ +package libp2p + +import ( + "github.com/libp2p/go-libp2p-core/connmgr" + "github.com/libp2p/go-libp2p-core/control" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + + ma "github.com/multiformats/go-multiaddr" +) + +// filtersConnectionGater is an adapter that turns multiaddr.Filter into a +// connmgr.ConnectionGater. +type filtersConnectionGater ma.Filters + +var _ connmgr.ConnectionGater = (*filtersConnectionGater)(nil) + +func (f *filtersConnectionGater) InterceptAddrDial(_ peer.ID, addr ma.Multiaddr) (allow bool) { + return !(*ma.Filters)(f).AddrBlocked(addr) +} + +func (f *filtersConnectionGater) InterceptPeerDial(p peer.ID) (allow bool) { + return true +} + +func (f *filtersConnectionGater) InterceptAccept(connAddr network.ConnMultiaddrs) (allow bool) { + return !(*ma.Filters)(f).AddrBlocked(connAddr.RemoteMultiaddr()) +} + +func (f *filtersConnectionGater) InterceptSecured(_ network.Direction, _ peer.ID, connAddr network.ConnMultiaddrs) (allow bool) { + return !(*ma.Filters)(f).AddrBlocked(connAddr.RemoteMultiaddr()) +} + +func (f *filtersConnectionGater) InterceptUpgraded(_ network.Conn) (allow bool, reason control.DisconnectReason) { + return true, 0 +} diff --git a/core/node/libp2p/libp2p.go b/core/node/libp2p/libp2p.go index 51183c9543a..57539259baa 100644 --- a/core/node/libp2p/libp2p.go +++ b/core/node/libp2p/libp2p.go @@ -25,8 +25,7 @@ type Libp2pOpts struct { } // Misc options - -var UserAgent = simpleOpt(libp2p.UserAgent(version.UserAgent)) +var UserAgent = simpleOpt(libp2p.UserAgent(version.GetUserAgentVersion())) func ConnectionManager(low, high int, grace time.Duration) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { diff --git a/core/node/libp2p/transport.go b/core/node/libp2p/transport.go index 6140b739dce..c5112e9c0b8 100644 --- a/core/node/libp2p/transport.go +++ b/core/node/libp2p/transport.go @@ -9,18 +9,10 @@ import ( libp2pquic "github.com/libp2p/go-libp2p-quic-transport" tcp "github.com/libp2p/go-tcp-transport" websocket "github.com/libp2p/go-ws-transport" - quic "github.com/lucas-clemente/quic-go" "go.uber.org/fx" ) -// See https://github.com/ipfs/go-ipfs/issues/7526 and -// https://github.com/lucas-clemente/quic-go/releases/tag/v0.17.3. -// TODO: remove this once the network has upgraded to > v0.6.0. -func init() { - quic.RetireBugBackwardsCompatibilityMode = true -} - func Transports(tptConfig config.Transports) interface{} { return func(pnet struct { fx.In diff --git a/core/node/provider.go b/core/node/provider.go index e865d2b5fd9..3dd7f9b728a 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -5,12 +5,12 @@ import ( "fmt" "time" + "github.com/ipfs/go-fetcher" "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipfs-provider" "github.com/ipfs/go-ipfs-provider/batched" q "github.com/ipfs/go-ipfs-provider/queue" "github.com/ipfs/go-ipfs-provider/simple" - ipld "github.com/ipfs/go-ipld-format" "github.com/libp2p/go-libp2p-core/routing" "github.com/multiformats/go-multihash" "go.uber.org/fx" @@ -172,7 +172,12 @@ func SimpleProviders(reprovideStrategy string, reprovideInterval string) fx.Opti } func pinnedProviderStrategy(onlyRoots bool) interface{} { - return func(pinner pin.Pinner, dag ipld.DAGService) simple.KeyChanFunc { - return simple.NewPinnedProvider(onlyRoots, pinner, dag) + type input struct { + fx.In + Pinner pin.Pinner + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + } + return func(in input) simple.KeyChanFunc { + return simple.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher) } } diff --git a/docs/PATCH_RELEASE_TEMPLATE.md b/docs/PATCH_RELEASE_TEMPLATE.md new file mode 100644 index 00000000000..cd2a1819dba --- /dev/null +++ b/docs/PATCH_RELEASE_TEMPLATE.md @@ -0,0 +1,27 @@ +# Patch Release Checklist + +This process handles patch releases from version `vX.Y.Z` to `vX.Y.Z+1` assuming that `vX.Y.Z` is the latest released version of go-ipfs. + +- [ ] Fork a new branch (`release-vX.Y.Z`) from `release` and cherry-pick the relevant commits from master (or custom fixes) onto this branch +- [ ] Make a minimal changelog update tracking the relevant fixes to CHANGELOG. +- [ ] version string in `version.go` has been updated (in the `release-vX.Y.Z+1` branch). +- [ ] Make a PR merging `release-vX.Y.Z+1` into the release branch +- [ ] tag the merge commit in the `release` branch with `vX.Y.Z+1` +- [ ] upload to dist.ipfs.io + 1. Build: https://github.com/ipfs/distributions#usage. + 2. Pin the resulting release. + 3. Make a PR against ipfs/distributions with the updated versions, including the new hash in the PR comment. + 4. Ask the infra team to update the DNSLink record for dist.ipfs.io to point to the new distribution. +- [ ] cut a release on [github](https://github.com/ipfs/go-ipfs/releases) and upload the result of the ipfs/distributions build in the previous step. +- Announce the Release: + - [ ] On IRC/Matrix (both #ipfs and #ipfs-dev) + - [ ] On discuss.ipfs.io +- [ ] Release published + - [ ] to [dist.ipfs.io](https://dist.ipfs.io) + - [ ] to [npm-go-ipfs](https://github.com/ipfs/npm-go-ipfs) + - [ ] to [chocolatey](https://chocolatey.org/packages/ipfs) + - [ ] to [snap](https://snapcraft.io/ipfs) + - [ ] to [github](https://github.com/ipfs/go-ipfs/releases) + - [ ] to [arch](https://www.archlinux.org/packages/community/x86_64/go-ipfs/) (flag it out of date) +- [ ] Cut a new ipfs-desktop release +- [ ] Merge the `release` branch back into `master`, ignoring the changes to `version.go` (keep the `-dev` version from master). diff --git a/docs/RELEASE_ISSUE_TEMPLATE.md b/docs/RELEASE_ISSUE_TEMPLATE.md index 7c34627c749..9b2d493172d 100644 --- a/docs/RELEASE_ISSUE_TEMPLATE.md +++ b/docs/RELEASE_ISSUE_TEMPLATE.md @@ -62,7 +62,10 @@ Checklist: - [ ] Deploy new version to a subset of Preload nodes - [ ] Collect metrics every day. Work with the Infrastructure team to learn of any hiccup - [ ] IPFS Application Testing - Run the tests of the following applications: - - [ ] [IPFS Desktop](https://github.com/ipfs-shipyard/ipfs-desktop) - @lidel + - [ ] [IPFS Desktop](https://github.com/ipfs-shipyard/ipfs-desktop) + - [ ] Ensure the RC is published to [the NPM package](https://www.npmjs.com/package/go-ipfs?activeTab=versions) ([happens automatically, just wait for CI](https://github.com/ipfs/npm-go-ipfs/actions)) + - [ ] Upgrade to the RC in [ipfs-desktop](https://github.com/ipfs-shipyard/ipfs-desktop) and push to a branch ([example](https://github.com/ipfs/ipfs-desktop/pull/1826/commits/b0a23db31ce942b46d95965ee6fe770fb24d6bde)), and open a draft PR to track through the final release ([example](https://github.com/ipfs/ipfs-desktop/pull/1826)) + - [ ] Ensure CI tests pass, repeat for new RCs - [ ] [IPFS Companion](https://github.com/ipfs-shipyard/ipfs-companion) - @lidel - [ ] [NPM on IPFS](https://github.com/ipfs-shipyard/npm-on-ipfs) - @achingbrain - [ ] **Stage 2 - Community Dev Testing** @@ -106,6 +109,7 @@ Checklist: - [ ] Merge the `release` branch back into `master`, ignoring the changes to `version.go` (keep the `-dev` version from master). - [ ] Create an issue using this release issue template for the _next_ release. - [ ] Make sure any last-minute changelog updates from the blog post make it back into the CHANGELOG. + - [ ] Mark PR draft created for IPFS Desktop as ready for review. ## ❤️ Contributors diff --git a/docs/command-completion.md b/docs/command-completion.md index 73394d41a8a..5ca1edd168a 100644 --- a/docs/command-completion.md +++ b/docs/command-completion.md @@ -1,29 +1,13 @@ -Command Completion -================== +# Command Completion -Shell command completion is provided by the script at -[/misc/completion/ipfs-completion.bash](../misc/completion/ipfs-completion.bash). +Shell command completions can be generated by running one of the `ipfs commands completions` +sub-commands. +The simplest way to "eval" the completions logic: -Installation ------------- -The simplest way to see it working is to run -`source misc/completion/ipfs-completion.bash` straight from your shell. This -is only temporary and to fully enable it, you'll have to follow one of the steps -below. - -### Bash on Linux -For bash, completion can be enabled in a couple of ways. One is to copy the -completion script to the directory `~/.ipfs/` and then in the file -`~/.bash_completion` add ```bash -source ~/.ipfs/ipfs-completion.bash +> eval "$(ipfs commands completion bash)" ``` -It will automatically be loaded the next time bash is loaded. -To enable ipfs command completion globally on your system you may also -copy the completion script to `/etc/bash_completion.d/`. - -Additional References ---------------------- -* https://www.debian-administration.org/article/316/An_introduction_to_bash_completion_part_1 +To install the completions permanently, they can be moved to +`/etc/bash_completion.d` or sourced from your `~/.bashrc` file. diff --git a/docs/config.md b/docs/config.md index 87a2ea14c0a..ae39a3f88ca 100644 --- a/docs/config.md +++ b/docs/config.md @@ -5,10 +5,134 @@ is read once at node instantiation, either for an offline command, or when starting the daemon. Commands that execute on a running daemon do not read the config file at runtime. +## Table of Contents + +- [The go-ipfs config file](#the-go-ipfs-config-file) + - [Table of Contents](#table-of-contents) + - [Profiles](#profiles) + - [Types](#types) + - [`flag`](#flag) + - [`priority`](#priority) + - [`strings`](#strings) + - [`duration`](#duration) + - [`optionalInteger`](#optionalinteger) + - [`Addresses`](#addresses) + - [`Addresses.API`](#addressesapi) + - [`Addresses.Gateway`](#addressesgateway) + - [`Addresses.Swarm`](#addressesswarm) + - [`Addresses.Announce`](#addressesannounce) + - [`Addresses.NoAnnounce`](#addressesnoannounce) + - [`API`](#api) + - [`API.HTTPHeaders`](#apihttpheaders) + - [`AutoNAT`](#autonat) + - [`AutoNAT.ServiceMode`](#autonatservicemode) + - [`AutoNAT.Throttle`](#autonatthrottle) + - [`AutoNAT.Throttle.GlobalLimit`](#autonatthrottlegloballimit) + - [`AutoNAT.Throttle.PeerLimit`](#autonatthrottlepeerlimit) + - [`AutoNAT.Throttle.Interval`](#autonatthrottleinterval) + - [`Bootstrap`](#bootstrap) + - [`Datastore`](#datastore) + - [`Datastore.StorageMax`](#datastorestoragemax) + - [`Datastore.StorageGCWatermark`](#datastorestoragegcwatermark) + - [`Datastore.GCPeriod`](#datastoregcperiod) + - [`Datastore.HashOnRead`](#datastorehashonread) + - [`Datastore.BloomFilterSize`](#datastorebloomfiltersize) + - [`Datastore.Spec`](#datastorespec) + - [`Discovery`](#discovery) + - [`Discovery.MDNS`](#discoverymdns) + - [`Discovery.MDNS.Enabled`](#discoverymdnsenabled) + - [`Discovery.MDNS.Interval`](#discoverymdnsinterval) + - [`Gateway`](#gateway) + - [`Gateway.NoFetch`](#gatewaynofetch) + - [`Gateway.NoDNSLink`](#gatewaynodnslink) + - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) + - [`Gateway.RootRedirect`](#gatewayrootredirect) + - [`Gateway.Writable`](#gatewaywritable) + - [`Gateway.PathPrefixes`](#gatewaypathprefixes) + - [`Gateway.PublicGateways`](#gatewaypublicgateways) + - [`Gateway.PublicGateways: Paths`](#gatewaypublicgateways-paths) + - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains) + - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink) + - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways) + - [`Gateway` recipes](#gateway-recipes) + - [`Identity`](#identity) + - [`Identity.PeerID`](#identitypeerid) + - [`Identity.PrivKey`](#identityprivkey) + - [`Internal`](#internal) + - [`Internal.Bitswap`](#internalbitswap) + - [`Internal.Bitswap.TaskWorkerCount`](#internalbitswaptaskworkercount) + - [`Internal.Bitswap.EngineBlockstoreWorkerCount`](#internalbitswapengineblockstoreworkercount) + - [`Internal.Bitswap.EngineTaskWorkerCount`](#internalbitswapenginetaskworkercount) + - [`Internal.Bitswap.MaxOutstandingBytesPerPeer`](#internalbitswapmaxoutstandingbytesperpeer) + - [`Ipns`](#ipns) + - [`Ipns.RepublishPeriod`](#ipnsrepublishperiod) + - [`Ipns.RecordLifetime`](#ipnsrecordlifetime) + - [`Ipns.ResolveCacheSize`](#ipnsresolvecachesize) + - [`Migration`](#migration) + - [`Migration.DownloadSources`](#migrationdownloadsources) + - [`Migration.Keep`](#migrationkeep) + - [`Mounts`](#mounts) + - [`Mounts.IPFS`](#mountsipfs) + - [`Mounts.IPNS`](#mountsipns) + - [`Mounts.FuseAllowOther`](#mountsfuseallowother) + - [`Pinning`](#pinning) + - [`Pinning.RemoteServices`](#pinningremoteservices) + - [`Pinning.RemoteServices: API`](#pinningremoteservices-api) + - [`Pinning.RemoteServices: API.Endpoint`](#pinningremoteservices-apiendpoint) + - [`Pinning.RemoteServices: API.Key`](#pinningremoteservices-apikey) + - [`Pinning.RemoteServices: Policies`](#pinningremoteservices-policies) + - [`Pinning.RemoteServices: Policies.MFS`](#pinningremoteservices-policiesmfs) + - [`Pinning.RemoteServices: Policies.MFS.Enabled`](#pinningremoteservices-policiesmfsenabled) + - [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname) + - [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval) + - [`Pubsub`](#pubsub) + - [`Pubsub.Router`](#pubsubrouter) + - [`Pubsub.DisableSigning`](#pubsubdisablesigning) + - [`Peering`](#peering) + - [`Peering.Peers`](#peeringpeers) + - [`Reprovider`](#reprovider) + - [`Reprovider.Interval`](#reproviderinterval) + - [`Reprovider.Strategy`](#reproviderstrategy) + - [`Routing`](#routing) + - [`Routing.Type`](#routingtype) + - [`Swarm`](#swarm) + - [`Swarm.AddrFilters`](#swarmaddrfilters) + - [`Swarm.DisableBandwidthMetrics`](#swarmdisablebandwidthmetrics) + - [`Swarm.DisableNatPortMap`](#swarmdisablenatportmap) + - [`Swarm.DisableRelay`](#swarmdisablerelay) + - [`Swarm.EnableRelayHop`](#swarmenablerelayhop) + - [`Swarm.EnableAutoRelay`](#swarmenableautorelay) + - [Mode 1: `EnableRelayHop` is `false`](#mode-1-enablerelayhop-is-false) + - [Mode 2: `EnableRelayHop` is `true`](#mode-2-enablerelayhop-is-true) + - [`Swarm.EnableAutoNATService`](#swarmenableautonatservice) + - [`Swarm.ConnMgr`](#swarmconnmgr) + - [`Swarm.ConnMgr.Type`](#swarmconnmgrtype) + - [Basic Connection Manager](#basic-connection-manager) + - [`Swarm.ConnMgr.LowWater`](#swarmconnmgrlowwater) + - [`Swarm.ConnMgr.HighWater`](#swarmconnmgrhighwater) + - [`Swarm.ConnMgr.GracePeriod`](#swarmconnmgrgraceperiod) + - [`Swarm.Transports`](#swarmtransports) + - [`Swarm.Transports.Network`](#swarmtransportsnetwork) + - [`Swarm.Transports.Network.TCP`](#swarmtransportsnetworktcp) + - [`Swarm.Transports.Network.Websocket`](#swarmtransportsnetworkwebsocket) + - [`Swarm.Transports.Network.QUIC`](#swarmtransportsnetworkquic) + - [`Swarm.Transports.Network.Relay`](#swarmtransportsnetworkrelay) + - [`Swarm.Transports.Security`](#swarmtransportssecurity) + - [`Swarm.Transports.Security.TLS`](#swarmtransportssecuritytls) + - [`Swarm.Transports.Security.SECIO`](#swarmtransportssecuritysecio) + - [`Swarm.Transports.Security.Noise`](#swarmtransportssecuritynoise) + - [`Swarm.Transports.Multiplexers`](#swarmtransportsmultiplexers) + - [`Swarm.Transports.Multiplexers.Yamux`](#swarmtransportsmultiplexersyamux) + - [`Swarm.Transports.Multiplexers.Mplex`](#swarmtransportsmultiplexersmplex) + - [`DNS`](#dns) + - [`DNS.Resolvers`](#dnsresolvers) + + + ## Profiles Configuration profiles allow to tweak configuration quickly. Profiles can be -applied with `--profile` flag to `ipfs init` or with the `ipfs config profile +applied with the `--profile` flag to `ipfs init` or with the `ipfs config profile apply` command. When a profile is applied a backup of the configuration file will be created in `$IPFS_PATH`. @@ -22,7 +146,7 @@ documented in `ipfs config profile --help`. - `randomports` - Use a random port number for swarm. + Use a random port number for the incoming swarm connections. - `default-datastore` @@ -34,8 +158,8 @@ documented in `ipfs config profile --help`. - `local-discovery` - Sets default values to fields affected by the server - profile, enables discovery in local networks. + Enables local discovery (enabled by default). Useful to re-enable local discovery after it's + disabled by another profile (e.g., the server profile). - `test` @@ -56,7 +180,7 @@ documented in `ipfs config profile --help`. - You need a very simple and very reliable datastore and you trust your filesystem. This datastore stores each block as a separate file in the - underlying filesystem so it's unlikely to loose data unless there's an issue + underlying filesystem so it's unlikely to lose data unless there's an issue with the underlying file system. - You need to run garbage collection on a small (<= 10GiB) datastore. The default datastore, badger, can leave several gigabytes of data behind when @@ -129,100 +253,13 @@ of strings, or null: Duration is a type for describing lengths of time, using the same format go does (e.g, `"1d2h4m40.01s"`). -## Table of Contents +### `optionalInteger` -- [`Addresses`](#addresses) - - [`Addresses.API`](#addressesapi) - - [`Addresses.Gateway`](#addressesgateway) - - [`Addresses.Swarm`](#addressesswarm) - - [`Addresses.Announce`](#addressesannounce) - - [`Addresses.NoAnnounce`](#addressesnoannounce) -- [`API`](#api) - - [`API.HTTPHeaders`](#apihttpheaders) -- [`AutoNAT`](#autonat) - - [`AutoNAT.ServiceMode`](#autonatservicemode) - - [`AutoNAT.Throttle`](#autonatthrottle) - - [`AutoNAT.Throttle.GlobalLimit`](#autonatthrottlegloballimit) - - [`AutoNAT.Throttle.PeerLimit`](#autonatthrottlepeerlimit) - - [`AutoNAT.Throttle.Interval`](#autonatthrottleinterval) -- [`Bootstrap`](#bootstrap) -- [`Datastore`](#datastore) - - [`Datastore.StorageMax`](#datastorestoragemax) - - [`Datastore.StorageGCWatermark`](#datastorestoragegcwatermark) - - [`Datastore.GCPeriod`](#datastoregcperiod) - - [`Datastore.HashOnRead`](#datastorehashonread) - - [`Datastore.BloomFilterSize`](#datastorebloomfiltersize) - - [`Datastore.Spec`](#datastorespec) -- [`Discovery`](#discovery) - - [`Discovery.MDNS`](#discoverymdns) - - [`Discovery.MDNS.Enabled`](#discoverymdnsenabled) - - [`Discovery.MDNS.Interval`](#discoverymdnsinterval) -- [`Gateway`](#gateway) - - [`Gateway.NoFetch`](#gatewaynofetch) - - [`Gateway.NoDNSLink`](#gatewaynodnslink) - - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - - [`Gateway.RootRedirect`](#gatewayrootredirect) - - [`Gateway.Writable`](#gatewaywritable) - - [`Gateway.PathPrefixes`](#gatewaypathprefixes) - - [`Gateway.PublicGateways`](#gatewaypublicgateways) -- [`Identity`](#identity) - - [`Identity.PeerID`](#identitypeerid) - - [`Identity.PrivKey`](#identityprivkey) -- [`Ipns`](#ipns) - - [`Ipns.RepublishPeriod`](#ipnsrepublishperiod) - - [`Ipns.RecordLifetime`](#ipnsrecordlifetime) - - [`Ipns.ResolveCacheSize`](#ipnsresolvecachesize) -- [`Migration`](#migration) - - [`Migration.DownloadSources`](#migrationdownloadsources) - - [`Migration.Keep`](#migrationkeep) -- [`Mounts`](#mounts) - - [`Mounts.IPFS`](#mountsipfs) - - [`Mounts.IPNS`](#mountsipns) - - [`Mounts.FuseAllowOther`](#mountsfuseallowother) -- [`Pinning`](#pinning) - - [`Pinning.RemoteServices`](#pinningremoteservices) - - [`Pinning.RemoteServices.API`](#pinningremoteservices-api) - - [`Pinning.RemoteServices.API.Endpoint`](#pinningremoteservices-apiendpoint) - - [`Pinning.RemoteServices.API.Key`](#pinningremoteservices-apikey) - - [`Pinning.RemoteServices.Policies`](#pinningremoteservices-policies) - - [`Pinning.RemoteServices.Policies.MFS`](#pinningremoteservices-policiesmfs) -- [`Pubsub`](#pubsub) - - [`Pubsub.Router`](#pubsubrouter) - - [`Pubsub.DisableSigning`](#pubsubdisablesigning) -- [`Peering`](#peering) - - [`Peering.Peers`](#peeringpeers) -- [`Reprovider`](#reprovider) - - [`Reprovider.Interval`](#reproviderinterval) - - [`Reprovider.Strategy`](#reproviderstrategy) -- [`Routing`](#routing) - - [`Routing.Type`](#routingtype) -- [`Swarm`](#swarm) - - [`Swarm.AddrFilters`](#swarmaddrfilters) - - [`Swarm.DisableBandwidthMetrics`](#swarmdisablebandwidthmetrics) - - [`Swarm.DisableNatPortMap`](#swarmdisablenatportmap) - - [`Swarm.DisableRelay`](#swarmdisablerelay) - - [`Swarm.EnableRelayHop`](#swarmenablerelayhop) - - [`Swarm.EnableAutoRelay`](#swarmenableautorelay) - - [`Swarm.ConnMgr`](#swarmconnmgr) - - [`Swarm.ConnMgr.Type`](#swarmconnmgrtype) - - [`Swarm.ConnMgr.LowWater`](#swarmconnmgrlowwater) - - [`Swarm.ConnMgr.HighWater`](#swarmconnmgrhighwater) - - [`Swarm.ConnMgr.GracePeriod`](#swarmconnmgrgraceperiod) - - [`Swarm.Transports`](#swarmtransports) - - [`Swarm.Transports.Security`](#swarmtransportssecurity) - - [`Swarm.Transports.Security.TLS`](#swarmtransportssecuritytls) - - [`Swarm.Transports.Security.SECIO`](#swarmtransportssecuritysecio) - - [`Swarm.Transports.Security.Noise`](#swarmtransportssecuritynoise) - - [`Swarm.Transports.Multiplexers`](#swarmtransportsmultiplexers) - - [`Swarm.Transports.Multiplexers.Yamux`](#swarmtransportsmultiplexersyamux) - - [`Swarm.Transports.Multiplexers.Mplex`](#swarmtransportsmultiplexersmplex) - - [`Swarm.Transports.Network`](#swarmtransportsnetwork) - - [`Swarm.Transports.Network.TCP`](#swarmtransportsnetworktcp) - - [`Swarm.Transports.Network.QUIC`](#swarmtransportsnetworkquic) - - [`Swarm.Transports.Network.Websocket`](#swarmtransportsnetworkwebsocket) - - [`Swarm.Transports.Network.Relay`](#swarmtransportsnetworkrelay) -- [`DNS`](#dns) - - [`DNS.Resolvers`](#dnsresolvers) +Optional Integers allow specifying some numerical value which has +an implicit default when `null` or missing from the config file: + +- `null`/missing (apply the default value defined in go-ipfs sources) +- an integer between `-2^63` and `2^63-1` (i.e. `-9223372036854775808` to `9223372036854775807`) ## `Addresses` @@ -258,7 +295,7 @@ Type: `strings` (multiaddrs) ### `Addresses.Swarm` -Array of multiaddrs describing which addresses to listen on for p2p swarm +An array of multiaddrs describing which addresses to listen on for p2p swarm connections. Supported Transports: @@ -289,7 +326,7 @@ Default: `[]` Type: `array[string]` (multiaddrs) ### `Addresses.NoAnnounce` -Array of swarm addresses not to announce to the network. +An array of swarm addresses not to announce to the network. Default: `[]` @@ -363,8 +400,7 @@ Type: `duration` (when `0`/unset, the default value is used) ## `Bootstrap` -Bootstrap is an array of multiaddrs of trusted nodes to connect to in order to -initiate a connection to the network. +Bootstrap is an array of multiaddrs of trusted nodes that your node connects to, to fetch other nodes of the network on startup. Default: The ipfs.io bootstrap nodes @@ -405,7 +441,7 @@ Type: `duration` (an empty string means the default value) ### `Datastore.HashOnRead` -A boolean value. If set to true, all block reads from disk will be hashed and +A boolean value. If set to true, all block reads from the disk will be hashed and verified. This will cause increased CPU utilization. Default: `false` @@ -416,13 +452,13 @@ Type: `bool` A number representing the size in bytes of the blockstore's [bloom filter](https://en.wikipedia.org/wiki/Bloom_filter). A value of zero represents -the feature being disabled. +the feature is disabled. This site generates useful graphs for various bloom filter values: You may use it to find a preferred optimal value, where `m` is `BloomFilterSize` in bits. Remember to convert the value `m` from bits, into bytes for use as `BloomFilterSize` in the -config file. For example, for 1,000,000 blocks, expecting a 1% false positive +config file. For example, for 1,000,000 blocks, expecting a 1% false-positive rate, you'd end up with a filter size of 9592955 bits, so for `BloomFilterSize` we'd want to use 1199120 bytes. As of writing, [7 hash functions](https://github.com/ipfs/go-ipfs-blockstore/blob/547442836ade055cc114b562a3cc193d4e57c884/caching.go#L22) @@ -496,7 +532,7 @@ Type: `bool` #### `Discovery.MDNS.Interval` -A number of seconds to wait between discovery checks. +The number of seconds between discovery checks. Default: `5` @@ -518,8 +554,8 @@ Type: `bool` ### `Gateway.NoDNSLink` A boolean to configure whether DNSLink lookup for value in `Host` HTTP header -should be performed. If DNSLink is present, content path stored in the DNS TXT -record becomes the `/` and respective payload is returned to the client. +should be performed. If DNSLink is present, the content path stored in the DNS TXT +record becomes the `/` and the respective payload is returned to the client. Default: `false` @@ -567,7 +603,7 @@ Type: `bool` **DEPRECATED:** see [go-ipfs#7702](https://github.com/ipfs/go-ipfs/issues/7702)