prepare for a v0.4.22 patch release

.circleci/config.yml

@@ -2,17 +2,17 @@ version: 2.0
 
 aliases:
   make_out_dirs: &make_out_dirs
-    run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS/{unit,sharness}
+    run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_WORKSPACE $CIRCLE_TEST_REPORTS/{unit,sharness}
   restore_gomod: &restore_gomod
     restore_cache:
       keys:
-        - v3-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
-        - v3-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-
-        - v3-dep-{{ .Branch }}-
-        - v3-dep-master-
+        - v4-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
+        - v4-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-
+        - v4-dep-{{ .Branch }}-
+        - v4-dep-master-
   store_gomod: &store_gomod
       save_cache:
-        key: v3-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
+        key: v4-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
         paths:
           - ~/go/pkg/mod
           - ~/.cache/go-build/
@@ -21,6 +21,7 @@ aliases:
 defaults: &defaults
   working_directory: ~/ipfs/go-ipfs
   environment:
+    GIT_PAGER: cat
     GO111MODULE: "on"
     TEST_NO_DOCKER: 1
     TEST_NO_FUSE: 1
@@ -29,14 +30,14 @@ defaults: &defaults
     CIRCLE: 1
     SERVICE: circle-ci
     CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    CIRCLE_WORKSPACE: /tmp/circleci-workspace
     TEST_VERBOSE: 1
     TRAVIS: 1
 
-  docker:
-    - image: circleci/golang:1.12
-
 jobs:
   gobuild:
+    docker:
+      - image: circleci/golang:1.12
     <<: *defaults
     steps:
     - checkout
@@ -53,6 +54,8 @@ jobs:
     - *store_gomod
   golint:
     <<: *defaults
+    docker:
+      - image: circleci/golang:1.12
     steps:
     - checkout
     - *make_out_dirs
@@ -61,6 +64,8 @@ jobs:
         make -O test_go_lint
     - *store_gomod
   gotest:
+    docker:
+      - image: circleci/golang:1.12
     <<: *defaults
     steps:
     - checkout
@@ -88,6 +93,8 @@ jobs:
     - store_artifacts:
         path: /tmp/circleci-test-results
   sharness:
+    docker:
+      - image: circleci/golang:1.12
     <<: *defaults
     steps:
     - run: sudo apt install socat
@@ -114,7 +121,125 @@ jobs:
         path: /tmp/circleci-artifacts
     - store_artifacts:
         path: /tmp/circleci-test-results
-
+  build:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - checkout
+      - *make_out_dirs
+      - *restore_gomod
+      - run:
+          name: Building
+          command: make build
+      - run:
+          name: Storing
+          command: |
+            mkdir -p /tmp/circleci-workspace/bin
+            cp cmd/ipfs/ipfs /tmp/circleci-workspace/bin
+      - persist_to_workspace:
+          root: /tmp/circleci-workspace
+          paths:
+            - bin/ipfs
+      - *store_gomod
+  interop:
+    docker:
+      - image: circleci/node:10
+    <<: *defaults
+    steps:
+      - *make_out_dirs
+      - attach_workspace:
+          at: /tmp/circleci-workspace
+      - run:
+          name: Cloning
+          command: |
+            git clone https://github.com/ipfs/interop.git
+            git -C interop log -1
+      - restore_cache:
+          keys:
+            - v1-interop-{{ checksum "~/ipfs/go-ipfs/interop/package-lock.json" }}
+            - v1-interop-
+      - run:
+          command: npm install
+          working_directory: ~/ipfs/go-ipfs/interop
+          environment:
+            IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
+      - save_cache:
+          key: v1-interop-{{ checksum "~/ipfs/go-ipfs/interop/package-lock.json" }}
+          paths:
+            - ~/ipfs/go-ipfs/interop/node_modules
+      - run:
+          command: npm run test:node
+          working_directory: ~/ipfs/go-ipfs/interop
+          environment:
+            IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
+  go-ipfs-api:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - *make_out_dirs
+      - attach_workspace:
+          at: /tmp/circleci-workspace
+      - run:
+          name: Cloning
+          command: |
+            git clone https://github.com/ipfs/go-ipfs-api.git
+            git -C go-ipfs-api log -1
+      - run:
+          name: Starting the daemon
+          command: /tmp/circleci-workspace/bin/ipfs daemon --init --enable-namesys-pubsub
+          background: true
+      - run:
+          name: Waiting for the daemon
+          no_output_timeout: 10s
+          command: |
+            while ! /tmp/circleci-workspace/bin/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do
+              sleep 1
+            done
+      - restore_cache:
+          keys:
+            - v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}
+            - v1-go-api-
+      - run:
+          command: go test -v ./...
+          working_directory: ~/ipfs/go-ipfs/go-ipfs-api
+      - save_cache:
+          key: v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}
+          paths:
+            - ~/go/pkg/mod
+            - ~/.cache/go-build/
+      - run:
+          name: Stopping the daemon
+          command: /tmp/circleci-workspace/bin/ipfs shutdown
+  go-ipfs-http-client:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - *make_out_dirs
+      - attach_workspace:
+          at: /tmp/circleci-workspace
+      - run:
+          name: Cloning
+          command: |
+            git clone https://github.com/ipfs/go-ipfs-http-client.git
+            git -C go-ipfs-http-client log -1
+      - restore_cache:
+          keys:
+            - v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}
+            - v1-http-client-
+      - run:
+          name: go test -v ./...
+          command: |
+            export PATH=/tmp/circleci-workspace/bin:$PATH
+            go test -v ./...
+          working_directory: ~/ipfs/go-ipfs/go-ipfs-http-client
+      - save_cache:
+          key: v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}
+          paths:
+            - ~/go/pkg/mod
+            - ~/.cache/go-build/
 workflows:
   version: 2
   test:
@@ -123,3 +248,13 @@ workflows:
     - golint
     - gotest
     - sharness
+    - build
+    - interop:
+        requires:
+          - build
+    - go-ipfs-api:
+        requires:
+          - build
+    - go-ipfs-http-client:
+        requires:
+          - build

CHANGELOG.md

@@ -1,5 +1,46 @@
 # go-ipfs changelog
 
+## 0.4.22 2019-08-06
+
+We're releasing a PATCH release of go-ipfs based on 0.4.21 containing some critical fixes.
+
+The past several releases have been shaky and the network has scaled to the
+point where where small changes can have a wide-reaching impact on the entire
+network. To keep this situation from escalating, we've put a hold on releasing
+new features until we can improve our [release
+process](https://github.com/ipfs/go-ipfs/pull/6482) (which we will be trialing
+in this release) and [testing
+procedures](https://github.com/ipfs/go-ipfs/issues/6483).
+
+This release includes fixes for the following regressions:
+
+1. A major bitswap throughput regression introduced in 0.4.21
+   ([ipfs/go-ipfs#6442](https://github.com/ipfs/go-ipfs/issues/6442)).
+2. High bitswap CPU usage when connected to many (e.g., 10,000) peers. See
+   [ipfs/go-bitswap#154](https://github.com/ipfs/go-bitswap/issues/154).
+2. The local network discovery service sometimes initializing before the
+   networking module, causing it to announce the wrong addresses and sometimes
+   complain about not being able to determine the IP address)
+   ([ipfs/go-ipfs#6415](https://github.com/ipfs/go-ipfs/pull/6415)).
+
+It also includes fixes for:
+
+1. Pins not being persisted after `ipfs block add --pin`
+   ([ipfs/go-ipfs#6441](https://github.com/ipfs/go-ipfs/pull/6441)).
+2. Concurrent map access on GC due to the pinner
+   ([ipfs/go-ipfs#6419](https://github.com/ipfs/go-ipfs/pull/6419)).
+3. Potential pin-set corruption given a concurrent `ipfs repo gc` and `ipfs pin
+   rm` ([ipfs/go-ipfs#6444](https://github.com/ipfs/go-ipfs/pull/6444)).
+4. Build failure due to a deleted git tag in one of our dependencies
+   ([ipfs/go-ds-badger#64](https://github.com/ipfs/go-ds-badger/pull/65)).
+
+
+Thanks to:
+
+* @hannahhoward for fixing both bitswap issues.
+* @sanderpick for catching and fixing the local discovery bug.
+* @campoy for fixing the build issue.
+
 ## 0.4.21 2019-05-30
 
 We're happy to announce go-ipfs 0.4.21. This release has some critical bug fixes

core/commands/block.go

@@ -121,10 +121,10 @@ var blockPutCmd = &cmds.Command{
 		Tagline: "Store input as an IPFS block.",
 		ShortDescription: `
 'ipfs block put' is a plumbing command for storing raw IPFS blocks.
-It reads from stdin, and <key> is a base58 encoded multihash.
+It reads from stdin, and outputs the block's CID to stdout.
 
-By default CIDv0 is going to be generated. Setting 'mhtype' to anything other
-than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
+Unless specified, this command returns dag-pb CIDv0 CIDs. Setting 'mhtype' to anything
+other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
 `,
 	},
 

core/coreapi/block.go

@@ -56,6 +56,9 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
 
 	if settings.Pin {
 		api.pinning.PinWithMode(b.Cid(), pin.Recursive)
+		if err := api.pinning.Flush(); err != nil {
+			return nil, err
+		}
 	}
 
 	return &BlockStat{path: path.IpldPath(b.Cid()), size: len(data)}, nil

core/coreapi/pin.go

@@ -67,6 +67,10 @@ func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOpti
 		return err
 	}
 
+	// Note: after unpin the pin sets are flushed to the blockstore, so we need
+	// to take a lock to prevent a concurrent garbage collection
+	defer api.blockstore.PinLock().Unlock()
+
 	if err = api.pinning.Unpin(ctx, rp.Cid(), settings.Recursive); err != nil {
 		return err
 	}

core/node/groups.go

@@ -96,11 +96,11 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option {
 		BaseLibP2P,
 
 		fx.Provide(libp2p.AddrFilters(cfg.Swarm.AddrFilters)),
-		fx.Invoke(libp2p.SetupDiscovery(cfg.Discovery.MDNS.Enabled, cfg.Discovery.MDNS.Interval)),
 		fx.Provide(libp2p.AddrsFactory(cfg.Addresses.Announce, cfg.Addresses.NoAnnounce)),
 		fx.Provide(libp2p.SmuxTransport(bcfg.getOpt("mplex"))),
 		fx.Provide(libp2p.Relay(cfg.Swarm.DisableRelay, cfg.Swarm.EnableRelayHop)),
 		fx.Invoke(libp2p.StartListening(cfg.Addresses.Swarm)),
+		fx.Invoke(libp2p.SetupDiscovery(cfg.Discovery.MDNS.Enabled, cfg.Discovery.MDNS.Interval)),
 
 		fx.Provide(libp2p.Security(!bcfg.DisableEncryptedConnections, cfg.Experimental.PreferTLS)),
 

go.mod

@@ -18,14 +18,14 @@ require (
 	github.com/hashicorp/golang-lru v0.5.1
 	github.com/hsanjuan/go-libp2p-http v0.0.2
 	github.com/ipfs/dir-index-html v1.0.3
-	github.com/ipfs/go-bitswap v0.0.7
+	github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-blockservice v0.0.3
 	github.com/ipfs/go-cid v0.0.2
 	github.com/ipfs/go-cidutil v0.0.2
 	github.com/ipfs/go-datastore v0.0.5
 	github.com/ipfs/go-detect-race v0.0.1
-	github.com/ipfs/go-ds-badger v0.0.3
+	github.com/ipfs/go-ds-badger v0.0.5
 	github.com/ipfs/go-ds-flatfs v0.0.2
 	github.com/ipfs/go-ds-leveldb v0.0.2
 	github.com/ipfs/go-ds-measure v0.0.1
@@ -102,7 +102,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mr-tron/base58 v1.1.2
 	github.com/multiformats/go-multiaddr v0.0.4
-	github.com/multiformats/go-multiaddr-dns v0.0.2
+	github.com/multiformats/go-multiaddr-dns v0.0.3
 	github.com/multiformats/go-multiaddr-net v0.0.1
 	github.com/multiformats/go-multibase v0.0.1
 	github.com/multiformats/go-multihash v0.0.5
@@ -121,7 +121,8 @@ require (
 	go.uber.org/goleak v0.10.0 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
-	golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5
+	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
+	golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gotest.tools/gotestsum v0.3.4
 )