feat: re-enable docker sharness tests

.circleci/main.yml

@@ -154,7 +154,7 @@ jobs:
     working_directory: ~/ipfs/go-ipfs
     environment:
         <<: *default_environment
-        TEST_NO_DOCKER: 1
+        TEST_NO_DOCKER: 0
         TEST_NO_FUSE: 1
         TEST_VERBOSE: 1
     steps:

.dockerignore

@@ -5,6 +5,8 @@ Dockerfile.fast
 !.git/refs/
 !.git/packed-refs
 test/sharness/lib/sharness/
+test/sharness/trash*
+rb-pinning-service-api/
 
 # The Docker client might not be running on Linux
 # so delete any compiled binaries

test/dependencies/pollEndpoint/main.go

@@ -2,7 +2,11 @@
 package main
 
 import (
+	"context"
 	"flag"
+	"io"
+	"net"
+	"net/http"
 	"os"
 	"time"
 
@@ -15,6 +19,8 @@ var (
 	host    = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on")
 	tries   = flag.Int("tries", 10, "how many tries to make before failing")
 	timeout = flag.Duration("tout", time.Second, "how long to wait between attempts")
+	httpURL = flag.String("http-url", "", "HTTP URL to fetch")
+	httpOut = flag.Bool("http-out", false, "Print the HTTP response body to stdout")
 	verbose = flag.Bool("v", false, "verbose logging")
 )
 
@@ -37,18 +43,76 @@ func main() {
 	start := time.Now()
 	log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr)
 
-	for *tries > 0 {
+	connTries := *tries
+	for connTries > 0 {
 		c, err := manet.Dial(addr)
 		if err == nil {
 			log.Debugf("ok -  endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start))
 			c.Close()
-			os.Exit(0)
+			break
 		}
 		log.Debug("connect failed: ", err)
 		time.Sleep(*timeout)
-		*tries--
+		connTries--
 	}
 
-	log.Error("failed.")
+	if err != nil {
+		goto Fail
+	}
+
+	if *httpURL != "" {
+		dialer := &connDialer{addr: addr}
+		httpClient := http.Client{Transport: &http.Transport{
+			DialContext: dialer.DialContext,
+		}}
+		reqTries := *tries
+		for reqTries > 0 {
+			try := (*tries - reqTries) + 1
+			log.Debugf("trying HTTP req %d: '%s'", try, *httpURL)
+			if tryHTTPGet(&httpClient, *httpURL) {
+				log.Debugf("HTTP req %d to '%s' succeeded", try, *httpURL)
+				goto Success
+			}
+			log.Debugf("HTTP req %d to '%s' failed", try, *httpURL)
+			time.Sleep(*timeout)
+			reqTries--
+		}
+		goto Fail
+	}
+
+Success:
+	os.Exit(0)
+
+Fail:
+	log.Error("failed")
 	os.Exit(1)
 }
+
+func tryHTTPGet(client *http.Client, url string) bool {
+	resp, err := client.Get(*httpURL)
+	if resp != nil && resp.Body != nil {
+		defer resp.Body.Close()
+	}
+	if err != nil {
+		return false
+	}
+	if resp.StatusCode != http.StatusOK {
+		return false
+	}
+	if *httpOut {
+		_, err := io.Copy(os.Stdout, resp.Body)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return true
+}
+
+type connDialer struct {
+	addr ma.Multiaddr
+}
+
+func (d connDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	return (&manet.Dialer{}).DialContext(ctx, d.addr)
+}

test/ipfs-test-lib.sh

@@ -62,12 +62,7 @@ docker_run() {
 
 # This takes a docker ID and a command as arguments
 docker_exec() {
-    if test "$CIRCLE" = 1
-    then
-        sudo lxc-attach -n "$(docker inspect --format '{{.Id}}' $1)" -- /bin/bash -c "$2"
-    else
-	docker exec -t "$1" /bin/bash -c "$2"
-    fi
+    docker exec -t "$1" /bin/sh -c "$2"
 }
 
 # This takes a docker ID as argument

test/sharness/t0300-docker-image.sh

@@ -29,30 +29,28 @@ TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR")
 APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR")
 
 test_expect_success "docker image build succeeds" '
-  docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >actual ||
+  docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >build-actual ||
   test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" ||
   test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" ||
-  test_fsh cat actual
+  test_fsh cat build-actual
 '
 
 test_expect_success "docker image build output looks good" '
-  SUCCESS_LINE=$(egrep "^Successfully built" actual) &&
+  SUCCESS_LINE=$(egrep "^Successfully built" build-actual) &&
   IMAGE_ID=$(expr "$SUCCESS_LINE" : "^Successfully built \(.*\)") ||
-  test_fsh cat actual
+  test_fsh cat build-actual
 '
 
 test_expect_success "docker image runs" '
-  DOC_ID=$(docker_run "$IMAGE_ID")
+  DOC_ID=$(docker run -d -p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 "$IMAGE_ID")
 '
 
-test_expect_success "docker image gateway is up" '
-  docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
-    -q -O - http://localhost:8080/version >/dev/null"
+test_expect_success "docker container gateway is up" '
+  pollEndpoint -host=/ip4/127.0.0.1/tcp/8080 -http-url http://localhost:8080/api/v0/version -v -tries 30 -tout 1s
 '
 
-test_expect_success "docker image API is up" '
-  docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
-    -q -O - http://localhost:5001/api/v0/version >/dev/null"
+test_expect_success "docker container API is up" '
+  pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -v -tries 30 -tout 1s
 '
 
 test_expect_success "simple ipfs add/cat can be run in docker container" '
@@ -63,8 +61,7 @@ test_expect_success "simple ipfs add/cat can be run in docker container" '
 '
 
 read testcode <<EOF
-  docker exec -i "$DOC_ID" wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
-    -q -O - http://localhost:8080/version | grep Commit | cut -d" " -f2 >actual ; \
+  pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -http-out | grep Commit | cut -d" " -f2 >actual ; \
   test -s actual ; \
   docker exec -i "$DOC_ID" ipfs version --enc json \
     | sed 's/^.*"Commit":"\\\([^"]*\\\)".*$/\\\1/g' >expected ; \

test/sharness/t0301-docker-migrate.sh

@@ -32,6 +32,10 @@ test_expect_success "docker image build succeeds" '
 
 test_init_ipfs
 
+test_expect_success "configure migration sources" '
+  ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]"
+'
+
 test_expect_success "make repo be version 4" '
   echo 4 > "$IPFS_PATH/version"
 '
@@ -43,17 +47,13 @@ test_expect_success "setup http response" '
   echo "v1.1.1" >> vers_resp
 '
 
-pretend_server() {
-    socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr 'SYSTEM:cat vers_resp'!!STDERR &
-}
-
 test_expect_success "startup fake dists server" '
-  pretend_server > dist_serv_out &
+  ( socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr "SYSTEM:cat vers_resp"!!STDERR 2> dist_serv_out ) &
   echo $! > netcat_pid
 '
 
 test_expect_success "docker image runs" '
-  DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host -e IPFS_DIST_PATH="http://localhost:17233" "$IMAGE_ID" --migrate)
+  DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_ID")
 '
 
 test_expect_success "docker container tries to pull migrations from netcat" '
@@ -74,7 +74,7 @@ test_expect_success "kill the net cat" '
 '
 
 test_expect_success "correct version was requested" '
-  grep "/fs-repo-migrations/v1.1.1/fs-repo-migrations_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null
+  grep "/fs-repo-6-to-7/v1.1.1/fs-repo-6-to-7_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null
 '
 
 test_done