From 3e23e9857ca22c30d81fce723f3d16c70cd1e8ba Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 10 Jun 2024 20:49:29 +0200 Subject: [PATCH] feat: improved subdomain-url handling (#211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: subdomain-url handling - removed implicit tests based on SubdomainLocalhostGatewayURL - subdomain tests are now run only against origins defined via `--subdomain-url` - DNSLink gateway tests are no longer tied to origin passed via `--subdomain-url` * fix(wip): avoid mangling Host header handling - splitting into subdomain and dnslink gateway variants - towards avoiding running everything three times - keep custom Host header when defined * refactor: remove subdomains from dnslink fixtures these concepts should not mix, and dnslinks should be testable independent of --subdomain-url i also added more useful export format for them, compatible with IPNS_NS_MAP used by Boxo/Kubo/Rainbow * refactor: handling Host header vs proxying towards fixing underlying issue * feat: make start-kubo-docker all-in-one command for running local kubo with all fixtures in * fix: correctly pass subdomain-url to go test * docs: skip and run examples * refactor: UnwrapSubdomainTests → proxy-gateway This replaces magic helpers.UnwrapSubdomainTests with explicit spec preset named proxy-gateway, which can be enabled/disabled the same way as other presets. While it was nice to run every test in both proxy modes, it was extremely noisy and hard to debug once things went wrong. By having explicit tests in *_proxy_test.go it is easy to skip them where proxy functionality is not relevant. Other: - Send meaningful User-Agent - Sent detached path in HTTP Connect proxy tunnel tests (instead of full URL – in tunnel mode we pretend we talk to remote server on the other end of tunnel, and not send full URL like it is done in plain proxy mode) * docs: fix/improve docker examples * refactor: require explicit gateway URLs This is important UX change. We no longer ship with default URLs. User has to provide explicit one, or the test suite will refuse to run. This ensures misconfigurations and testing different gateway endpoint than desired do not happen. Explicit is better than implicit. --- .github/workflows/test-kubo-e2e.yml | 8 +- .gitignore | 2 + CHANGELOG.md | 30 ++ Dockerfile | 6 +- Makefile | 23 +- README.md | 59 ++- cmd/gateway-conformance/main.go | 237 ++++++---- cmd/gateway-conformance/main_test.go | 67 +++ docs/commands.md | 15 +- docs/examples.md | 25 +- fixtures/dir_listing/dnslink.yml | 4 +- fixtures/redirects_file/dnslink.yml | 8 +- fixtures/subdomain_gateway/dnslink.yml | 6 +- kubo-config.example.sh | 9 +- tests/dnslink_gateway_test.go | 33 +- tests/metadata_test.go | 4 +- tests/redirects_file_test.go | 491 ++++++++++--------- tests/subdomain_gateway_ipfs_test.go | 630 +++++++++++++------------ tests/subdomain_gateway_ipns_test.go | 349 +++++++------- tests/subdomain_gateway_proxy_test.go | 141 ++++++ tooling/dnslink/dnslink.go | 17 +- tooling/dnslink/merge.go | 51 +- tooling/helpers/subdomain.go | 87 ---- tooling/specs/specs.go | 2 + tooling/test/config.go | 52 +- tooling/test/proxy.go | 4 + tooling/test/run.go | 43 +- tooling/test/sugar.go | 20 +- 28 files changed, 1368 insertions(+), 1055 deletions(-) create mode 100644 cmd/gateway-conformance/main_test.go create mode 100644 tests/subdomain_gateway_proxy_test.go delete mode 100644 tooling/helpers/subdomain.go diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index 205fdd2d3..60e65b090 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -46,11 +46,7 @@ jobs: - name: Provision Kubo Gateway run: | # Import car files - cars=$(find ./fixtures -name '*.car') - for car in $cars - do - ipfs dag import --pin-roots=false --stats "$car" - done + find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false {} \; # Import ipns records records=$(find ./fixtures -name '*.ipns-record') @@ -63,7 +59,7 @@ jobs: uses: ./gateway-conformance/.github/actions/test with: gateway-url: http://127.0.0.1:8080 - subdomain-url: http://example.com + subdomain-url: http://example.com:8080 json: output.json xml: output.xml html: output.html diff --git a/.gitignore b/.gitignore index 655f91c9f..22304b4fe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ /gateway-conformance /dnslinks.json /*.ipns-record +extracted-fixtures # Logs logs @@ -142,3 +143,4 @@ dist # reports reports +report.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 874dce5de..577f40288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.6.0] - 2024-05-27 ### Changed +- Gateway URL + - `--gateway-url` is no longer defaulting to predefined URL. User has to + provide it via CLI or `GATEWAY_URL` environment variable or the test suite + will refuse to start. + - This aims to ensure no confusion about which gateway endpoint is being + tested. + - Docs and examples use `--gateway-url http://127.0.0.1:8080` to ensure no + confusion with `localhost:8080` subdomain gateway feature in IPFS + implementations like Kubo. +- Subdomain URL and UX related to subdomain tests + - The `--subdomain-url` is no longer set by default. + - User has to provide the origin of the subdomain gateway via CLI or + `SUBDOMAIN_GATEWAY_URL` to be used during subdomain tests. This aims to + ensure no confusion about which domain name is being tested. + - Simplified the way `--subdomain-url` works. We no longer run implicit tests + against `http://localhost` in addition to the URL passed via + `--subdomain-url`. To test more than one domain, run test multiple times. +- DNSLink test fixtures changed + - DNSLink fixtures no longer depend on `--subdomain-url` and use unrelated + `*.example.org` domains instead. + - `gateway-conformance extract-fixtures` creates `dnslinks.IPFS_NS_MAP` with + content that can be directly set as `IPNS_NS_MAP` environment variable + supported by various implementations, incl. + [Kubo](https://github.com/ipfs/kubo/blob/master/docs/environment-variables.md#ipfs_ns_map) + and + [Rainbow](https://github.com/ipfs/rainbow/blob/main/docs/environment-variables.md#ipfs_ns_map). +- Docker + - The image can now be run under non-root user + ## [0.5.2] - 2024-05-20 ### Changed - Fixed: relaxed dag-cbor error check ([#205](https://github.com/ipfs/gateway-conformance/pull/205)) diff --git a/Dockerfile b/Dockerfile index 468f2b208..f033b7dcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM golang:1.20-alpine WORKDIR /app -ENV GATEWAY_CONFORMANCE_HOME=/app +ENV GATEWAY_CONFORMANCE_HOME=/app \ + GOCACHE=/go/cache COPY ./go.mod ./go.sum ./ RUN go mod download @@ -9,4 +10,7 @@ COPY . . ARG VERSION=dev RUN go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=${VERSION}" -o ./gateway-conformance ./cmd/gateway-conformance +# Relaxed perms for cache dir to allow running under regular user +RUN mkdir -p $GOCACHE && chmod -R 777 $GOCACHE + ENTRYPOINT ["/app/gateway-conformance"] diff --git a/Makefile b/Makefile index 4654f491c..af272befd 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ GIT_COMMIT := $(shell git rev-parse --short HEAD) DIRTY_SUFFIX := $(shell test -n "`git status --porcelain`" && echo "-dirty" || true) CLI_VERSION := dev-$(GIT_COMMIT)$(DIRTY_SUFFIX) +KUBO_VERSION ?= latest +KUBO_DOCKER_NAME ?= kubo-$(KUBO_VERSION)-gateway-conformance all: gateway-conformance @@ -8,8 +10,9 @@ clean: clean-docker rm -f ./gateway-conformance rm -f *.ipns-record rm -f fixtures.car - rm -f dnslinks.json - rm -f ./reports/* + rm -f dnslinks.* + rm -f dnslink*.yml + rm -rf ./reports/* test-cargateway: provision-cargateway fixtures.car gateway-conformance ./gateway-conformance test --json reports/output.json --gateway-url http://127.0.0.1:8040 --specs -subdomain-gateway @@ -22,13 +25,27 @@ test-kubo: provision-kubo gateway-conformance ./gateway-conformance test --json reports/output.json --gateway-url http://127.0.0.1:8080 --specs -subdomain-gateway provision-cargateway: ./fixtures.car - # cd go-libipfs/examples/car && go install car -c ./fixtures.car & provision-kubo: find ./fixtures -name '*.car' -exec ipfs dag import --stats --pin-roots=false {} \; find ./fixtures -name '*.ipns-record' -exec sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}" | cut -d'_' -f1) "{}"' \; +#start-kubo-docker: stop-kubo-docker gateway-conformance +# ./gateway-conformance extract-fixtures --dir=.temp/fixtures +# docker pull ipfs/kubo:$(KUBO_VERSION) +# docker run -d --rm --net=host --name $(KUBO_DOCKER_NAME) -v "$(shell realpath .temp/fixtures)":/fixtures -v kubo-config.example.sh:/container-init.d/001-config.sh ipfs/kubo:$(KUBO_VERSION) daemon --init --offline +# @until docker exec $(KUBO_DOCKER_NAME) ipfs --api=/ip4/127.0.0.1/tcp/5001 dag stat /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn >/dev/null 2>&1; do sleep 0.1; done +# docker exec $(KUBO_DOCKER_NAME) find /fixtures -name '*.car' -exec ipfs dag import --stats --pin-roots=false {} \; +# docker exec $(KUBO_DOCKER_NAME) find /fixtures -name '*.ipns-record' -exec sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}" | cut -d'_' -f1) "{}"' \; +# TODO: provision Kubo config at Gateway.PublicGateways to have subdomain gateway on example.com and also enable inlining on localhost +# See: https://github.com/ipfs/kubo/blob/a07852a3f0294974b802923fb136885ad077384e/.github/workflows/gateway-conformance.yml#L22-L34 +# (this is not as trivial as it sounds because Kubo does not apply config inrealtime, and a restart is required.) + +stop-kubo-docker: clean + docker stop $(KUBO_DOCKER_NAME) || true + docker rm -f $(KUBO_DOCKER_NAME) || true + # tools fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. diff --git a/README.md b/README.md index 00498379a..76dc4c26a 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,12 @@ - [CLI](#cli) - [Docker](#docker) - [Github Action](#github-action) - - [Web Dashboard](#web-dashboard) - [Commands](#commands) - [Examples](#examples) - [Releases](#releases) - [Development](#development) - [Test DSL Syntax](#test-dsl-syntax) + - [Web Dashboard](#web-dashboard) - [License](#license) @@ -67,25 +67,32 @@ Two high level [commands](/docs/commands.md) exist: ```console $ # Install the gateway-conformance binary $ go install github.com/ipfs/gateway-conformance/cmd/gateway-conformance@latest -$ # run subdomain-gateway tests against endpoint at http://localhost:8080 output as JSON -$ gateway-conformance test --gateway-url http://localhost:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 30m + +$ # skip path gateway tests, and run subdomain-gateway tests against endpoint at http://127.0.0.1:8080 and use *.ipfs.example.com subdomains, output as JSON +$ gateway-conformance test --gateway-url http://127.0.0.1:8080 --subdomain-url http://example.com:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 5m ``` -If you are looking for TLDR, see [examples](/docs/examples.md). +> [!TIP] +> If want skip individual tests, or only run specific ones based on a regex, see [`/docs/examples`](/docs/examples.md). ### Docker -Prebuilt image at `ghcr.io/ipfs/gateway-conformance` can be used for both `test` and `extract-fixtures` commands: +The `gateway-conformace` requires golang runtime to be present to facilitate `go test`. +If you want to run it on a box without having to instal golang runtime, prebuilt image at `ghcr.io/ipfs/gateway-conformance` is provided. + +It can be used for both `test` and `extract-fixtures` commands: ```console -$ # extract fixtures to ./fixtures directory -$ docker run -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance:vA.B.C extract-fixtures --directory fixtures --merged false +$ # extract fixtures to ./extracted-fixtures directory under the current user's permissions +$ docker run -u "$(id -u):$(id -g)" -v "$(pwd):/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance:latest extract-fixtures --directory extracted-fixtures -$ # run subdomain-gateway tests against endpoint at http://localhost:8080 -$ docker run --network host -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance:vA.B.C test --gateway-url http://localhost:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 30m +$ # skip path gateway tests, and run subdomain-gateway tests against endpoint at http://127.0.0.1:8080 and use *.ipfs.example.com subdomains, output as JSON +$ docker run --net=host -u "$(id -u):$(id -g)" -v "$(pwd):/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance:latest test --gateway-url http://127.0.0.1:8080 --subdomain-url http://example.com:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 5m ``` -**NOTE:** replace `vA.B.C` with a [semantic version](https://github.com/ipfs/gateway-conformance/releases) version you want to test against +> [!IMPORTANT] +> - for stable CI/CD, replace `latest` with a [semantic version](https://github.com/ipfs/gateway-conformance/releases) version you want to test against +> - `-u` ensures extracted fixtures and created report files can be read by your local user, make sure to adjust it to suit your use case ### Github Action @@ -94,22 +101,9 @@ Common operations are possible via reusable GitHub actions: - [`ipfs/gateway-conformance/.github/actions/extract-fixtures`](https://github.com/ipfs/gateway-conformance/blob/main/.github/actions/extract-fixtures/action.yml) To learn how to integrate them in the CI of your project, see real world examples in: -- [`kubo/../gateway-conformance.yml`](https://github.com/ipfs/kubo/blob/master/.github/workflows/gateway-conformance.yml) (fixtures imported into tested node) -- [`boxo/../gateway-conformance.yml`](https://github.com/ipfs/boxo/blob/main/.github/workflows/gateway-conformance.yml) (fixtures imported into a sidecar kubo node that is peered with tested library) -- [`rainbow/../gateway-conformance.yml`](https://github.com/ipfs/rainbow/blob/main/.github/workflows/gateway-conformance.yml) (fixtures imported into a kubo node that acts as a delegated block backend) - -### Web Dashboard - -Conformance test suite output can be plain text or JSON, which in turn can be -represented as a web dashboard which aggregates results from many test runs and -renders them on a static website. - -The Implementation Dashboard instance at -[conformance.ipfs.tech](https://conformance.ipfs.tech/) is a view that -showcases some of well known and complete implementations of IPFS Gateways -in the ecosystem. - -Learn more at [`/docs/web-dashboard.md`](/docs/web-dashboard.md) +- [`kubo/../gateway-conformance.yml`](https://github.com/ipfs/kubo/blob/master/.github/workflows/gateway-conformance.yml) (fixtures imported into tested kubo node that exposes HTTP gateway feature) +- [`boxo/../gateway-conformance.yml`](https://github.com/ipfs/boxo/blob/main/.github/workflows/gateway-conformance.yml) (fixtures imported into a sidecar kubo node that is peered with small HTTP server used for testing `boxo/gateway` library) +- [`rainbow/../gateway-conformance.yml`](https://github.com/ipfs/rainbow/blob/main/.github/workflows/gateway-conformance.yml) (fixtures imported into a kubo node that acts as a remote block provider, than tested against different `boxo/gateway` backends) ## Commands @@ -142,6 +136,19 @@ Interested in write a new test case? Test cases are written in Domain Specific Language (DLS) based on Golang. More details at [`/docs/test-dsl-syntax.md`](/docs/test-dsl-syntax.md) +### Web Dashboard + +Conformance test suite output can be plain text or JSON, which in turn can be +represented as a web dashboard which aggregates results from many test runs and +renders them on a static website. + +Experimental Implementation Dashboard instance at +[conformance.ipfs.tech](https://conformance.ipfs.tech/) is a view that +showcases some of well known and complete implementations of IPFS Gateways +in the ecosystem. + +Learn more at [`/docs/web-dashboard.md`](/docs/web-dashboard.md) + ## License This project is dual-licensed under Apache 2.0 and MIT terms: diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index 994ea8455..1200e7649 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -14,6 +14,7 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" "github.com/ipfs/gateway-conformance/tooling/dnslink" "github.com/ipfs/gateway-conformance/tooling/fixtures" + specPresets "github.com/ipfs/gateway-conformance/tooling/specs" "github.com/urfave/cli/v2" ) @@ -69,15 +70,6 @@ func copyFiles(inputPaths []string, outputDirectoryPath string) error { } func main() { - var gatewayURL string - var subdomainGatewayURL string - var jsonOutput string - var jobURL string - var specs string - var directory string - var merged bool - var verbose bool - app := &cli.App{ Name: "gateway-conformance", Usage: "Tooling for the gateway test suite", @@ -89,68 +81,92 @@ func main() { Usage: "Run the conformance test suite against your gateway", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "gateway-url", - Aliases: []string{"url", "g"}, - Usage: "The URL of the IPFS Gateway implementation to be tested.", - Value: "http://localhost:8080", - Destination: &gatewayURL, + Name: "gateway-url", + EnvVars: []string{"GATEWAY_URL"}, + Aliases: []string{"url", "g"}, + Usage: "The URL of the IPFS Gateway implementation to be tested.", + Value: "", // unset by default, requires end user to either provide configured gateway endpoint URL }, &cli.StringFlag{ - Name: "subdomain-url", - Usage: "The Subdomain URL of the IPFS Gateway implementation to be tested.", - Value: "http://example.com", - Destination: &subdomainGatewayURL, + Name: "subdomain-url", + EnvVars: []string{"SUBDOMAIN_GATEWAY_URL"}, + Usage: "URL of the HTTP Host that should be used when testing https://specs.ipfs.tech/http-gateways/subdomain-gateway/ functionality", + Value: "", // unset by default, requires end user to either provide configured subdomain gateway origin URL, or pass '--specs -subdomain-gateway' to disable these tests }, &cli.StringFlag{ - Name: "json-output", - Aliases: []string{"json", "j"}, - Usage: "The path where the JSON test report should be generated.", - Value: "", - Destination: &jsonOutput, + Name: "json-output", + Aliases: []string{"json", "j"}, + Usage: "The path where the JSON test report should be generated.", + Value: "", }, &cli.StringFlag{ - Name: "job-url", - Aliases: []string{}, - Usage: "The Job URL where this run will be visible.", - Value: "", - Destination: &jobURL, + Name: "job-url", + Aliases: []string{}, + Usage: "The Job URL where this run will be visible.", + Value: "", }, &cli.StringFlag{ - Name: "specs", - Usage: "Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec).", - Value: "", - Destination: &specs, + Name: "specs", + EnvVars: []string{"SPECS"}, + Usage: "Adjust the scope of tests to run. Accepts a 'spec' (test only this spec), a '+spec' (test also this immature spec), or a '-spec' (do not test this mature spec). Available spec presets: " + strings.Join(getAvailableSpecPresets(), ","), + Value: "", }, &cli.BoolFlag{ - Name: "verbose", - Usage: "Prints all the output to the console.", - Value: false, - Destination: &verbose, + Name: "verbose", + Usage: "Prints all the output to the console.", + Value: false, }, }, - Action: func(cCtx *cli.Context) error { - args := []string{"test", "./tests", "-test.v=test2json"} + Action: func(cctx *cli.Context) error { + env := os.Environ() + verbose := cctx.Bool("verbose") + specs := cctx.String("specs") + + // Handle Gateway Endpoint URL + gatewayURL := cctx.String("gateway-url") + if gatewayURL != "" { + envGwURL := fmt.Sprintf("GATEWAY_URL=%s", gatewayURL) + if verbose { + fmt.Println(envGwURL) + } + env = append(env, envGwURL) + } else { + return cli.Exit("⚠️ GATEWAY_URL (or --gateway-url) with the endpoint to receive HTTP requests has to be set", 2) + } + + // Handle Subdomain URL + subdomainGatewayURL := cctx.String("subdomain-url") + if subdomainGatewayURL != "" { + // If set, pass to `go test` via env + envSubdomainGwURL := fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL) + if verbose { + fmt.Println(envSubdomainGwURL) + } + env = append(env, envSubdomainGwURL) + } else if isSubdomainPresetEnabled(specs) { + // If not set, check if `specs` is not set to explicitly disable it, + // provide user with a meaningful error + return cli.Exit("⚠️ SUBDOMAIN_GATEWAY_URL (or --subdomain-url) must be set when 'subdomain-gateway' tests are enabled. Set the URL and try again, or disable related tests by passing --specs -subdomain-gateway", 2) + } + // Set other parameters + args := []string{"test", "./tests", "-test.v=test2json"} if specs != "" { args = append(args, fmt.Sprintf("-specs=%s", specs)) } - ldFlag := fmt.Sprintf("-ldflags=-X github.com/ipfs/gateway-conformance/tooling.Version=%s -X github.com/ipfs/gateway-conformance/tooling.JobURL=%s", tooling.Version, jobURL) + ldFlag := fmt.Sprintf("-ldflags=-X github.com/ipfs/gateway-conformance/tooling.Version=%s -X github.com/ipfs/gateway-conformance/tooling.JobURL=%s", tooling.Version, cctx.String("job-url")) args = append(args, ldFlag) - args = append(args, cCtx.Args().Slice()...) + args = append(args, cctx.Args().Slice()...) fmt.Println("go " + strings.Join(args, " ")) + // Execute tests against URLs output := &bytes.Buffer{} cmd := exec.Command("go", args...) cmd.Dir = tooling.Home() - cmd.Env = append(os.Environ(), fmt.Sprintf("GATEWAY_URL=%s", gatewayURL)) - - if subdomainGatewayURL != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL)) - } - + cmd.Env = env cmd.Stdout = out{ Writer: output, Filter: func(line string) bool { @@ -190,9 +206,11 @@ func main() { fmt.Println() } + jsonOutput := cctx.String("json-output") if jsonOutput != "" { json := &bytes.Buffer{} cmd = exec.Command("go", "tool", "test2json", "-p", "Gateway Tests", "-t") + cmd.Env = env cmd.Stdin = output cmd.Stdout = json cmd.Stderr = os.Stderr @@ -230,20 +248,35 @@ func main() { Usage: "Extract gateway testing fixtures that are used by the conformance test suite", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "directory", - Aliases: []string{"dir"}, - Usage: "The directory to extract the fixtures to", - Required: true, - Destination: &directory, + Name: "directory", + Aliases: []string{"dir"}, + Usage: "The directory to extract the fixtures to", + Required: true, }, &cli.BoolFlag{ - Name: "merged", - Usage: "Merge the fixtures into a single CAR file", - Value: false, - Destination: &merged, + Name: "merged", + Usage: "Merge the CAR fixtures into a single CAR file", + Value: false, + }, + &cli.BoolFlag{ + Name: "car", + Usage: "Include CAR fixtures", + Value: true, + }, + &cli.BoolFlag{ + Name: "ipns", + Usage: "Include IPNS Record fixtures", + Value: true, + }, + &cli.BoolFlag{ + Name: "dnslink", + Usage: "Include DNSLink fixtures", + Value: true, }, }, - Action: func(cCtx *cli.Context) error { + Action: func(cctx *cli.Context) error { + directory := cctx.String("directory") + err := os.MkdirAll(directory, 0755) if err != nil { return err @@ -254,46 +287,48 @@ func main() { return err } - merged := cCtx.Bool("merged") - if merged { - err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car")) - if err != nil { - return err - } - - err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) - if err != nil { - return err - } - - // TODO: when https://github.com/ipfs/specs/issues/369 has been completed, - // merge the IPNS records into a car file. + // IPNS Records + if cctx.Bool("ipns") { err = copyFiles(fxs.IPNSRecords, directory) if err != nil { return err } - } else { - err = copyFiles(fxs.CarFiles, directory) - if err != nil { - return err - } + } + // DNSLink fixtures as YAML, JSON, and IPNS_NS_MAP env variable + if cctx.Bool("dnslink") { err = copyFiles(fxs.ConfigFiles, directory) if err != nil { return err } - - err = copyFiles(fxs.IPNSRecords, directory) + err = dnslink.MergeJSON(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) if err != nil { return err } - - err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) + err = dnslink.MergeNsMapEnv(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.IPFS_NS_MAP")) if err != nil { return err } } + if cctx.Bool("car") { + if cctx.Bool("merged") { + // All .car fixtures merged into a single .car file + err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car")) + if err != nil { + return err + } + // TODO: when https://github.com/ipfs/specs/issues/369 has been completed, + // implement merge support to include the IPNS records in the car file. + } else { + // Copy .car fixtures as -is + err = copyFiles(fxs.CarFiles, directory) + if err != nil { + return err + } + } + } + return nil }, }, @@ -304,3 +339,47 @@ func main() { log.Fatal(err) } } + +func getAvailableSpecPresets() []string { + var presets []string + for _, preset := range specPresets.All() { + var p string + if preset.IsEnabled() && !preset.IsMature() { + p += "+" + } + if !preset.IsEnabled() { + p += "-" + } + p += preset.Name() + presets = append(presets, p) + } + return presets +} + +func isSubdomainPresetEnabled(specs string) bool { + isEnabledByDefault := specPresets.SubdomainGateway.IsEnabled() + if specs == "" && isEnabledByDefault { + return true + } + subdomainSpec := specPresets.SubdomainGateway.Name() + userProvidedSpecsList := strings.Split(specs, ",") + manualList := false // did user set --specs to at least one without the -/+ prefix + for _, s := range userProvidedSpecsList { + // Return early if user-provided spec entry is one that controls subdomain gateway tests + if s == "-"+subdomainSpec { + return false + } + if strings.HasSuffix(s, subdomainSpec) { + return true // at this point it can be + or manual entry + } + // Subdomain gateway preset is implicitly enabled, but it gets disabled + // if user explicitly enabled other one (without - or + prefix) + if !strings.HasPrefix(s, "-") && !strings.HasPrefix(s, "+") { + manualList = true + } + } + // at this point, if the list was manual, and we did not return yet, + // subdomain preset is enabled only if user-provided list had no explicit entries + // (empty or only with -/+ entries) + return !manualList +} diff --git a/cmd/gateway-conformance/main_test.go b/cmd/gateway-conformance/main_test.go new file mode 100644 index 000000000..7c11e42c2 --- /dev/null +++ b/cmd/gateway-conformance/main_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "testing" +) + +// Test cases for isSubdomainPresetEnabled function +func TestIsSubdomainPresetEnabled(t *testing.T) { + tests := []struct { + specs string + expectedEnabled bool + description string + }{ + // Test case 1: Empty specs and subdomain preset is enabled by default + { + specs: "", + expectedEnabled: true, + description: "Empty specs and subdomain preset is enabled by default", + }, + // Test case 2: User provides "-subdomain", should be disabled explicitly + { + specs: "-subdomain-gateway", + expectedEnabled: false, + description: "User provides '-subdomain', should be disabled explicitly", + }, + // Test case 3: User provides "+subdomain", should be enabled explicitly + { + specs: "+subdomain-gateway", + expectedEnabled: true, + description: "User provides '+subdomain', should be enabled explicitly", + }, + // Test case 4: User provides "+other", should not affect subdomain preset + { + specs: "+proxy-gateway", + expectedEnabled: true, + description: "User provides '+proxy-gateway', should not affect subdomain-gateway preset default", + }, + // Test case 5: User provides "other", subdomain preset should be enabled by default + { + specs: "path-gateway", + expectedEnabled: false, + description: "User provides 'path-gateway', subdomain preset should be disabled due to explicit (manual) list", + }, + // Test case 6: User provides "-other,+subdomain", should be enabled due to +subdomain + { + specs: "-path-gateway,+subdomain-gateway", + expectedEnabled: true, + description: "User provides '-path-gateway,+subdomain-gateway', should be enabled due to +subdomain-gateway", + }, + // Test case 7: User provides "+other,-subdomain", should be disabled due to -subdomain + { + specs: "+path-gateway,-subdomain-gateway", + expectedEnabled: false, + description: "User provides '+path-gateway,-subdomain-gateway', should be disabled due to -subdomain-gateway", + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + actualEnabled := isSubdomainPresetEnabled(test.specs) + if actualEnabled != test.expectedEnabled { + t.Errorf("Expected isSubdomainPresetEnabled(%q) to be %v, but got %v", + test.specs, test.expectedEnabled, actualEnabled) + } + }) + } +} diff --git a/docs/commands.md b/docs/commands.md index 58f60c555..3ba794570 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -30,8 +30,8 @@ The `test` command is the main command of the tool. It is used to test a given I | Input | Availability | Description | Default | |---|---|---|---| -| gateway-url | Both | The URL of the IPFS Gateway implementation to be tested. | http://localhost:8080 | -| subdomain-url | Both | The URL to be used in Subdomain feature tests based on Host HTTP header. | http://example.com | +| gateway-url | Both | The URL of the IPFS Gateway implementation to be tested. | http://127.0.0.1:8080 | +| subdomain-url | Both | The URL to be used in Subdomain feature tests based on Host HTTP header. | http://localhost:8080 | | json | Both | The path where the JSON test report should be generated. | `./report.json` | | xml | GitHub Action | The path where the JUnit XML test report should be generated. | `./report.xml` | | html | GitHub Action | The path where the one-page HTML test report should be generated. | `./report.html` | @@ -61,8 +61,8 @@ A few examples: | Use Case | gateway-url | subdomain-url | |----------|-------------|---------------| -| CI & Dev | http://127.0.0.1:8080 | http://example.com | -| Production | https://dweb.link | https://dweb.link | +| CI & Dev | `http://127.0.0.1:8080` | `http://localhost:8080` | +| Production | `https://ipfs.io` | `https://dweb.link` | #### Usage @@ -70,9 +70,10 @@ A few examples: ```yaml - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@v1 + uses: ipfs/gateway-conformance/.github/actions/test@v6 # TODO make sure to use latest with: - gateway-url: http://localhost:8080 + gateway-url: http://127.0.0.1:8080 + subdomain-url: http://localhost:8080 specs: +subdomain-gateway,-path-gateway json: report.json xml: report.xml @@ -84,7 +85,7 @@ A few examples: ##### Docker ```bash -docker run --network host -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance test --gateway-url http://localhost:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 30m +docker run --network host -v "${PWD}:/workspace" -w "/workspace" ghcr.io/ipfs/gateway-conformance test --gateway-url http://127.0.0.1:8080 --subdomain-url http://localhost:8080 --json report.json --specs +subdomain-gateway,-path-gateway -- -timeout 30m ``` ### extract-fixtures diff --git a/docs/examples.md b/docs/examples.md index 953cdbe66..f2420e39f 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -46,9 +46,32 @@ gateway-conformance test --specs trustless-gateway,-trustless-gateway-ipns Tests are skipped using Go's standard syntax: ```bash -gateway-conformance test -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' +gateway-conformance test -- -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' ``` +It supports regex, allowing for skipping specific group or individual test case: + +```bash +gateway-conformance test -- -skip 'TestGatewayCar/*' +``` + +```bash +./gateway-conformance test --verbose -- -skip '.*/.*TODO.*' +``` + +### Run only a specific test + +Same syntax as for skipping: + +```bash +gateway-conformance test -- -run 'TestGatewayCar/*' +``` + +```bash +./gateway-conformance test --verbose -- -run '.*/.*TODO.*' +``` + + ### Extracting the test fixtures ```bash diff --git a/fixtures/dir_listing/dnslink.yml b/fixtures/dir_listing/dnslink.yml index 7e8be2607..1d9f51829 100644 --- a/fixtures/dir_listing/dnslink.yml +++ b/fixtures/dir_listing/dnslink.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=../fixture.schema.json dnslinks: - website: - subdomain: website + dir-listing-website: + domain: dnslink-dir-listing.example.org path: /ipfs/bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/ diff --git a/fixtures/redirects_file/dnslink.yml b/fixtures/redirects_file/dnslink.yml index f5fb2cde9..0bafb8867 100644 --- a/fixtures/redirects_file/dnslink.yml +++ b/fixtures/redirects_file/dnslink.yml @@ -1,10 +1,10 @@ # yaml-language-server: $schema=fixture.schema.json dnslinks: - custom-dnslink: - subdomain: dnslink-enabled-on-fqdn + redirects-examples: + domain: dnslink-redirects-examples.example.org # cid of ./redirects.car:/examples/ path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj - dnslink-spa: - subdomain: dnslink-enabled-with-spa + redirects-spa: + domain: dnslink-enabled-with-spa.example.org # cid of ./redirects-spa.car path: /ipfs/bafybeib5lboymwd6p2eo4qb2lkueaine577flvsjjeuevmp2nlio72xv5q diff --git a/fixtures/subdomain_gateway/dnslink.yml b/fixtures/subdomain_gateway/dnslink.yml index 54c793b72..5e165dd0e 100644 --- a/fixtures/subdomain_gateway/dnslink.yml +++ b/fixtures/subdomain_gateway/dnslink.yml @@ -1,10 +1,10 @@ # yaml-language-server: $schema=../fixture.schema.json dnslinks: wikipedia: - domain: dnslink-subdomain-gw-test.example.org + domain: en.wikipedia-on-ipfs.example.org # Wikipedia CID path: /ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze test: - domain: dnslink-test.example.com + domain: dnslink-test.example.org # CIDv1=$(echo "hello" | ipfs add --cid-version 1 -Q) - path: /ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am \ No newline at end of file + path: /ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am diff --git a/kubo-config.example.sh b/kubo-config.example.sh index 48f17a82f..87e30da6d 100755 --- a/kubo-config.example.sh +++ b/kubo-config.example.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash FIXTURES_PATH=${1:-$(pwd)} @@ -6,17 +6,16 @@ ipfs config --json Gateway.PublicGateways '{ "example.com": { "UseSubdomains": true, "InlineDNSLink": true, - "Paths": ["/ipfs", "/ipns", "/api"] + "Paths": ["/ipfs", "/ipns"] }, "localhost": { "UseSubdomains": true, "InlineDNSLink": true, - "Paths": ["/ipfs", "/ipns", "/api"] + "Paths": ["/ipfs", "/ipns"] } }' -export IPFS_NS_MAP=$(cat "${FIXTURES_PATH}/dnslinks.json" | jq -r '.subdomains | to_entries | map("\(.key).example.com:\(.value)") | join(",")') -export IPFS_NS_MAP="$(cat "${FIXTURES_PATH}/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" +export IPFS_NS_MAP="$(cat "${FIXTURES_PATH}/dnslinks.IPFS_NS_MAP")" echo "Set the following IPFS_NS_MAP before starting the kubo daemon:" echo "IPFS_NS_MAP=${IPFS_NS_MAP}" diff --git a/tests/dnslink_gateway_test.go b/tests/dnslink_gateway_test.go index 110ec21c3..adbc9c023 100644 --- a/tests/dnslink_gateway_test.go +++ b/tests/dnslink_gateway_test.go @@ -1,17 +1,14 @@ package tests import ( - "net/url" "testing" "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/dnslink" - "github.com/ipfs/gateway-conformance/tooling/helpers" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" - "github.com/ipfs/gateway-conformance/tooling/tmpl" ) func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) { @@ -21,24 +18,14 @@ func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) { file := fixture.MustGetNode("ą", "ę", "file-źł.txt") dnsLinks := dnslink.MustOpenDNSLink("dir_listing/dnslink.yml") - dnsLink := dnsLinks.MustGet("website") + dnsLink := dnsLinks.MustGet("dir-listing-website") - gatewayURL := SubdomainGatewayURL - - tests := SugarTests{} - - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } - - dnsLinkHostname := tmpl.Fmt("{{dnslink}}.{{host}}", dnsLink, u.Host) - - tests = append(tests, SugarTests{ + tests := SugarTests{ { Name: "Backlink on root CID should be hidden (TODO: cleanup Kubo-specifics)", Request: Request(). - URL(`{{scheme}}://{{hostname}}/`, u.Scheme, dnsLinkHostname), + Path("/"). + Header("Host", dnsLink), Response: Expect(). Body( And( @@ -50,7 +37,8 @@ func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) { { Name: "Redirect dir listing to URL with trailing slash", Request: Request(). - URL(`{{scheme}}://{{hostname}}/ą/ę`, u.Scheme, dnsLinkHostname), + Path("/ą/ę"). + Header("Host", dnsLink), Response: Expect(). Status(301). Headers( @@ -60,7 +48,8 @@ func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) { { Name: "Regular dir listing (TODO: cleanup Kubo-specifics)", Request: Request(). - URL(`{{scheme}}://{{hostname}}/ą/ę/`, u.Scheme, dnsLinkHostname), + Path("/ą/ę/"). + Header("Host", dnsLink), Response: Expect(). Headers( Header("Etag").Contains(`"DirIndex-`), @@ -76,13 +65,13 @@ func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) { And( Contains("Index of"), Contains(`..`), - Contains(`/ipns/{{hostname}}/ą/ę`, dnsLinkHostname), + Contains(`/ipns/{{hostname}}/ą/ę`, dnsLink), Contains(`file-źł.txt`), Contains(``, file.Cid()), ), ), }, - }...) + } - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.DNSLinkGateway) + RunWithSpecs(t, tests, specs.DNSLinkGateway) } diff --git a/tests/metadata_test.go b/tests/metadata_test.go index 738d44aef..b76c55e6e 100644 --- a/tests/metadata_test.go +++ b/tests/metadata_test.go @@ -12,8 +12,8 @@ func logGatewayURL(t *testing.T) { GatewayURL string `json:"gateway_url"` SubdomainGatewayURL string `json:"subdomain_gateway_url"` }{ - GatewayURL: test.GatewayURL, - SubdomainGatewayURL: test.SubdomainGatewayURL, + GatewayURL: test.GatewayURL().String(), + SubdomainGatewayURL: test.SubdomainGatewayURL().String(), }) } diff --git a/tests/redirects_file_test.go b/tests/redirects_file_test.go index c9f7eca3e..25c9ee3d1 100644 --- a/tests/redirects_file_test.go +++ b/tests/redirects_file_test.go @@ -1,14 +1,12 @@ package tests import ( - "net/url" "testing" "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/dnslink" - "github.com/ipfs/gateway-conformance/tooling/helpers" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" . "github.com/ipfs/gateway-conformance/tooling/tmpl" @@ -26,236 +24,234 @@ func TestRedirectsFileSupport(t *testing.T) { tests := SugarTests{} - // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) - gatewayURLs := []string{ - SubdomainGatewayURL, - SubdomainLocalhostGatewayURL, - } - - for _, gatewayURL := range gatewayURLs { - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } + // Redirects require origin isolation (https://specs.ipfs.tech/http-gateways/web-redirects-file/) + // This means we only run these tests against origins explicitly passed via --subdomain-url + u := SubdomainGatewayURL() - redirectDirBaseURL := Fmt("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, redirectDirCID, u.Host) + dirCIDInSubdomain := Fmt("{{cid}}.ipfs.{{host}}", redirectDirCID, u.Host) - tests = append(tests, SugarTests{ - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", - Request: Request(). - Header("Host", u.Host). - URL("{{url}}/redirect-one", redirectDirBaseURL), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/one.html"), - ), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file", - Request: Request(). - URL("{{url}}/301-redirect-one", redirectDirBaseURL), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/one.html"), - ), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file", - Request: Request(). - URL("{{url}}/302-redirect-two", redirectDirBaseURL), - Response: Expect(). - Status(302). - Headers( - Header("Location").Equals("/two.html"), - ), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file", - Request: Request(). - URL("{{url}}/200-index", redirectDirBaseURL), - Response: Expect(). - Status(200). - Body(Contains("my index")), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file", - Request: Request(). - URL("{{url}}/posts/2022/01/01/hello-world", redirectDirBaseURL), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/articles/2022/01/01/hello-world"), - ), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file", - Request: Request(). - URL("{{url}}/splat/one.html", redirectDirBaseURL), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/redirected-splat/one.html"), - ), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file", - Request: Request(). - URL("{{url}}/not-found/has-no-redirects-entry", redirectDirBaseURL), - Response: Expect(). - Status(404). - Headers( - Header("Cache-Control").Equals("public, max-age=29030400, immutable"), - Header("Etag").Equals(`"{{etag}}"`, custom404.Cid().String()), - ). - Body(Contains(custom404.ReadFile())), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file", - Request: Request(). - URL("{{url}}/gone/has-no-redirects-entry", redirectDirBaseURL), - Response: Expect(). - Status(410). - Headers( - Header("Cache-Control").Equals("public, max-age=29030400, immutable"), - Header("Etag").Equals(`"{{etag}}"`, custom410.Cid().String()), - ). - Body(Contains(custom410.ReadFile())), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file", - Request: Request(). - URL("{{url}}/unavail/has-no-redirects-entry", redirectDirBaseURL), - Response: Expect(). - Status(451). - Headers( - Header("Cache-Control").Equals("public, max-age=29030400, immutable"), - Header("Etag").Equals(`"{{etag}}"`, custom451.Cid().String()), - ). - Body(Contains(custom451.ReadFile())), - }, - { - Name: "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file", - Request: Request(). - URL("{{url}}/catch-all", redirectDirBaseURL), - Response: Expect(). - Status(200). - Body(Contains("my index")), - }, - }...) + tests = append(tests, SugarTests{ + { + Name: "request for {cid}.ipfs.example.com/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/redirect-one"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + { + Name: "request for {cid}.ipfs.example.com/301-redirect-one redirects with 301, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/301-redirect-one"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + { + Name: "request for {cid}.ipfs.example.com/302-redirect-two redirects with 302, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/302-redirect-two"), + Response: Expect(). + Status(302). + Headers( + Header("Location").Equals("/two.html"), + ), + }, + { + Name: "request for {cid}.ipfs.example.com/200-index returns 200, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/200-index"), + Response: Expect(). + Status(200). + Body(Contains("my index")), + }, + { + Name: "request for {cid}.ipfs.example.com/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/posts/2022/01/01/hello-world"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/articles/2022/01/01/hello-world"), + ), + }, + { + Name: "request for {cid}.ipfs.example.com/splat/one.html redirects with 301 and splat placeholder, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/splat/one.html"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/redirected-splat/one.html"), + ), + }, + { + Name: "request for {cid}.ipfs.example.com/not-found/has-no-redirects-entry returns custom 404, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/not-found/has-no-redirects-entry"), + Response: Expect(). + Status(404). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals(`"{{etag}}"`, custom404.Cid().String()), + ). + Body(Contains(custom404.ReadFile())), + }, + { + Name: "request for {cid}.ipfs.example.com/gone/has-no-redirects-entry returns custom 410, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/gone/has-no-redirects-entry"), + Response: Expect(). + Status(410). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals(`"{{etag}}"`, custom410.Cid().String()), + ). + Body(Contains(custom410.ReadFile())), + }, + { + Name: "request for {cid}.ipfs.example.com/unavail/has-no-redirects-entry returns custom 451, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/unavail/has-no-redirects-entry"), + Response: Expect(). + Status(451). + Headers( + Header("Cache-Control").Equals("public, max-age=29030400, immutable"), + Header("Etag").Equals(`"{{etag}}"`, custom451.Cid().String()), + ). + Body(Contains(custom451.ReadFile())), + }, + { + Name: "request for {cid}.ipfs.example.com/catch-all returns 200, per _redirects file", + Request: Request(). + Header("Host", dirCIDInSubdomain). + Path("/catch-all"), + Response: Expect(). + Status(200). + Body(Contains("my index")), + }, + }...) - // # Invalid file, containing forced redirect - invalidRedirectsDirCID := fixture.MustGetNode("forced").Base32Cid() - invalidDirBaseURL := Fmt("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, invalidRedirectsDirCID, u.Host) + // # Invalid file, containing forced redirect + invalidRedirectsDirCID := fixture.MustGetNode("forced").Base32Cid() + invalidDirSubdomain := Fmt("{{cid}}.ipfs.{{host}}", invalidRedirectsDirCID, u.Host) - tooLargeRedirectsDirCID := fixture.MustGetNode("too-large").Base32Cid() - tooLargeDirBaseURL := Fmt("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, tooLargeRedirectsDirCID, u.Host) + tooLargeRedirectsDirCID := fixture.MustGetNode("too-large").Base32Cid() + tooLargeDirSubdomain := Fmt("{{cid}}.ipfs.{{host}}", tooLargeRedirectsDirCID, u.Host) - tests = append(tests, SugarTests{ - { - Name: "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file", - Hint: `if accessing a path that doesn't exist, read _redirects and fail parsing, and return error`, - Request: Request(). - URL("{{url}}/not-found", invalidDirBaseURL), - Response: Expect(). - Status(500). - Body( - And( - Contains("could not parse _redirects:"), - Contains(`forced redirects (or "shadowing") are not supported`), - ), - ).Spec("https://specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"), - Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling", - }, - { - Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file", - Hint: `if accessing a path that doesn't exist and _redirects file is too large, return error`, - Request: Request(). - URL("{{url}}/not-found", tooLargeDirBaseURL), - Response: Expect(). - Status(500). - Body( - And( - Contains("could not parse _redirects:"), - Contains("redirects file size cannot exceed"), - ), + tests = append(tests, SugarTests{ + { + Name: "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file", + Hint: `if accessing a path that doesn't exist, read _redirects and fail parsing, and return error`, + Request: Request(). + Header("Host", invalidDirSubdomain). + Path("/not-found"), + Response: Expect(). + Status(500). + Body( + And( + Contains("could not parse _redirects:"), + Contains(`forced redirects (or "shadowing") are not supported`), ), - Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size", - }, - }...) + ).Spec("https://specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"), + Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling", + }, + { + Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file", + Hint: `if accessing a path that doesn't exist and _redirects file is too large, return error`, + Request: Request(). + Header("Host", tooLargeDirSubdomain). + Path("/not-found"), + Response: Expect(). + Status(500). + Body( + And( + Contains("could not parse _redirects:"), + Contains("redirects file size cannot exceed"), + ), + ), + Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size", + }, + }...) - // # With CRLF line terminator - newlineRedirectsDirCID := fixture.MustGetNode("newlines").Base32Cid() - newlineBaseURL := Fmt("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, newlineRedirectsDirCID, u.Host) + // # With CRLF line terminator + newlineRedirectsDirCID := fixture.MustGetNode("newlines").Base32Cid() + newlineHost := Fmt("{{cid}}.ipfs.{{host}}", newlineRedirectsDirCID, u.Host) - // # Good codes - goodRedirectDirCID := fixture.MustGetNode("good-codes").Base32Cid() - goodRedirectDirBaseURL := Fmt("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, goodRedirectDirCID, u.Host) + // # Good codes + goodRedirectDirCID := fixture.MustGetNode("good-codes").Base32Cid() + goodRedirectDirHost := Fmt("{{cid}}.ipfs.{{host}}", goodRedirectDirCID, u.Host) - // # Bad codes - badRedirectDirCID := fixture.MustGetNode("bad-codes").Base32Cid() - badRedirectDirBaseURL := Fmt("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, badRedirectDirCID, u.Host) + // # Bad codes + badRedirectDirCID := fixture.MustGetNode("bad-codes").Base32Cid() + badRedirectDirHost := Fmt("{{cid}}.ipfs.{{host}}", badRedirectDirCID, u.Host) - tests = append(tests, SugarTests{ - { - Name: "newline: request for $NEWLINE_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", - Request: Request(). - URL("{{url}}/redirect-one", newlineBaseURL), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/one.html"), - ), - }, - { - Name: "good codes: request for $GOOD_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", - Request: Request(). - URL("{{url}}/a301", goodRedirectDirBaseURL), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/b301"), - ), - }, - { - Name: "bad codes: request for $BAD_REDIRECTS_DIR_HOSTNAME/found.html doesn't return error about bad code", - Request: Request(). - URL("{{url}}/found.html", badRedirectDirBaseURL), - Response: Expect(). - Status(200). - Body( - And( - Contains("my found"), - Not(Contains("unsupported redirect status")), - ), + tests = append(tests, SugarTests{ + { + Name: "newline: request for $NEWLINE_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + Header("Host", newlineHost). + Path("/redirect-one"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/one.html"), + ), + }, + { + Name: "good codes: request for $GOOD_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + Header("Host", goodRedirectDirHost). + Path("/a301"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/b301"), + ), + }, + { + Name: "bad codes: request for $BAD_REDIRECTS_DIR_HOSTNAME/found.html doesn't return error about bad code", + Request: Request(). + Header("Host", badRedirectDirHost). + Path("/found.html"), + Response: Expect(). + Status(200). + Body( + And( + Contains("my found"), + Not(Contains("unsupported redirect status")), ), - }, - }...) - } + ), + }, + }...) - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.SubdomainGatewayIPFS, specs.RedirectsFile) + RunWithSpecs(t, tests, specs.SubdomainGatewayIPFS, specs.RedirectsFile) } func TestRedirectsFileSupportWithDNSLink(t *testing.T) { tooling.LogTestGroup(t, GroupDNSLink) dnsLinks := dnslink.MustOpenDNSLink("redirects_file/dnslink.yml") - dnsLink := dnsLinks.MustGet("custom-dnslink") - - gatewayURL := SubdomainGatewayURL - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } - - dnsLinkBaseUrl := Fmt("{{scheme}}://{{dnslink}}.{{host}}", u.Scheme, dnsLink, u.Host) + dnsLink := dnsLinks.MustGet("redirects-examples") tests := SugarTests{ { - Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file", + Name: "request for //{dnslink} redirects with default of 301, per _redirects file", Request: Request(). - URL("{{url}}/redirect-one", dnsLinkBaseUrl), + Header("Host", dnsLink). + Path("/redirect-one"), Response: Expect(). Status(301). Headers( @@ -263,10 +259,11 @@ func TestRedirectsFileSupportWithDNSLink(t *testing.T) { ), }, { - Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file", + Name: "request for //{dnslink}/en/has-no-redirects-entry returns custom 404, per _redirects file", Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`, Request: Request(). - URL("{{url}}/not-found/has-no-redirects-entry", dnsLinkBaseUrl), + Header("Host", dnsLink). + Path("/not-found/has-no-redirects-entry"), Response: Expect(). Status(404). Headers( @@ -281,31 +278,66 @@ func TestRedirectsFileSupportWithDNSLink(t *testing.T) { }, } - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.DNSLinkGateway, specs.RedirectsFile) + // TODO: + RunWithSpecs(t, tests, specs.DNSLinkGateway, specs.RedirectsFile) } func TestRedirectsFileWithIfNoneMatchHeader(t *testing.T) { fixture := car.MustOpenUnixfsCar("redirects_file/redirects-spa.car") dnsLinks := dnslink.MustOpenDNSLink("redirects_file/dnslink.yml") - dnsLink := dnsLinks.MustGet("dnslink-spa") + dnsLink := dnsLinks.MustGet("redirects-spa") - gatewayURL := SubdomainGatewayURL - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } + u := SubdomainGatewayURL() - pageURL := Fmt("{{scheme}}://{{dnslink}}.{{host}}/missing-page", u.Scheme, dnsLink, u.Host) + dnslinkAtSubdomainGw := Fmt("{{dnslink}}.ipns.{{host}}", dnslink.InlineDNS(dnsLink), u.Host) var etag string - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, SugarTests{ + RunWithSpecs(t, SugarTests{ + { + Name: "request for //{dnslink}.ipns.{subdomain-gateway}/missing-page returns body of index.html as per _redirects", + Request: Request(). + Path("/missing-page"). + Headers( + Header("Host", dnslinkAtSubdomainGw), + Header("Accept", "text/html"), + ), + Response: Expect(). + Status(200). + Headers( + Header("Etag"). + Checks(func(v string) bool { + etag = v + return v != "" + }), + ). + Body(fixture.MustGetRawData("index.html")), + }, + }, specs.SubdomainGatewayIPNS, specs.RedirectsFile) + + RunWithSpecs(t, SugarTests{ + { + Name: "request for //{dnslink}.ipns.{subdomain-gateway}/missing-page with If-None-Match returns 304", + Request: Request(). + Path("/missing-page"). + Headers( + Header("Host", dnslinkAtSubdomainGw), + Header("Accept", "text/html"), + Header("If-None-Match", etag), + ), + Response: Expect(). + Status(304), + }, + }, specs.SubdomainGatewayIPNS, specs.RedirectsFile) + + RunWithSpecs(t, SugarTests{ { - Name: "request for $DNSLINK_FQDN/missing-page returns body of index.html as per _redirects", + Name: "request for //{dnslink}/missing-page returns body of index.html as per _redirects", Request: Request(). - URL(pageURL). + Path("/missing-page"). Headers( + Header("Host", dnsLink), Header("Accept", "text/html"), ), Response: Expect(). @@ -319,19 +351,20 @@ func TestRedirectsFileWithIfNoneMatchHeader(t *testing.T) { ). Body(fixture.MustGetRawData("index.html")), }, - }), specs.DNSLinkGateway, specs.RedirectsFile) + }, specs.DNSLinkGateway, specs.RedirectsFile) - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, SugarTests{ + RunWithSpecs(t, SugarTests{ { - Name: "request for $DNSLINK_FQDN/missing-page with If-None-Match returns 301", + Name: "request for //{dnslink}/missing-page with If-None-Match returns 304", Request: Request(). - URL(pageURL). + Path("/missing-page"). Headers( + Header("Host", dnsLink), Header("Accept", "text/html"), Header("If-None-Match", etag), ), Response: Expect(). Status(304), }, - }), specs.DNSLinkGateway, specs.RedirectsFile) + }, specs.DNSLinkGateway, specs.RedirectsFile) } diff --git a/tests/subdomain_gateway_ipfs_test.go b/tests/subdomain_gateway_ipfs_test.go index f0a61c69f..8d86624b6 100644 --- a/tests/subdomain_gateway_ipfs_test.go +++ b/tests/subdomain_gateway_ipfs_test.go @@ -1,15 +1,14 @@ package tests import ( - "net/url" "testing" "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" - "github.com/ipfs/gateway-conformance/tooling/helpers" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" + . "github.com/ipfs/gateway-conformance/tooling/tmpl" ) func TestUnixFSDirectoryListingOnSubdomainGateway(t *testing.T) { @@ -19,92 +18,71 @@ func TestUnixFSDirectoryListingOnSubdomainGateway(t *testing.T) { root := fixture.MustGetNode() file := fixture.MustGetNode("ą", "ę", "file-źł.txt") - // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) - gatewayURLs := []string{ - SubdomainGatewayURL, - SubdomainLocalhostGatewayURL, - } - tests := SugarTests{} - for _, gatewayURL := range gatewayURLs { - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } + // run against origins explicitly passed via --subdomain-url + u := SubdomainGatewayURL() - tests = append(tests, SugarTests{ - { - Name: "backlink on root CID should be hidden (TODO: cleanup Kubo-specifics)", - Request: Request(). - URL( - "{{scheme}}://{{cid}}.ipfs.{{host}}/", - u.Scheme, - root.Cid(), - u.Host, - ), - Response: Expect(). - BodyWithHint("backlink on root CID should be hidden", - And( - Contains("Index of"), - Not(Contains(`..`)), - )), - }, - { - Name: "redirect dir listing to URL with trailing slash", - Request: Request(). - URL( - "{{scheme}}://{{cid}}.ipfs.{{host}}/ą/ę", - u.Scheme, - root.Cid(), - u.Host, - ), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals(`/%c4%85/%c4%99/`), - ), - }, - { - Name: "Regular dir listing HTML (TODO: cleanup Kubo-specifics)", - Request: Request().URL( - "{{scheme}}://{{cid}}.ipfs.{{host}}/ą/ę/", - u.Scheme, - root.Cid(), - u.Host, + tests = append(tests, SugarTests{ + { + Name: "backlink on root CID should be hidden (TODO: cleanup Kubo-specifics)", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", root.Cid(), u.Host)). + Path("/"), + Response: Expect(). + BodyWithHint("backlink on root CID should be hidden", + And( + Contains("Index of"), + Not(Contains(`..`)), + )), + }, + { + Name: "redirect dir listing to URL with trailing slash", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", root.Cid(), u.Host)). + Path("/ą/ę"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals(`/%c4%85/%c4%99/`), ), - Response: Expect(). - Headers( - Header("Etag").Contains(`"DirIndex-`), - ).BodyWithHint(` + }, + { + Name: "Regular dir listing HTML (TODO: cleanup Kubo-specifics)", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", root.Cid(), u.Host)). + Path("/ą/ę/"), + Response: Expect(). + Headers( + Header("Etag").Contains(`"DirIndex-`), + ).BodyWithHint(` - backlink on subdirectory should point at parent directory (TODO: kubo-specific) - breadcrumbs should leverage path-based router mounted on the parent domain (TODO: kubo-specific) - name column should be a link to content root mounted at subdomain origin `, - And( - Contains("Index of"), - Contains( - `..`, - ), - Contains( - `/ipfs/{{cid}}/ą/ę`, - u.Host, // We don't have a subdomain here which prevents issues with normalization and cidv0 - root.Cid(), - ), - Contains( - `file-źł.txt`, - ), - Contains( - ``, - u.Host, // We don't have a subdomain here which prevents issues with normalization and cidv0 - file.Cid(), - ), - )), - }, - }...) - } + And( + Contains("Index of"), + Contains( + `..`, + ), + Contains( + `/ipfs/{{cid}}/ą/ę`, + u.Host, // We don't have a subdomain here which prevents issues with normalization and cidv0 + root.Cid(), + ), + Contains( + `file-źł.txt`, + ), + Contains( + ``, + u.Host, // We don't have a subdomain here which prevents issues with normalization and cidv0 + file.Cid(), + ), + )), + }, + }...) - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.SubdomainGatewayIPFS) + RunWithSpecs(t, tests, specs.SubdomainGatewayIPFS) } func TestGatewaySubdomains(t *testing.T) { @@ -125,242 +103,276 @@ func TestGatewaySubdomains(t *testing.T) { tests := SugarTests{} - // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) - gatewayURLs := []string{ - SubdomainGatewayURL, - SubdomainLocalhostGatewayURL, - } - - for _, gatewayURL := range gatewayURLs { - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } + // run against origins explicitly passed via --subdomain-url + u := SubdomainGatewayURL() - tests = append(tests, SugarTests{ - { - Name: "request for example.com/ipfs/{CIDv1} redirects to subdomain", - Hint: ` - path requests to gateways with subdomain support - should not return payload directly, - but redirect to URL with proper origin isolation + tests = append(tests, SugarTests{ + { + Name: "request for example.com/ipfs/{cid} redirects to {cid}.ipfs.example.com", + Hint: ` + path requests to gateways with subdomain support should not + return payload directly, but redirect to URL with proper + origin isolation `, - Request: Request().URL("{{url}}/ipfs/{{cid}}/", gatewayURL, CIDv1), - Response: Expect(). - Status(301). - Headers( - Header("Location"). - Hint("request for example.com/ipfs/{CIDv1} returns Location HTTP header for subdomain redirect in browsers"). - Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv1, u.Host), - ), - }, - { - Name: "request for example.com/ipfs/{CIDv1}/{filename with percent encoding} redirects to subdomain", - Request: Request().URL("{{url}}/ipfs/{{cid}}/Portugal%252C+España=Peninsula%20Ibérica.txt", gatewayURL, dirWithPercentEncodedFilenameCID), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("{{scheme}}://{{cid}}.ipfs.{{host}}/Portugal%252C+Espa%C3%B1a=Peninsula%20Ib%C3%A9rica.txt", u.Scheme, dirWithPercentEncodedFilenameCID, u.Host), - ), - }, - { - Name: "request for example.com/ipfs/{DirCID} redirects to subdomain", - Hint: ` - path requests to gateways with subdomain support - should not return payload directly, - but redirect to URL with proper origin isolation + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/{{cid}}/", CIDv1), + Response: Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{CIDv1} returns Location HTTP header for subdomain redirect in browsers"). + Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv1, u.Host), + ), + }, + { + Name: "request for example.com/ipfs/{CIDv1}/{filename with percent encoding} redirects to subdomain", + Hint: "the path remainder MUST be preserved", + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/{{cid}}/Portugal%252C+España=Peninsula%20Ibérica.txt", dirWithPercentEncodedFilenameCID), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("{{scheme}}://{{cid}}.ipfs.{{host}}/Portugal%252C+Espa%C3%B1a=Peninsula%20Ib%C3%A9rica.txt", u.Scheme, dirWithPercentEncodedFilenameCID, u.Host), + ), + }, + { + Name: "request for example.com/ipfs/{DirCID}/ redirects to subdomain", + Hint: ` + path requests to gateways with subdomain support should not + return payload directly, but redirect to URL with proper + origin isolation `, - Request: Request().URL("{{url}}/ipfs/{{cid}}/", gatewayURL, DirCID), - Response: Expect(). - Status(301). - Headers( - Header("Location"). - Hint("request for example.com/ipfs/{DirCID} returns Location HTTP header for subdomain redirect in browsers"). - Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, DirCID, u.Host), - ), - }, - { - Name: "request for example.com/ipfs/{CIDv0} redirects to CIDv1 representation in subdomain", - Request: Request().URL("{{url}}/ipfs/{{cid}}/", gatewayURL, CIDv0), - Response: Expect(). - Status(301). - Headers( - Header("Location"). - Hint("request for example.com/ipfs/{CIDv0to1} returns Location HTTP header for subdomain redirect in browsers"). - Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv0to1, u.Host), - ), - }, - { - Name: "request for {CID}.ipfs.example.com should return expected payload", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}", u.Scheme, CIDv1, u.Host), - Response: Expect(). - Status(200). - Body(Contains(CIDVal)), - }, - { - Name: "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404", - Hint: "ensure /ipfs/ namespace is not mounted on subdomain", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/ipfs/{{cid}}", u.Scheme, CIDv1, u.Host), - Response: Expect(). - Status(404), - }, - { - Name: "request for {CID}.ipfs.example.com/ipfs/file.txt should return data from a file in CID content root", - Hint: "ensure requests to /ipfs/* are not blocked, if content root has such subdirectory", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/ipfs/file.txt", u.Scheme, DirCID, u.Host), - Response: Expect(). - Status(200). - Body(Contains("I am a txt file")), - }, - { - Name: "valid file and subdirectory paths in directory listing at {cid}.ipfs.example.com", - Hint: "{CID}.ipfs.example.com/sub/dir (Directory Listing)", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, DirCID, u.Host), - Response: Expect(). - Status(200). - Body(And( - // TODO: implement html expectations - Contains(`hello`), - Contains(`ipfs`), - )), - }, - { - Name: "valid parent directory path in directory listing at {cid}.ipfs.example.com/sub/dir", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/ipfs/ipns/", u.Scheme, DirCID, u.Host), - Response: Expect(). - Status(200). - Body(And( - // TODO: implement html expectations - Contains(`..`), - Contains(`bar`), - )), - }, - { - Name: "request for deep path resource at {cid}.ipfs.example.com/sub/dir/file", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/ipfs/ipns/bar", u.Scheme, DirCID, u.Host), - Response: Expect(). - Status(200). - Body(Contains("text-file-content")), - }, - { - Name: "valid breadcrumb links in the header of directory listing at {cid}.ipfs.example.com/sub/dir (TODO: cleanup Kubo-specifics)", - Hint: ` + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/{{cid}}/", DirCID), + Response: Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{DirCID} returns Location HTTP header for subdomain redirect in browsers"). + Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, DirCID, u.Host), + ), + }, + { + Name: "request for example.com/ipfs/{CIDv0} redirects to {CIDv1}.ipfs.example.com", + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/{{cid}}/", CIDv0), + Response: Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{CIDv0to1} returns Location HTTP header for subdomain redirect in browsers"). + Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv0to1, u.Host), + ), + }, + { + Name: "request for {CID}.ipfs.example.com should return expected payload", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", CIDv1, u.Host)). + Path("/"), + Response: Expect(). + Status(200). + Body(Contains(CIDVal)), + }, + { + Name: "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404", + Hint: "ensure /ipfs/ namespace is not mounted on subdomain", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", CIDv1, u.Host)). + Path("/ipfs/{{cid}}/", CIDv1), + Response: Expect(). + Status(404), + }, + { + Name: "request for {CID}.ipfs.example.com/ipfs/file.txt should return data from a file in CID content root", + Hint: "ensure requests to /ipfs/* are not blocked, if content root has such subdirectory", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", DirCID, u.Host)). + Path("/ipfs/file.txt"), + Response: Expect(). + Status(200). + Body(Contains("I am a txt file")), + }, + { + Name: "valid file and subdirectory paths in directory listing at {cid}.ipfs.example.com", + Hint: "{CID}.ipfs.example.com (Directory Listing)", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", DirCID, u.Host)). + Path("/"), + Response: Expect(). + Status(200). + Body(And( + // TODO: implement html expectations https://github.com/ipfs/gateway-conformance/issues/21 + Contains(`hello`), + Contains(`ipfs`), + )), + }, + { + Name: "valid parent directory path in directory listing at {cid}.ipfs.example.com/sub/dir", + Hint: "{CID}.ipfs.example.com/ipfs/ipns/ (if exists) should produce a valid directory listing", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", DirCID, u.Host)). + Path("/ipfs/ipns/"), + Response: Expect(). + Status(200). + Body(And( + // TODO: implement html expectations https://github.com/ipfs/gateway-conformance/issues/21 + Contains(`..`), + Contains(`bar`), + )), + }, + { + Name: "request for deep path resource at {cid}.ipfs.example.com/sub/dir/file", + Hint: "{CID}.ipfs.example.com/ipfs/ipns/bar (if exists) should return expected file", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", DirCID, u.Host)). + Path("/ipfs/ipns/bar"), + Response: Expect(). + Status(200). + Body(Contains("text-file-content")), + }, + { + Name: "valid breadcrumb links in the header of directory listing at {cid}.ipfs.example.com/sub/dir (TODO: cleanup Kubo-specifics)", + Hint: ` Note 1: we test for sneaky subdir names {cid}.ipfs.example.com/ipfs/ipns/ :^) Note 2: example.com/ipfs/.. present in HTML will be redirected to subdomain, so this is expected behavior `, - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/ipfs/ipns/", u.Scheme, DirCID, u.Host), - Response: Expect(). - Status(200). - Body( - And( - Contains("Index of"), - Contains(`/ipfs/{{cid}}/ipfs/ipns`, - u.Host, DirCID), - ), - ), - }, - { - Name: "request for example.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.example.com", - Hint: "path requests to the root hostname should redirect to a subdomain URL with proper origin isolation", - Request: Request().URL("{{scheme}}://{{host}}/ipfs/{{cid}}/", u.Scheme, u.Host, CIDv1), - Response: Expect(). - Headers( - Header("Location").Equals("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv1, u.Host), - ), - }, - { - Name: "request for example.com/ipfs/{InvalidCID} produces useful error before redirect", - Hint: "error message should include original CID (and it should be case-sensitive, as we can't assume everyone uses base32)", - Request: Request().URL("{{scheme}}://{{host}}/ipfs/QmInvalidCID", u.Scheme, u.Host), - Response: Expect(). - Body(Contains(`invalid path "/ipfs/QmInvalidCID"`)), - }, - - { - Name: "request for example.com/ipfs/{CIDv0} produces redirect to {CIDv1}.ipfs.example.com", - Request: Request().URL("{{scheme}}://{{host}}/ipfs/{{cid}}/", u.Scheme, u.Host, CIDv0), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv0to1, u.Host), - ), - }, - - { - Name: "request for http://example.com/ipfs/{CID} with X-Forwarded-Proto: https produces redirect to HTTPS URL", - Hint: "Support X-Forwarded-Proto", - Request: Request().URL("{{scheme}}://{{host}}/ipfs/{{cid}}/", u.Scheme, u.Host, CIDv1). - Header("X-Forwarded-Proto", "https"), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("https://{{cid}}.ipfs.{{host}}/", CIDv1, u.Host), - ), - }, - { - Name: "request for example.com/ipfs/?uri=ipfs%3A%2F%2F.. produces redirect to /ipfs/.. content path", - Hint: "Support ipfs:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler", - Request: Request().URL("{{scheme}}://{{host}}/ipfs/", u.Scheme, u.Host). - Query( - "uri", "ipfs://{{host}}/wiki/Diego_Maradona.html", CIDWikipedia, - ), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("/ipfs/{{cid}}/wiki/Diego_Maradona.html", CIDWikipedia), - ), - }, - { - Name: "request for a too long CID at example.com/ipfs/{CIDv1} returns human readable error", - Hint: "router should not redirect to hostnames that could fail due to DNS limits", - Request: Request().URL("{{url}}/ipfs/{{cid}}", gatewayURL, CIDv1_TOO_LONG), - Response: Expect(). - Status(400). - Body(Contains("CID incompatible with DNS label length limit of 63")), - }, - { - Name: "request for a too long CID at {CIDv1}.ipfs.example.com returns expected payload", - Hint: "direct request should also fail (provides the same UX as router and avoids confusion)", - Request: Request().URL("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv1_TOO_LONG, u.Host), - Response: Expect(). - Status(400). - Body(Contains("CID incompatible with DNS label length limit of 63")), - }, - // ## ============================================================================ - // ## Test support for X-Forwarded-Host - // ## ============================================================================ - { - Name: "request for http://fake.domain.com/ipfs/{CID} doesn't match the example.com gateway", - Request: Request(). - URL("{{scheme}}://{{domain}}/ipfs/{{cid}}", u.Scheme, "fake.domain.com", CIDv1), - Response: Expect(). - Status(200), - }, - { - Name: "request for http://fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com match the example.com gateway", - Request: Request(). - URL("{{scheme}}://{{domain}}/ipfs/{{cid}}", u.Scheme, "fake.domain.com", CIDv1). - Header("X-Forwarded-Host", u.Host), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv1, u.Host), - ), - }, - { - Name: "request for http://fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com and X-Forwarded-Proto: https match the example.com gateway, redirect with https", - Request: Request(). - URL("{{scheme}}://{{domain}}/ipfs/{{cid}}", u.Scheme, "fake.domain.com", CIDv1). - Header("X-Forwarded-Host", u.Host). - Header("X-Forwarded-Proto", "https"), - Response: Expect(). - Status(301). - Headers( - Header("Location").Equals("https://{{cid}}.ipfs.{{host}}/", CIDv1, u.Host), + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", DirCID, u.Host)). + Path("/ipfs/ipns/"), + Response: Expect(). + Status(200). + Body( + And( + Contains("Index of"), + Contains(`/ipfs/{{cid}}/ipfs/ipns`, + u.Host, DirCID), ), - }, - }...) - } + ), + }, + { + Name: "request for example.com/ipfs/{InvalidCID} produces useful error before redirect", + Hint: "error message should include original CID (and it should be case-sensitive, as we can't assume everyone uses base32)", + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/QmInvalidCID"), + Response: Expect(). + Body(Contains(`invalid path "/ipfs/QmInvalidCID"`)), + }, + { + Name: "request for example.com/ipfs/{CID} with X-Forwarded-Proto: https produces redirect to HTTPS URL", + Hint: "Support X-Forwarded-Proto", + Request: Request(). + Header("Host", u.Host). + Header("X-Forwarded-Proto", "https"). + Path("/ipfs/{{cid}}/", CIDv1), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("https://{{cid}}.ipfs.{{host}}/", CIDv1, u.Host), + ), + }, + { + Name: "request for example.com/ipfs/{CID} with X-Forwarded-Proto: http produces redirect to HTTP URL", + Hint: "Support X-Forwarded-Proto", + Request: Request(). + Header("Host", u.Host). + Header("X-Forwarded-Proto", "http"). + Path("/ipfs/{{cid}}/", CIDv1), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("http://{{cid}}.ipfs.{{host}}/", CIDv1, u.Host), + ), + }, + { + Name: "request for example.com/ipfs/?uri=ipfs%3A%2F%2F.. produces redirect to /ipfs/.. content path", + Hint: "Support ipfs:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler", + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/"). + Query("uri", "ipfs://{{host}}/wiki/Diego_Maradona.html", CIDWikipedia), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("/ipfs/{{cid}}/wiki/Diego_Maradona.html", CIDWikipedia), + ), + }, + { + Name: "request for a too long CID at example.com/ipfs/{CIDv1} returns human readable error", + Hint: "router should not redirect to hostnames that could fail due to DNS limits", + Request: Request(). + Header("Host", u.Host). + Path("/ipfs/{{cid}}/", CIDv1_TOO_LONG), + Response: Expect(). + Status(400). + Body(Contains("CID incompatible with DNS label length limit of 63")), + }, + { + Name: "request for a too long CID at {CIDv1}.ipfs.example.com returns expected payload", + Hint: "direct request should also fail (provides the same UX as router and avoids confusion)", + Request: Request(). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", CIDv1_TOO_LONG, u.Host)). + Path("/"), + Response: Expect(). + Status(400). + Body(Contains("CID incompatible with DNS label length limit of 63")), + }, + // ## ============================================================================ + // ## Test support for X-Forwarded-Host + // ## ============================================================================ + { + Name: "request for fake.domain.com/ipfs/{CID} doesn't match the example.com gateway", + Hint: "when there is no Host match, request is processed as a path gateway", + Request: Request(). + Header("Host", "fake.domain.com"). + Path("/ipfs/{{cid}}", CIDv1), + Response: Expect(). + Status(200), + }, + { + Name: "request for fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com match the example.com gateway", + Hint: "X-Forwarded-Host overrides Host, request should be processed as a subdomain gateway", + Request: Request(). + Header("Host", "fake.domain.com"). + Header("X-Forwarded-Host", u.Host). + Path("/ipfs/{{cid}}", CIDv1), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("{{scheme}}://{{cid}}.ipfs.{{host}}/", u.Scheme, CIDv1, u.Host), + ), + }, + { + Name: "request for fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com and X-Forwarded-Proto: https match the example.com gateway, redirect with https", + Request: Request(). + Header("Host", "fake.domain.com"). + Path("/ipfs/{{cid}}", CIDv1). + Header("X-Forwarded-Host", u.Host). + Header("X-Forwarded-Proto", "https"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("https://{{cid}}.ipfs.{{host}}/", CIDv1, u.Host), + ), + }, + { + Name: "request for fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com and X-Forwarded-Proto: http match the example.com gateway, redirect with http", + Request: Request(). + Header("Host", "fake.domain.com"). + Path("/ipfs/{{cid}}", CIDv1). + Header("X-Forwarded-Host", u.Host). + Header("X-Forwarded-Proto", "http"), + Response: Expect(). + Status(301). + Headers( + Header("Location").Equals("http://{{cid}}.ipfs.{{host}}/", CIDv1, u.Host), + ), + }, + }...) - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.SubdomainGatewayIPFS) + RunWithSpecs(t, tests, specs.SubdomainGatewayIPFS) } diff --git a/tests/subdomain_gateway_ipns_test.go b/tests/subdomain_gateway_ipns_test.go index 58211eacc..997595a93 100644 --- a/tests/subdomain_gateway_ipns_test.go +++ b/tests/subdomain_gateway_ipns_test.go @@ -1,16 +1,15 @@ package tests import ( - "net/url" "testing" "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" "github.com/ipfs/gateway-conformance/tooling/dnslink" - "github.com/ipfs/gateway-conformance/tooling/helpers" "github.com/ipfs/gateway-conformance/tooling/ipns" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" + . "github.com/ipfs/gateway-conformance/tooling/tmpl" "github.com/multiformats/go-multibase" "github.com/multiformats/go-multicodec" ) @@ -26,208 +25,216 @@ func TestGatewaySubdomainAndIPNS(t *testing.T) { car := car.MustOpenUnixfsCar("subdomain_gateway/fixtures.car") payload := string(car.MustGetRawData("hello-CIDv1")) - // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) - gatewayURLs := []string{ - SubdomainGatewayURL, - SubdomainLocalhostGatewayURL, - } - ipnsRecords := []*ipns.IpnsRecord{ rsaFixture, ed25519Fixture, } - for _, gatewayURL := range gatewayURLs { - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } - - for _, record := range ipnsRecords { - tests = append(tests, SugarTests{ - { - Name: "request for /ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain", - Request: Request(). - URL("{{url}}/ipns/{{cid}}", gatewayURL, record.IdV0()), - Response: Expect(). - Status(301). - Headers( - Header("Location"). - Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.IdV1(), u.Host), - ), - }, - { - Name: "request for {CIDv1-libp2p-key}.ipns.{gateway} returns expected payload", - Request: Request(). - URL("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.IdV1(), u.Host), - Response: Expect(). - Status(200). - BodyWithHint("Request for {{cid}}.ipns.{{host}} returns expected payload", payload), - }, - { - Name: "request for {CIDv1-dag-pb}.ipns.{gateway} redirects to CID with libp2p-key multicodec", - Request: Request(). - URL("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.ToCID(multicodec.DagPb, multibase.Base36), u.Host), - Response: Expect(). - Status(301). - Headers( - Header("Location"). - Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.IdV1(), u.Host), - ), - }, - // # *.ipns.example.com - // # ============================================================================ - - // # .ipns.example.com - - // test_hostname_gateway_response_should_contain \ - // "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ - // "${RSA_IPNS_IDv1}.ipns.example.com" \ - // "http://127.0.0.1:$GWAY_PORT" \ - // "$CID_VAL" - - // test_hostname_gateway_response_should_contain \ - // "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ - // "${ED25519_IPNS_IDv1}.ipns.example.com" \ - // "http://127.0.0.1:$GWAY_PORT" \ - // "$CID_VAL" - - // test_hostname_gateway_response_should_contain \ - // "hostname request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ - // "${RSA_IPNS_IDv1_DAGPB}.ipns.example.com" \ - // "http://127.0.0.1:$GWAY_PORT" \ - // "Location: http://${RSA_IPNS_IDv1}.ipns.example.com/" - - // test_hostname_gateway_response_should_contain \ - // "hostname request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ - // "${ED25519_IPNS_IDv1_DAGPB}.ipns.example.com" \ - // "http://127.0.0.1:$GWAY_PORT" \ - // "Location: http://${ED25519_IPNS_IDv1}.ipns.example.com/" - // # disable /ipns for the hostname by not whitelisting it - // ipfs config --json Gateway.PublicGateways '{ - // "example.com": { - // "UseSubdomains": true, - // "Paths": ["/ipfs"] - // } - // }' || exit 1 - // # restart daemon to apply config changes - // test_kill_ipfs_daemon - // test_launch_ipfs_daemon_without_network - - // TODO: what to do with these? - // # refuse requests to Paths that were not explicitly whitelisted for the hostname - // test_hostname_gateway_response_should_contain \ - // "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ - // "${RSA_IPNS_IDv1}.ipns.example.com" \ - // "http://127.0.0.1:$GWAY_PORT" \ - // "404 Not Found" - - // test_hostname_gateway_response_should_contain \ - // "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ - // "${ED25519_IPNS_IDv1}.ipns.example.com" \ - // "http://127.0.0.1:$GWAY_PORT" \ - // "404 Not Found" - - // # refuse requests to Paths that were not explicitly whitelisted for the hostname - // test_hostname_gateway_response_should_contain \ - // "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ - // "example.com" \ - // "http://127.0.0.1:$GWAY_PORT/ipns/$RSA_IPNS_IDv1" \ - // "404 Not Found" - - // test_hostname_gateway_response_should_contain \ - // "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ - // "example.com" \ - // "http://127.0.0.1:$GWAY_PORT/ipns/$ED25519_IPNS_IDv1" \ - // "404 Not Found" - }...) - } + // run against origins passed via --subdomain-url (e.g. http://localhost:port) + u := SubdomainGatewayURL() + for _, record := range ipnsRecords { tests = append(tests, SugarTests{ { - Name: "request for a ED25519 libp2p-key at example.com/ipns/{b58mh} returns Location HTTP header for DNS-safe subdomain redirect in browsers", + Name: "request for /ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain", Request: Request(). - URL("{{url}}/ipns/{{cid}}", gatewayURL, ed25519Fixture.B58MH()), + Header("Host", u.Host). + Path("/ipns/{{id}}", record.IdV0()), Response: Expect(). + Status(301). Headers( Header("Location"). - Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, ed25519Fixture.ToCID(multicodec.Libp2pKey, multibase.Base36), u.Host), + Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.IdV1(), u.Host), ), }, - }...) - - } - - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.SubdomainGatewayIPNS) -} - -func TestSubdomainGatewayDNSLinkInlining(t *testing.T) { - tooling.LogTestGroup(t, GroupSubdomains) - - tests := SugarTests{} - - // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) - gatewayURLs := []string{ - SubdomainGatewayURL, - SubdomainLocalhostGatewayURL, - } - - dnsLinks := dnslink.MustOpenDNSLink("subdomain_gateway/dnslink.yml") - wikipedia := dnsLinks.MustGet("wikipedia") - dnsLinkTest := dnsLinks.MustGet("test") - - for _, gatewayURL := range gatewayURLs { - u, err := url.Parse(gatewayURL) - if err != nil { - t.Fatal(err) - } - - tests = append(tests, SugarTests{ { - Name: "request for /ipns/{fqdn} redirects to DNSLink in subdomain", + Name: "request for /ipns/{CIDv1} redirects to same CIDv1 on subdomain", Request: Request(). - URL("{{url}}/ipns/{{fqdn}}/wiki/", gatewayURL, wikipedia), + Header("Host", u.Host). + Path("/ipns/{{id}}", record.IdV1()), Response: Expect(). + Status(301). Headers( Header("Location"). - Equals("{{scheme}}://{{fqdn}}.ipns.{{host}}/wiki/", u.Scheme, dnslink.InlineDNS(wikipedia), u.Host), + Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.IdV1(), u.Host), ), }, { - Name: "request for {dnslink}.ipns.{gateway} returns expected payload", + Name: "request for {CIDv1-base36-libp2p-key}.ipns.{gateway} returns expected payload", Request: Request(). - URL("{{scheme}}://{{fqdn}}.ipns.{{host}}", u.Scheme, dnsLinkTest, u.Host), + Header("Host", Fmt("{{cid}}.ipns.{{host}}", record.IdV1(), u.Host)). + Path("/"), Response: Expect(). - Body("hello\n"), + Status(200). + BodyWithHint("Request for {{cid}}.ipns.{{host}} returns expected payload", payload), }, { - Name: "request for example.com/ipns/{fqdn} with X-Forwarded-Proto redirects to TLS-safe label in subdomain", - Hint: ` - DNSLink on Public gateway with a single-level wildcard TLS cert - "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 - `, + Name: "request for {CIDv1-dag-pb}.ipns.{gateway} redirects to CID with libp2p-key multicodec", Request: Request(). - Header("X-Forwarded-Proto", "https"). - URL("{{url}}/ipns/{{wikipedia}}/wiki/", gatewayURL, wikipedia), + Header("Host", Fmt("{{cid}}.ipns.{{host}}", record.ToCID(multicodec.DagPb, multibase.Base36), u.Host)). + Path("/"), Response: Expect(). + Status(301). Headers( Header("Location"). - Equals("https://{{inlined}}.ipns.{{host}}/wiki/", dnslink.InlineDNS(wikipedia), u.Host), - ), - }, - { - Name: `request for example.com/ipns/?uri=ipns%3A%2F%2F.. produces redirect to /ipns/.. content path`, - Hint: "Support ipns:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler", - Request: Request(). - // TODO: use Query or future QueryRaw here - URL(`{{url}}/ipns/?uri=ipns%3A%2F%2F{{dnslink}}`, gatewayURL, wikipedia), - Response: Expect(). - Headers( - Header("Location").Equals("/ipns/{{wikipedia}}", wikipedia), + Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, record.IdV1(), u.Host), ), }, + // # *.ipns.example.com + // # ============================================================================ + + // # .ipns.example.com + + // test_hostname_gateway_response_should_contain \ + // "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ + // "${RSA_IPNS_IDv1}.ipns.example.com" \ + // "http://127.0.0.1:$GWAY_PORT" \ + // "$CID_VAL" + + // test_hostname_gateway_response_should_contain \ + // "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ + // "${ED25519_IPNS_IDv1}.ipns.example.com" \ + // "http://127.0.0.1:$GWAY_PORT" \ + // "$CID_VAL" + + // test_hostname_gateway_response_should_contain \ + // "hostname request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ + // "${RSA_IPNS_IDv1_DAGPB}.ipns.example.com" \ + // "http://127.0.0.1:$GWAY_PORT" \ + // "Location: http://${RSA_IPNS_IDv1}.ipns.example.com/" + + // test_hostname_gateway_response_should_contain \ + // "hostname request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ + // "${ED25519_IPNS_IDv1_DAGPB}.ipns.example.com" \ + // "http://127.0.0.1:$GWAY_PORT" \ + // "Location: http://${ED25519_IPNS_IDv1}.ipns.example.com/" + // # disable /ipns for the hostname by not whitelisting it + // ipfs config --json Gateway.PublicGateways '{ + // "example.com": { + // "UseSubdomains": true, + // "Paths": ["/ipfs"] + // } + // }' || exit 1 + // # restart daemon to apply config changes + // test_kill_ipfs_daemon + // test_launch_ipfs_daemon_without_network + + // TODO: what to do with these? + // # refuse requests to Paths that were not explicitly whitelisted for the hostname + // test_hostname_gateway_response_should_contain \ + // "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ + // "${RSA_IPNS_IDv1}.ipns.example.com" \ + // "http://127.0.0.1:$GWAY_PORT" \ + // "404 Not Found" + + // test_hostname_gateway_response_should_contain \ + // "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ + // "${ED25519_IPNS_IDv1}.ipns.example.com" \ + // "http://127.0.0.1:$GWAY_PORT" \ + // "404 Not Found" + + // # refuse requests to Paths that were not explicitly whitelisted for the hostname + // test_hostname_gateway_response_should_contain \ + // "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ + // "example.com" \ + // "http://127.0.0.1:$GWAY_PORT/ipns/$RSA_IPNS_IDv1" \ + // "404 Not Found" + + // test_hostname_gateway_response_should_contain \ + // "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ + // "example.com" \ + // "http://127.0.0.1:$GWAY_PORT/ipns/$ED25519_IPNS_IDv1" \ + // "404 Not Found" }...) } - RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.SubdomainGatewayIPNS) + tests = append(tests, SugarTests{ + { + Name: "request for a ED25519 libp2p-key at example.com/ipns/{b58mh} returns Location HTTP header for DNS-safe subdomain redirect in browsers", + Request: Request(). + Header("Host", u.Host). + Path("/ipns/{{b58mh}}", ed25519Fixture.B58MH()), + Response: Expect(). + Headers( + Header("Location"). + Equals("{{scheme}}://{{cid}}.ipns.{{host}}/", u.Scheme, ed25519Fixture.ToCID(multicodec.Libp2pKey, multibase.Base36), u.Host), + ), + }, + }...) + + RunWithSpecs(t, tests, specs.SubdomainGatewayIPNS) +} + +func TestSubdomainGatewayDNSLinkInlining(t *testing.T) { + tooling.LogTestGroup(t, GroupSubdomains) + + tests := SugarTests{} + + dnsLinks := dnslink.MustOpenDNSLink("subdomain_gateway/dnslink.yml") + wikipedia := dnsLinks.MustGet("wikipedia") + dnsLinkTest := dnsLinks.MustGet("test") + + // run against origins passed via --subdomain-url + u := SubdomainGatewayURL() + + tests = append(tests, SugarTests{ + { + Name: "request for /ipns/{dnslink}/foo/ redirects to {inlined-dnslink}.ipns.example.com", + Hint: "https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header", + Request: Request(). + Header("Host", u.Host). + Path("/ipns/{{dnslink}}/wiki/", wikipedia), + Response: Expect(). + Headers( + Header("Location"). + Equals("{{scheme}}://{{inlined}}.ipns.{{host}}/wiki/", u.Scheme, dnslink.InlineDNS(wikipedia), u.Host), + ), + }, + { + Name: "request for {dnslink}.ipns.{gateway} returns expected payload", + Request: Request(). + Header("Host", Fmt("{{dnslink}}.ipns.{{host}}", dnsLinkTest, u.Host)). + Path("/"), + Response: Expect(). + Body("hello\n"), + }, + { + Name: "request for {inlineddnslink}.ipns.{gateway} returns expected payload", + Request: Request(). + Header("Host", Fmt("{{inlined}}.ipns.{{host}}", dnslink.InlineDNS(dnsLinkTest), u.Host)). + Path("/"), + Response: Expect(). + Body("hello\n"), + }, + { + Name: "request for example.com/ipns/{fqdn} with X-Forwarded-Proto redirects to TLS-safe label in subdomain", + Hint: ` + DNSLink on Public gateway with a single-level wildcard TLS cert + "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 + `, + Request: Request(). + Header("X-Forwarded-Proto", "https"). + Header("Host", u.Host). + Path("/ipns/{{wikipedia}}/wiki/", wikipedia), + Response: Expect(). + Headers( + Header("Location"). + Equals("https://{{inlined}}.ipns.{{host}}/wiki/", dnslink.InlineDNS(wikipedia), u.Host), + ), + }, + { + Name: `request for example.com/ipns/?uri=ipns%3A%2F%2F.. produces redirect to /ipns/.. content path`, + Hint: "Support ipns:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler", + Request: Request(). + Header("Host", u.Host). + Path("/ipns/"). + Query("uri", "ipns://{{dnslink}}", wikipedia), + Response: Expect(). + Headers( + Header("Location").Equals("/ipns/{{wikipedia}}", wikipedia), + ), + }, + }...) + + RunWithSpecs(t, tests, specs.SubdomainGatewayIPNS) } diff --git a/tests/subdomain_gateway_proxy_test.go b/tests/subdomain_gateway_proxy_test.go new file mode 100644 index 000000000..4e1f7c348 --- /dev/null +++ b/tests/subdomain_gateway_proxy_test.go @@ -0,0 +1,141 @@ +package tests + +import ( + "testing" + + "github.com/ipfs/gateway-conformance/tooling/car" + . "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/specs" + . "github.com/ipfs/gateway-conformance/tooling/test" + . "github.com/ipfs/gateway-conformance/tooling/tmpl" +) + +var ( + fixture = car.MustOpenUnixfsCar("subdomain_gateway/fixtures.car") + + CIDVal = string(fixture.MustGetRawData("hello-CIDv1")) // hello + DirCID = fixture.MustGetCid("testdirlisting") + CIDv1 = fixture.MustGetCid("hello-CIDv1") + CIDv0 = fixture.MustGetCid("hello-CIDv0") + CIDv0to1 = fixture.MustGetCid("hello-CIDv0to1") + //CIDv1_TOO_LONG = fixture.MustGetCid("hello-CIDv1_TOO_LONG") + + // the gateway endpoint is used as HTTP proxy + gatewayAsProxyURL = GatewayURL().String() + + // run against origins explicitly passed via --subdomain-url + s = SubdomainGatewayURL() +) + +func TestProxyGatewaySubdomains(t *testing.T) { + tests := SugarTests{ + { + Name: "request for {CID}.ipfs.example.com should return expected payload", + Hint: "HTTP proxy gateway accepts requests for GETs of full URLs as Paths", + Request: Request(). + Proxy(gatewayAsProxyURL). + Path("{{scheme}}://{{cid}}.ipfs.{{host}}", s.Scheme, CIDv1, s.Host), + Response: Expect(). + Status(200). + Body(Contains(CIDVal)), + }, + { + Name: "request for example.com/ipfs/{CIDv0} redirects to {CIDv1}.ipfs.example.com", + Hint: "HTTP proxy gateway accepts requests for GETs of full URLs as Paths", + Request: Request(). + Proxy(gatewayAsProxyURL). + Path("{{scheme}}://{{host}}/ipfs/{{cid}}/", s.Scheme, s.Host, CIDv0), + Response: Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{CIDv0to1} returns Location HTTP header for subdomain redirect in browsers"). + Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", s.Scheme, CIDv0to1, s.Host), + ), + }, + { + Name: "request for {CID}.ipfs.example.com/ipfs/file.txt should return data from a file in CID content root", + Hint: "ensure subdomain gateway takes priority over processing /ipfs/* paths", + Request: Request(). + Proxy(gatewayAsProxyURL). + Path("{{scheme}}://{{cid}}.ipfs.{{host}}/ipfs/file.txt", s.Scheme, DirCID, s.Host), + Response: Expect(). + Status(200). + Body(Contains("I am a txt file")), + }, + /* TODO: value added + { + Name: "request for a too long CID at {CIDv1}.ipfs.example.com returns expected payload", + Hint: "HTTP proxy mode allows responding to requests with 'DNS labels' longer than 63 characters", + Request: Request(). + Proxy(gatewayAsProxyURL). + TODO turn to Path: Header("Host", Fmt("{{cid}}.ipfs.{{host}}", CIDv1_TOO_LONG, s.Host)). + Path("/"), + Response: Expect(). + Status(400). + Body(Contains("TODO")), + }, + */ + } + RunWithSpecs(t, tests, specs.ProxyGateway, specs.SubdomainGatewayIPFS) +} + +func TestProxyTunnelGatewaySubdomains(t *testing.T) { + tests := SugarTests{ + { + Name: "request for {CID}.ipfs.example.com should return expected payload", + Hint: "HTTP CONNECT is how some proxy setups convert an HTTP connection into a tunnel to a remote host https://tools.ietf.org/html/rfc7231#section-4.3.6", + Request: Request(). + WithProxyTunnel(). + Proxy(gatewayAsProxyURL). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", CIDv1, s.Host)). + Path("/"), + Response: Expect(). + Status(200). + Body(Contains(CIDVal)), + }, + { + Name: "request for example.com/ipfs/{CIDv0} redirects to {CIDv1}.ipfs.example.com", + Hint: "proxy tunnel follows ", + Request: Request(). + WithProxyTunnel(). + Proxy(gatewayAsProxyURL). + Header("Host", s.Host). + Path("/ipfs/{{cid}}/", CIDv0), + Response: Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{CIDv0to1} returns Location HTTP header for subdomain redirect in browsers"). + Contains("{{scheme}}://{{cid}}.ipfs.{{host}}/", s.Scheme, CIDv0to1, s.Host), + ), + }, + { + Name: "request for {CID}.ipfs.example.com/ipfs/file.txt should return data from a file in CID content root", + Hint: "ensure subdomain gateway takes priority over processing /ipfs/* paths", + Request: Request(). + WithProxyTunnel(). + Proxy(gatewayAsProxyURL). + Header("Host", Fmt("{{cid}}.ipfs.{{host}}", DirCID, s.Host)). + Path("/ipfs/file.txt"), + Response: Expect(). + Status(200). + Body(Contains("I am a txt file")), + }, + /* TODO: value added + { + Name: "request for a too long CID at {CIDv1}.ipfs.example.com returns expected payload", + Hint: "HTTP proxy mode allows responding to requests with 'DNS labels' longer than 63 characters", + Request: Request(). + WithProxyTunnel(). + Proxy(gatewayAsProxyURL). + TODO turn to Path: Header("Host", Fmt("{{cid}}.ipfs.{{host}}", CIDv1_TOO_LONG, s.Host)). + Path("/"), + Response: Expect(). + Status(400). + Body(Contains("TODO")), + }, + */ + } + RunWithSpecs(t, tests, specs.ProxyGateway, specs.SubdomainGatewayIPFS) +} diff --git a/tooling/dnslink/dnslink.go b/tooling/dnslink/dnslink.go index 481eaa872..5be4328a0 100644 --- a/tooling/dnslink/dnslink.go +++ b/tooling/dnslink/dnslink.go @@ -15,9 +15,8 @@ type ConfigFixture struct { } type DNSLink struct { - Subdomain string `yaml:"subdomain"` - Domain string `yaml:"domain"` - Path string `yaml:"path"` + Domain string `yaml:"domain"` + Path string `yaml:"path"` } func InlineDNS(s string) string { @@ -57,19 +56,13 @@ func (d *ConfigFixture) MustGet(id string) string { if !ok { panic(fmt.Errorf("dnslink %s not found", id)) } - if dnsLink.Domain != "" && dnsLink.Subdomain != "" { - panic(fmt.Errorf("dnslink %s has both domain and subdomain", id)) - } - if dnsLink.Domain == "" && dnsLink.Subdomain == "" { - panic(fmt.Errorf("dnslink %s has neither domain nor subdomain", id)) + if dnsLink.Domain == "" { + panic(fmt.Errorf("dnslink %s has no domain", id)) } if dnsLink.Path == "" { panic(fmt.Errorf("dnslink %s has no path", id)) } - if dnsLink.Domain != "" { - return dnsLink.Domain - } + return dnsLink.Domain - return dnsLink.Subdomain } diff --git a/tooling/dnslink/merge.go b/tooling/dnslink/merge.go index 446af9b19..36c513be2 100644 --- a/tooling/dnslink/merge.go +++ b/tooling/dnslink/merge.go @@ -4,17 +4,16 @@ import ( "encoding/json" "fmt" "os" + "strings" ) type DNSLinksAggregate struct { - Domains map[string]string `json:"domains"` - Subdomains map[string]string `json:"subdomains"` + Domains map[string]string `json:"domains"` } func Aggregate(inputPaths []string) (*DNSLinksAggregate, error) { agg := DNSLinksAggregate{ - Domains: make(map[string]string), - Subdomains: make(map[string]string), + Domains: make(map[string]string), } for _, file := range inputPaths { @@ -24,34 +23,19 @@ func Aggregate(inputPaths []string) (*DNSLinksAggregate, error) { } for _, link := range dnsLinks.DNSLinks { - if link.Domain != "" && link.Subdomain != "" { - return nil, fmt.Errorf("dnslink %s has both domain and subdomain", link.Subdomain) + if _, ok := agg.Domains[link.Domain]; ok { + return nil, fmt.Errorf("collision detected for domain %s", link.Domain) } - if link.Domain != "" { - if _, ok := agg.Domains[link.Domain]; ok { - return nil, fmt.Errorf("collision detected for domain %s", link.Domain) - } - - agg.Domains[link.Domain] = link.Path - continue - } - - if link.Subdomain != "" { - if _, ok := agg.Subdomains[link.Subdomain]; ok { - return nil, fmt.Errorf("collision detected for subdomain %s", link.Subdomain) - } - - agg.Subdomains[link.Subdomain] = link.Path - continue - } + agg.Domains[link.Domain] = link.Path + continue } } return &agg, nil } -func Merge(inputPaths []string, outputPath string) error { +func MergeJSON(inputPaths []string, outputPath string) error { kvs, err := Aggregate(inputPaths) if err != nil { return err @@ -65,3 +49,22 @@ func Merge(inputPaths []string, outputPath string) error { err = os.WriteFile(outputPath, j, 0644) return err } + +// MergeEnv produces a string compatible with IPFS_NS_MAP env veriable syntax +// which can be used by tools to pre-populate namesys (IPNS, DNSLink) resolution +// results to facilitate tests based on static fixtures. +func MergeNsMapEnv(inputPaths []string, outputPath string) error { + kvs, err := Aggregate(inputPaths) + if err != nil { + return err + } + + var result []string + for key, value := range kvs.Domains { + result = append(result, fmt.Sprintf("%s:%s", key, value)) + } + nsMapValue := strings.Join(result, ",") + + err = os.WriteFile(outputPath, []byte(nsMapValue), 0644) + return err +} diff --git a/tooling/helpers/subdomain.go b/tooling/helpers/subdomain.go deleted file mode 100644 index ae38bf25f..000000000 --- a/tooling/helpers/subdomain.go +++ /dev/null @@ -1,87 +0,0 @@ -package helpers - -import ( - "fmt" - "net/url" - "testing" - - "github.com/ipfs/gateway-conformance/tooling/test" -) - -/** - * UnwrapSubdomainTests takes a list of tests and returns a (larger) list of tests - * that will run on the subdomain gateway. - */ -func UnwrapSubdomainTests(t *testing.T, tests test.SugarTests) test.SugarTests { - t.Helper() - - var out test.SugarTests - for _, test := range tests { - out = append(out, unwrapSubdomainTest(t, test)...) - } - return out -} - -func unwrapSubdomainTest(t *testing.T, unwraped test.SugarTest) test.SugarTests { - t.Helper() - - baseURL := unwraped.Request.GetURL() - req := unwraped.Request - expected := unwraped.Response - - u, err := url.Parse(baseURL) - if err != nil { - t.Fatal(err) - } - // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed - // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is - // actually living on http://127.0.0.1:8080 or somewhere else. - // - // The test knows two addresses: - // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. - // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. - - // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` - host := u.Host - - // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` - u.Host = test.GatewayHost - rawURL := u.String() - - return test.SugarTests{ - { - Name: fmt.Sprintf("%s (direct HTTP)", unwraped.Name), - Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), - Request: req. - URL(rawURL). - Headers( - test.Header("Host", host), - ), - Response: expected, - }, - { - Name: fmt.Sprintf("%s (HTTP proxy)", unwraped.Name), - Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, "HTTP proxy (hostname is passed via URL)"), - Request: req. - URL(baseURL). - Proxy(test.GatewayURL), - Response: expected, - }, - { - Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", unwraped.Name), - Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, `HTTP proxy - In HTTP/1.x, the pseudo-method CONNECT, - can be used to convert an HTTP connection into a tunnel to a remote host - https://tools.ietf.org/html/rfc7231#section-4.3.6 - `), - Request: req. - URL(baseURL). - Proxy(test.GatewayURL). - WithProxyTunnel(). - Headers( - test.Header("Host", host), - ), - Response: expected, - }, - } -} diff --git a/tooling/specs/specs.go b/tooling/specs/specs.go index af4d136e8..f99bd7d37 100644 --- a/tooling/specs/specs.go +++ b/tooling/specs/specs.go @@ -121,6 +121,7 @@ var ( SubdomainGateway = Collection{"subdomain-gateway", []Spec{SubdomainGatewayIPFS, SubdomainGatewayIPNS}} DNSLinkGateway = Leaf{"dnslink-gateway", stable} RedirectsFile = Leaf{"redirects-file", stable} + ProxyGateway = Leaf{"proxy-gateway", stable} ) // All specs MUST be listed here. @@ -141,6 +142,7 @@ var specs = []Spec{ SubdomainGateway, DNSLinkGateway, RedirectsFile, + ProxyGateway, } var specEnabled = map[Spec]bool{} diff --git a/tooling/test/config.go b/tooling/test/config.go index 506c03785..f0ba32700 100644 --- a/tooling/test/config.go +++ b/tooling/test/config.go @@ -10,46 +10,26 @@ import ( var log = logging.Logger("conformance") -func GetEnv(key string, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value +func env2url(key string) *url.URL { + value, ok := os.LookupEnv(key) + if !ok { + // Actual validation of GATEWAY_URL and SUBDOMAIN_GATEWAY_URL happens in + // in cmd/gateway-conformance/main.go. Here we create a bogus URL + // to ensure no error at compilation time. + value = "http://todo-changeme.example.com" } - return fallback -} - -var GatewayURL = strings.TrimRight( - GetEnv("GATEWAY_URL", "http://127.0.0.1:8080"), - "/") - -var SubdomainGatewayURL = strings.TrimRight( - GetEnv("SUBDOMAIN_GATEWAY_URL", "http://example.com"), - "/") - -var GatewayHost = "" -var SubdomainGatewayHost = "" -var SubdomainGatewayScheme = "" - -var SubdomainLocalhostGatewayURL = "http://localhost" - -func init() { - parsed, err := url.Parse(GatewayURL) - if err != nil { - panic(err) - } - - GatewayHost = parsed.Host - - parsed, err = url.Parse(SubdomainGatewayURL) + gatewayURL := strings.TrimRight(value, "/") + parsed, err := url.Parse(gatewayURL) if err != nil { panic(err) } + return parsed +} - SubdomainGatewayHost = parsed.Host - SubdomainGatewayScheme = parsed.Scheme - - log.Debugf("GatewayURL: %s", GatewayURL) +func GatewayURL() *url.URL { + return env2url("GATEWAY_URL") +} - log.Debugf("SubdomainGatewayURL: %s", SubdomainGatewayURL) - log.Debugf("SubdomainGatewayHost: %s", SubdomainGatewayHost) - log.Debugf("SubdomainGatewayScheme: %s", SubdomainGatewayScheme) +func SubdomainGatewayURL() *url.URL { + return env2url("SUBDOMAIN_GATEWAY_URL") } diff --git a/tooling/test/proxy.go b/tooling/test/proxy.go index 55e893e3e..d1888f68f 100644 --- a/tooling/test/proxy.go +++ b/tooling/test/proxy.go @@ -10,6 +10,8 @@ import ( "net/url" ) +// NewProxyTunnelClient creates an HTTP client that routes requests through an HTTP proxy +// using the CONNECT method, as described in RFC 7231 Section 4.3.6. func NewProxyTunnelClient(proxyURL string) *http.Client { proxy, err := url.Parse(proxyURL) if err != nil { @@ -52,6 +54,7 @@ func NewProxyTunnelClient(proxyURL string) *http.Client { return conn, nil }, + // Skip TLS cert verification to make it easier to test on CI and dev envs TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } @@ -62,6 +65,7 @@ func NewProxyTunnelClient(proxyURL string) *http.Client { return client } +// NewProxyClient creates an HTTP client that routes requests through an HTTP proxy. func NewProxyClient(proxyURL string) *http.Client { proxy, err := url.Parse(proxyURL) if err != nil { diff --git a/tooling/test/run.go b/tooling/test/run.go index 8e6e91dea..d408ef270 100644 --- a/tooling/test/run.go +++ b/tooling/test/run.go @@ -6,7 +6,10 @@ import ( "fmt" "io" "net/http" + "strings" "testing" + + "github.com/ipfs/gateway-conformance/tooling" ) type Reporter func(t *testing.T, msg interface{}, rest ...interface{}) @@ -18,8 +21,9 @@ func runRequest(ctx context.Context, t *testing.T, test SugarTest, builder Reque } // Prepare a client, - // use proxy, deal with redirects, etc. client := &http.Client{} + + // HTTP proxy tests require additional prep if builder.UseProxyTunnel_ { if builder.Proxy_ == "" { t.Fatal("ProxyTunnel requires a proxy") @@ -30,6 +34,7 @@ func runRequest(ctx context.Context, t *testing.T, test SugarTest, builder Reque client = NewProxyClient(builder.Proxy_) } + // Handle redirect tests if !builder.FollowRedirects_ { client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse @@ -54,21 +59,26 @@ func runRequest(ctx context.Context, t *testing.T, test SugarTest, builder Reque } var url string - if builder.URL_ != "" && builder.Path_ != "" { - localReport(t, "Both 'URL' and 'Path' are set") - } - if builder.URL_ == "" && builder.Path_ == "" { - localReport(t, "Neither 'URL' nor 'Path' are set") - } - if builder.URL_ != "" { - url = builder.URL_ + if builder.Path_ == "" { + localReport(t, "'Path' is not set") } if builder.Path_ != "" { - if builder.Path_[0] != '/' { - localReport(t, "Path must start with '/'") + if builder.Proxy_ != "" && !builder.UseProxyTunnel_ { + // plain HTTP proxy test uses custom client, and the Path is the full URL + // to be used in the request + if !strings.HasPrefix(builder.Path_, "http") { + t.Fatalf("plain Proxy tests require requested Path to be full URL starting with http. builder.Path_ was %q", builder.Path_) + } + // plain proxy requests use Path as-is + url = builder.Path_ + } else { + // no HTTP proxy, make a regular HTTP request for Path at GatewayURL (+ optional Host header) + if builder.Path_[0] != '/' { + localReport(t, "When proxy mode is not used, the Path must start with '/'") + } + // regular requests attach Path to gateway endpoint URL + url = fmt.Sprintf("%s%s", strings.TrimRight(GatewayURL().String(), "/"), builder.Path_) } - - url = fmt.Sprintf("%s%s", GatewayURL, builder.Path_) } query := builder.Query_.Encode() @@ -97,7 +107,12 @@ func runRequest(ctx context.Context, t *testing.T, test SugarTest, builder Reque } } - // send request + // Set meaningful User-Agent, if custom one was not set by a test + if _, exists := builder.Headers_["User-Agent"]; !exists { + req.Header.Set("User-Agent", "ipfs/gateway-conformance/"+tooling.Version) + } + + // Send request log.Debugf("Querying %s", url) req = req.WithContext(ctx) diff --git a/tooling/test/sugar.go b/tooling/test/sugar.go index 5c5a7a2a9..cbe0228da 100644 --- a/tooling/test/sugar.go +++ b/tooling/test/sugar.go @@ -14,7 +14,6 @@ import ( type RequestBuilder struct { Method_ string `json:"method,omitempty"` Path_ string `json:"path,omitempty"` - URL_ string `json:"url,omitempty"` Proxy_ string `json:"proxy,omitempty"` UseProxyTunnel_ bool `json:"useProxyTunnel,omitempty"` Headers_ map[string]string `json:"headers,omitempty"` @@ -40,22 +39,20 @@ func (r RequestBuilder) Path(path string, args ...any) RequestBuilder { return r } -func (r RequestBuilder) URL(path string, args ...any) RequestBuilder { - r.URL_ = tmpl.Fmt(path, args...) - return r -} - func (r RequestBuilder) Query(key, value string, args ...any) RequestBuilder { r.Query_.Add(key, tmpl.Fmt(value, args...)) return r } -func (r RequestBuilder) GetURL() string { - if r.Path_ != "" { - panic("not supported") +func (r RequestBuilder) RemoveHeader(hdr string) string { + if r.Headers_ != nil { + v, ok := r.Headers_[hdr] + if ok { + delete(r.Headers_, v) + return v + } } - - return r.URL_ + return "" } func (r RequestBuilder) Proxy(path string, args ...any) RequestBuilder { @@ -125,7 +122,6 @@ func (r RequestBuilder) Clone() RequestBuilder { return RequestBuilder{ Method_: r.Method_, Path_: r.Path_, - URL_: r.URL_, Proxy_: r.Proxy_, UseProxyTunnel_: r.UseProxyTunnel_, Headers_: clonedHeaders,