From dad71cba088cd27c08f5cb42149f186f05b12106 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Mar 2023 13:37:14 +0100 Subject: [PATCH 01/11] ci: add gateway-confromance test suite to the CI --- .github/workflows/gateway-conformance.yml | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/gateway-conformance.yml diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml new file mode 100644 index 0000000..3cfa57d --- /dev/null +++ b/.github/workflows/gateway-conformance.yml @@ -0,0 +1,65 @@ +name: Gateway Conformance + +on: + push: + +jobs: + gateway-conformance: + runs-on: ubuntu-latest + steps: + # 1. Start the Kubo gateway + - name: Download Kubo gateway + uses: ipfs/download-ipfs-distribution-action@v1 + - name: Start Kubo gateway + uses: ipfs/start-ipfs-daemon-action@v1 + + # 2. Download the gateway-conformance fixtures + - name: Download gateway-conformance fixtures + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@main + with: + output: fixtures + + # 3. Populate the Kubo gateway with the gateway-conformance fixtures + - name: Import fixtures + run: find ./fixtures -name '*.car' -exec ipfs dag import {} \; + + # 4. Build the bifrost-gateway + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.x + - name: Checkout bifrost-gateway + uses: actions/checkout@v3 + with: + path: bifrost-gateway + - name: Build bifrost-gateway + run: go build + working-directory: bifrost-gateway + + # 5. Start the bifrost-gateway + - name: Start bifrost-gateway + env: + PROXY_GATEWAY_URL: http://127.0.0.1:8080 + run: ./bifrost-gateway & + working-directory: bifrost-gateway + + # 6. Run the gateway-conformance tests + - name: Run gateway-conformance tests + uses: ipfs/gateway-conformance/.github/actions/test@main + with: + gateway-url: http://127.0.0.1:8081 + json: output.json + xml: output.xml + html: output.html + markdown: output.md + + # 7. Upload the results + - name: Upload MD summary + if: failure() || success() + run: cat output.md >> $GITHUB_STEP_SUMMARY + - name: Upload HTML report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance.html + path: output.html From 3dc6457def56b11f214d0bbb59cc2ff766b034ca Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Mar 2023 13:58:30 +0100 Subject: [PATCH 02/11] fix: disable subdomain-gateway tests --- .github/workflows/gateway-conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 3cfa57d..f7402e2 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -52,6 +52,7 @@ jobs: xml: output.xml html: output.html markdown: output.md + specs: -subdomain-gateway # 7. Upload the results - name: Upload MD summary From 9ef96bc29672983e253dd0800cd1e18da465c810 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Mar 2023 14:03:34 +0100 Subject: [PATCH 03/11] fix: disable TestGatewayCar --- .github/workflows/gateway-conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index f7402e2..6f8950d 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -53,6 +53,7 @@ jobs: html: output.html markdown: output.md specs: -subdomain-gateway + args: -skip TestGatewayCar # 7. Upload the results - name: Upload MD summary From 97cbdce4f42069f35890a591dbb4eb3ca4746a09 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Mar 2023 14:04:51 +0100 Subject: [PATCH 04/11] ci: pin gateway-conformance to a SHA because it doesn't have release tags yet --- .github/workflows/gateway-conformance.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 6f8950d..a8d1828 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -15,7 +15,8 @@ jobs: # 2. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@main + # TODO: use a release tag once https://github.com/ipfs/gateway-conformance/pull/19 is merged + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e320c2b1f576f69732d824c4d48a1c37bb0ad320 with: output: fixtures @@ -45,7 +46,8 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@main + # TODO: use a release tag once https://github.com/ipfs/gateway-conformance/pull/19 is merged + uses: ipfs/gateway-conformance/.github/actions/test@e320c2b1f576f69732d824c4d48a1c37bb0ad320 with: gateway-url: http://127.0.0.1:8081 json: output.json From c0aa98c901513f01ef2f3400f81e244e1aa4afa1 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Mar 2023 14:08:18 +0100 Subject: [PATCH 05/11] ci: update triggers for gateway-conformance workflow --- .github/workflows/gateway-conformance.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index a8d1828..595d98e 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -2,6 +2,9 @@ name: Gateway Conformance on: push: + branches: + - main + pull_request: jobs: gateway-conformance: From eda6fb4ea761438fa209a0b018be517ec979dc26 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 24 Mar 2023 14:46:19 +0100 Subject: [PATCH 06/11] chore: enable subdomain tests, add env variable --- .github/workflows/gateway-conformance.yml | 2 +- handlers.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 595d98e..d4a564b 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -44,6 +44,7 @@ jobs: - name: Start bifrost-gateway env: PROXY_GATEWAY_URL: http://127.0.0.1:8080 + GATEWAY_CONFORMANCE_TEST: true run: ./bifrost-gateway & working-directory: bifrost-gateway @@ -57,7 +58,6 @@ jobs: xml: output.xml html: output.html markdown: output.md - specs: -subdomain-gateway args: -skip TestGatewayCar # 7. Upload the results diff --git a/handlers.go b/handlers.go index 3bfe0b4..08b88a9 100644 --- a/handlers.go +++ b/handlers.go @@ -4,6 +4,7 @@ import ( "fmt" "math/rand" "net/http" + "os" "strconv" "time" @@ -96,6 +97,15 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC }, } + // If we're doing tests, ensure the right public gateways are enabled. + if os.Getenv("GATEWAY_CONFORMANCE_TEST") == "true" { + publicGateways["example.com"] = &gateway.Specification{ + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: noDNSLink, + UseSubdomains: true, + } + } + // Creates metrics handler for total response size. Matches the same metrics // from Kubo: // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 From d31a7a831cf531a14f2bfcd00cacd25d87c05723 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 24 Mar 2023 14:56:15 +0100 Subject: [PATCH 07/11] fix: support http.MethodConnect --- handlers.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/handlers.go b/handlers.go index 08b88a9..0a29d92 100644 --- a/handlers.go +++ b/handlers.go @@ -33,6 +33,20 @@ func makeMetricsHandler(port int) (*http.Server, error) { }, nil } +func withConnect(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // ServeMux does not support requests with CONNECT method, + // so we need to handle them separately + // https://golang.org/src/net/http/request.go#L111 + if r.Method == http.MethodConnect { + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) +} + func withRequestLogger(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { goLog.Infow(r.Method, "url", r.URL, "host", r.Host) @@ -122,7 +136,8 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC } // Construct the HTTP handler for the gateway. - handler := http.Handler(gateway.WithHostname(mux, gwAPI, publicGateways, noDNSLink)) + handler := withConnect(mux) + handler = http.Handler(gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink)) handler = promhttp.InstrumentHandlerResponseSize(sum, handler) // Add logging From 4576bc6afb152222422a4d154c4769eab1fbb084 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 25 Mar 2023 00:00:24 +0100 Subject: [PATCH 08/11] docs: environment variables for testing --- .github/workflows/gateway-conformance.yml | 2 +- docs/environment-variables.md | 24 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index d4a564b..d655903 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -25,7 +25,7 @@ jobs: # 3. Populate the Kubo gateway with the gateway-conformance fixtures - name: Import fixtures - run: find ./fixtures -name '*.car' -exec ipfs dag import {} \; + run: find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false {} \; # 4. Build the bifrost-gateway - name: Setup Go diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 2e52f1d..1db6ba8 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -16,6 +16,9 @@ - [`GOLOG_LOG_FMT`](#golog_log_fmt) - [`GOLOG_FILE`](#golog_file) - [`GOLOG_TRACING_FILE`](#golog_tracing_file) +- [Testing](#testing) + - [`GATEWAY_CONFORMANCE_TEST`](#gateway_conformance_test) + - [`IPFS_NS_MAP`](#ipfs_ns_map) ## Configuration @@ -108,3 +111,24 @@ Sets the file to which the Bifrost Gateway sends tracing events. By default, tracing is disabled. Warning: Enabling tracing will likely affect performance. + + +## Testing + +### `GATEWAY_CONFORMANCE_TEST` + +Setting to `true` enables support for test fixtures required by [ipfs/gateway-conformance](https://github.com/ipfs/gateway-conformance) test suite. + +### `IPFS_NS_MAP` + +Adds static namesys records for deterministic tests and debugging. +Useful for testing `/ipns/` support without having to do real IPNS/DNS lookup. + +Example: + +```console +$ IPFS_NS_MAP="dnslink-test1.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am,dnslink-test2.example.com:/ipns/dnslink-test1.example.com" ./gateway-binary +... +$ curl -is http://127.0.0.1:8081/dnslink-test2.example.com/ | grep Etag +Etag: "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" +``` From ed81585a5d3a3996ad8cc72b4d05ab153771b95d Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Sat, 25 Mar 2023 17:55:35 +0100 Subject: [PATCH 09/11] fix: exclude TestGatewayCar/GET_response_for_application/vnd.ipld.car only --- .github/workflows/gateway-conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index d655903..317e04f 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -58,7 +58,7 @@ jobs: xml: output.xml html: output.html markdown: output.md - args: -skip TestGatewayCar + args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car' # 7. Upload the results - name: Upload MD summary From 534d14ce65cd315057f366c5c26dc7fbd440ac61 Mon Sep 17 00:00:00 2001 From: galargh Date: Mon, 27 Mar 2023 11:38:45 +0200 Subject: [PATCH 10/11] ci: exclude only Content-Length check from the TestGatewayCar test --- .github/workflows/gateway-conformance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 317e04f..e2bab7f 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -19,7 +19,7 @@ jobs: # 2. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures # TODO: use a release tag once https://github.com/ipfs/gateway-conformance/pull/19 is merged - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e320c2b1f576f69732d824c4d48a1c37bb0ad320 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@f10c39191e68550f8d9340c6793cbd6dcd118bb4 with: output: fixtures @@ -51,14 +51,14 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests # TODO: use a release tag once https://github.com/ipfs/gateway-conformance/pull/19 is merged - uses: ipfs/gateway-conformance/.github/actions/test@e320c2b1f576f69732d824c4d48a1c37bb0ad320 + uses: ipfs/gateway-conformance/.github/actions/test@f10c39191e68550f8d9340c6793cbd6dcd118bb4 with: gateway-url: http://127.0.0.1:8081 json: output.json xml: output.xml html: output.html markdown: output.md - args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car' + args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' # 7. Upload the results - name: Upload MD summary From 190afa10e536d072593543263d2c2e0c36488893 Mon Sep 17 00:00:00 2001 From: galargh Date: Mon, 27 Mar 2023 13:18:54 +0200 Subject: [PATCH 11/11] ci: use v0.0 of gateway-conformance actions --- .github/workflows/gateway-conformance.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index e2bab7f..724df4a 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -18,8 +18,7 @@ jobs: # 2. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - # TODO: use a release tag once https://github.com/ipfs/gateway-conformance/pull/19 is merged - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@f10c39191e68550f8d9340c6793cbd6dcd118bb4 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.0 with: output: fixtures @@ -50,8 +49,7 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests - # TODO: use a release tag once https://github.com/ipfs/gateway-conformance/pull/19 is merged - uses: ipfs/gateway-conformance/.github/actions/test@f10c39191e68550f8d9340c6793cbd6dcd118bb4 + uses: ipfs/gateway-conformance/.github/actions/test@v0.0 with: gateway-url: http://127.0.0.1:8081 json: output.json