v0.11.0

.github/workflows/e2e.yml

@@ -36,22 +36,29 @@ jobs:
           dgate-cli --version
           go install github.com/dgate-io/dgate/cmd/dgate-server
           dgate-server --version
-
-      - run: go run cmd/dgate-server/main.go &
-
-      - name: Wait for server to start
-        run: sleep 5
 
       - name: Install jq
         run: |
           sudo apt install -y jq
           jq --version
 
-      - name: Functional Tests
+      - name: Install goreman
+        run: |
+          go install github.com/mattn/goreman@latest
+          goreman version
+
+      - run: go run cmd/dgate-server/main.go &
+
+      - run: cd functional-tests/raft_tests && goreman start &
+
+      - name: Wait for server to start
+        run: sleep 10
+
+      - name: Functional Standalone Tests
         run: |
           for i in functional-tests/admin_tests/*.sh; \
             do bash -c $i; done
-  
+
       - name: Run local k6 test
         uses: grafana/k6-action@v0.3.1
         with:

README.md

@@ -15,7 +15,7 @@ DGate is a distributed API Gateway built for developers. DGate allows you to use
 
 ## Getting Started
 
-Coming soon @ http://dgate.io/docs/getting-started
+http://dgate.io/docs/getting-started
 
 ### Installing
 

TODO.md

@@ -8,6 +8,12 @@
   - cluster management (raft commands, replica commands, etc.) (low priority)
   - other commands (backup, restore, etc.) (low priority)
 
+# Raft Snapshots
+
+- Add support for Raft snapshots to reduce the size of the Raft log. This can be used to reduce the size of the Raft log and improve the performance of the cluster.
+  - [ ] - Snapshot documents
+  - [ ] - Snapshot resources (with correlactions)
+
 ## Add Module Tests
 
 - Test multiple modules being used at the same time
@@ -138,13 +144,12 @@ Make it easier to debug modules by adding more logging and error handling. This
 
 Add stack tracing for typescript modules.
 
-
-## Decouple Admin API from Raft Implementation
-
-Currently, Raft Implementation is tightly coupled with the Admin API. This makes it difficult to change the Raft Implementation without changing the Admin API. Decouple the Raft Implementation from the Admin API to make it easier to change the Raft Implementation.
-
 ## Add Telemetry (sentry, datadog, etc.)
 
 ## ResourceManager callback for resource changes
 
-Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.
+Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.
+
+## Enable WAF
+
+https://github.com/corazawaf/coraza

cmd/dgate-server/main.go

@@ -58,19 +58,23 @@
 		}
 		if dgateConfig, err := config.LoadConfig(*configPath); err != nil {
 			fmt.Printf("Error loading config: %s\n", err)
-			os.Exit(1)
+			panic(err)
 		} else {
 			logger, err := dgateConfig.GetLogger()
 			if err != nil {
 				fmt.Printf("Error setting up logger: %s\n", err)
-				os.Exit(1)
+				panic(err)
 			}
 			defer logger.Sync()
 			proxyState := proxy.NewProxyState(logger.Named("proxy"), dgateConfig)
-			admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState)
+			err = admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState)
+			if err != nil {
+				fmt.Printf("Error starting admin api: %s\n", err)
+				panic(err)
+			}
 			if err := proxyState.Start(); err != nil {
 				fmt.Printf("Error loading config: %s\n", err)
-				os.Exit(1)
+				panic(err)
 			}
 
 			sigchan := make(chan os.Signal, 1)

config.dgate.yaml

@@ -1,9 +1,8 @@
 version: v1
 debug: true
-log_level: ${LOG_LEVEL:-info}
+log_level: ${LOG_LEVEL:-debug}
 disable_default_namespace: true
-tags:
-  - debug
+tags: [debug, local, test]
 storage:
   type: file
   dir: .dgate/data/
@@ -15,7 +14,7 @@ test_server:
 proxy:
   port: ${PORT:-80}
   host: 0.0.0.0
-  enable_console_logger: true
+  console_log_level: info
   transport:
     dns_prefer_go: true
   init_resources:

functional-tests/admin_tests/admin_test.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eo xtrace
+
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
+PROXY_URL=${PROXY_URL:-"http://localhost"}
+TEST_URL=${TEST_URL:-"http://localhost:8888"}
+
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+export DGATE_ADMIN_API=$ADMIN_URL
+
+# check if uuid is available
+if ! command -v uuid > /dev/null; then
+    id=X$RANDOM-$RANDOM-$RANDOM
+else
+    id=$(uuid)
+fi
+
+dgate-cli -Vf namespace create name=ns-$id
+
+dgate-cli -Vf domain create name=dm-$id \
+    namespace=ns-$id priority:=$RANDOM patterns="$id.example.com"
+
+dgate-cli -Vf service create \
+    name=svc-$id namespace=ns-$id \
+    urls="$TEST_URL/$RANDOM"
+
+dgate-cli -Vf module create name=module1 \
+    payload@=$DIR/admin_test.ts \
+    namespace=ns-$id
+
+dgate-cli -Vf route create \
+    name=rt-$id \
+    service=svc-$id \
+    namespace=ns-$id \
+    paths="/,/{id},/$id,/$id/{id}" \
+    methods=GET,POST,PUT \
+    modules=module1 \
+    preserveHost:=false \
+    stripPath:=false
+
+curl -sf $ADMIN_URL/readyz > /dev/null
+
+curl -f ${PROXY_URL}/$id/$RANDOM -H Host:$id.example.com
+
+echo "Admin Test Succeeded"

functional-tests/admin_tests/admin_test.ts

@@ -0,0 +1,4 @@
+
+export const responseModifier = async (ctx: any) => {
+    console.log("responseModifier -> path params", ctx.pathParams());
+}

functional-tests/admin_tests/change_checker.sh

@@ -9,41 +9,41 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=change_checker-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=change_checker-dm \
-    patterns:='["change_checker.com"]' \
+    patterns:='["change_checker.example.com"]' \
     namespace=change_checker-ns
 
-dgate-cli module create name=change_checker-mod \
+dgate-cli -Vf module create name=change_checker-mod \
     payload@=$DIR/change_checker_1.ts \
     namespace=change_checker-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt paths:='["/", "/{id}"]' \
     modules:='["change_checker-mod"]' \
     methods:='["GET","POST"]' \
     stripPath:=true \
     preserveHost:=true \
     namespace=change_checker-ns
 
-MODID1=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+MODID1=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod')
 
 if [ "$MODID1" != "module1" ]; then
     echo "Initial assert failed"
     exit 1
 fi
 
 
-dgate-cli module create name=change_checker-mod \
+dgate-cli -Vf module create name=change_checker-mod \
     payload@=$DIR/change_checker_2.ts \
     namespace=change_checker-ns
 
 # dgate-cli r.ker-ns
 
-MODID2=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+MODID2=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod')
 
 if [ "$MODID2" != "module2" ]; then
     echo "module update failed"

functional-tests/admin_tests/iphash_load_balancer_test.sh

@@ -4,32 +4,33 @@ set -eo xtrace
 
 ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
+TEST_URL=${TEST_URL:-"http://localhost:8888"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=test-lb-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=test-lb-dm \
-    patterns:='["test-lb.com"]' \
+    patterns:='["test-lb.example.com"]' \
     namespace=test-lb-ns
 
 MOD_B64="$(base64 < $DIR/iphash_load_balancer.ts)"
-dgate-cli module create \
+dgate-cli -Vf module create \
     name=printer \
     payload="$MOD_B64" \
     namespace=test-lb-ns
 
 
-dgate-cli service create \
+dgate-cli -Vf service create \
     name=base_svc \
-    urls:='["http://localhost:8888/a","http://localhost:8888/b","http://localhost:8888/c"]' \
+    urls="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \
     namespace=test-lb-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt \
     paths:='["/test-lb","/hello"]' \
     methods:='["GET"]' \
@@ -39,9 +40,9 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=test-lb-ns
 
-path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com | jq -r '.data.path')"
+path1="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')"
 
-path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
+path2="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
 
 if [ "$path1" != "$path2" ]; then
     echo "IP Hash Load Balancer Test Passed"

functional-tests/admin_tests/merge_responses_test.sh

@@ -9,21 +9,21 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=test-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=test-dm \
-    patterns:='["test.com"]' \
+    patterns:='["test.example.com"]' \
     namespace=test-ns
 
 MOD_B64="$(base64 < $DIR/merge_responses.ts)"
-dgate-cli module create \
+dgate-cli -Vf module create \
     name=printer \
     payload="$MOD_B64" \
     namespace=test-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt \
     paths:='["/test","/hello"]' \
     methods:='["GET"]' \
@@ -32,6 +32,6 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=test-ns
 
-curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.com
+curl -sf ${PROXY_URL}/hello -H Host:test.example.com
 
 echo "Merge Responses Test Passed"

functional-tests/admin_tests/modify_request_test.sh

@@ -4,30 +4,31 @@ set -eo xtrace
 
 ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
+TEST_URL=${TEST_URL:-"http://localhost:8888"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=modify_request_test-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=modify_request_test-dm \
-    patterns:='["modify_request_test.com"]' \
+    patterns:='["modify_request_test.example.com"]' \
     namespace=modify_request_test-ns
 
 MOD_B64="$(base64 < $DIR/modify_request.ts)"
-dgate-cli module create \
+dgate-cli -Vf module create \
     name=printer payload="$MOD_B64" \
     namespace=modify_request_test-ns
 
-dgate-cli service create \
+dgate-cli -Vf service create \
     name=base_svc \
-    urls:='["http://localhost:8888"]' \
+    urls="$TEST_URL" \
     namespace=modify_request_test-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt \
     paths:='["/modify_request_test"]' \
     methods:='["GET"]' \
@@ -37,8 +38,8 @@ dgate-cli route create \
     namespace=modify_request_test-ns \
     service='base_svc'
 
-curl -s --fail-with-body ${PROXY_URL}/modify_request_test \
-    -H Host:modify_request_test.com \
+curl -sf ${PROXY_URL}/modify_request_test \
+    -H Host:modify_request_test.example.com \
     -H X-Forwarded-For:1.1.1.1
 
 echo "Modify Request Test Passed"