update test to be inclusive for distributed nodes as well, service ur…

…l validation, improve document loading and storing

.github/workflows/e2e.yml

@@ -36,22 +36,47 @@ jobs:
           dgate-cli --version
           go install github.com/dgate-io/dgate/cmd/dgate-server
           dgate-server --version
-
-      - run: go run cmd/dgate-server/main.go &
-
-      - name: Wait for server to start
-        run: sleep 5
     
       - name: Install jq
         run: |
           sudo apt install -y jq
           jq --version
       
-      - name: Functional Tests
+      - name: Install goreman
         run: |
+          go get go install github.com/mattn/goreman@latest
+          goreman version
+
+      - run: go run cmd/dgate-server/main.go &
+
+      - run: |
+          cd functional-tests/raft_tests
+          goreman start &
+
+      - name: Wait for server to start
+        run: sleep 5
+
+      - name: Functional Standalone Tests
+        run: |
+          for i in functional-tests/admin_tests/*.sh; \
+            do bash -c $i; done
+  
+      - name: Functional Distributed Tests [Node 1]
+        run: |
+          export ADMIN_URL=http://localhost:9081
+          export PROXY_URL=http://localhost:81
+          export TEST_URL=http://localhost:8081
           for i in functional-tests/admin_tests/*.sh; \
             do bash -c $i; done
   
+      - name: Functional Distributed Tests [Node 2]
+        run: |
+          export ADMIN_URL=http://localhost:9083
+          export PROXY_URL=http://localhost:83
+          export TEST_URL=http://localhost:8083
+          for i in functional-tests/admin_tests/*.sh; \
+            do bash -c $i; done
+
       - name: Run local k6 test
         uses: grafana/k6-action@v0.3.1
         with:

functional-tests/admin_tests/admin_test.sh

@@ -11,7 +11,7 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 # domain setup
 # check if uuid is available
 if ! command -v uuid > /dev/null; then
-    id=x-$RANDOM-$RANDOM-$RANDOM
+    id=X$RANDOM-$RANDOM-$RANDOM
 else
     id=$(uuid)
 fi
@@ -23,7 +23,7 @@ dgate-cli -Vf domain create name=dm-$id \
 
 dgate-cli -Vf service create \
     name=svc-$id namespace=ns-$id \
-    urls="$TEST/$RANDOM"
+    urls="$TEST_URL/$RANDOM"
 
 dgate-cli -Vf module create name=module1 \
     payload@=$DIR/admin_test.ts \
@@ -33,14 +33,14 @@ dgate-cli -Vf route create \
     name=rt-$id \
     service=svc-$id \
     namespace=ns-$id \
-    paths="/,/{},/$id,/$id/{id}" \
+    paths="/,/{id},/$id,/$id/{id}" \
     methods=GET,POST,PUT \
     modules=module1 \
     preserveHost:=false \
     stripPath:=false
 
-curl -f $ADMIN_URL/readyz
+curl -sf $ADMIN_URL/readyz > /dev/null
 
-curl -f ${PROXY_URL}/$id/$RANDOM-$j -H Host:$id.example.com
+curl -f ${PROXY_URL}/$id/$RANDOM -H Host:$id.example.com
 
 echo "Admin Test Succeeded"

functional-tests/admin_tests/iphash_load_balancer_test.sh

@@ -27,7 +27,7 @@ dgate-cli -Vf module create \
 
 dgate-cli -Vf service create \
     name=base_svc \
-    urls:="$TEST/a","$TEST/b","$TEST/c" \
+    urls:="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \
     namespace=test-lb-ns
 
 dgate-cli -Vf route create \
@@ -40,9 +40,9 @@ dgate-cli -Vf route create \
     preserveHost:=true \
     namespace=test-lb-ns
 
-path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')"
+path1="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')"
 
-path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
+path2="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
 
 if [ "$path1" != "$path2" ]; then
     echo "IP Hash Load Balancer Test Passed"

functional-tests/admin_tests/merge_responses_test.sh

@@ -32,6 +32,6 @@ dgate-cli -Vf route create \
     preserveHost:=true \
     namespace=test-ns
 
-curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.example.com
+curl -sf ${PROXY_URL}/hello -H Host:test.example.com
 
 echo "Merge Responses Test Passed"

functional-tests/admin_tests/modify_request_test.sh

@@ -25,7 +25,7 @@ dgate-cli -Vf module create \
 
 dgate-cli -Vf service create \
     name=base_svc \
-    urls:="$TEST" \
+    urls:="$TEST_URL" \
     namespace=modify_request_test-ns
 
 dgate-cli -Vf route create \
@@ -38,7 +38,7 @@ dgate-cli -Vf route create \
     namespace=modify_request_test-ns \
     service='base_svc'
 
-curl -s --fail-with-body ${PROXY_URL}/modify_request_test \
+curl -sf ${PROXY_URL}/modify_request_test \
     -H Host:modify_request_test.example.com \
     -H X-Forwarded-For:1.1.1.1
 

functional-tests/admin_tests/modify_response_test.sh

@@ -25,7 +25,7 @@ dgate-cli -Vf module create \
 
 dgate-cli -Vf service create \
     name=base_svc \
-    urls:="$TEST"\
+    urls:="$TEST_URL"\
     namespace=test-ns
 
 dgate-cli -Vf route create \

functional-tests/admin_tests/multi_module_test.sh

@@ -58,7 +58,7 @@ dgate-cli -Vf module create name=multimod2 \
     payload="$MOD_B64" namespace=multimod-test-ns
 
 dgate-cli -Vf service create name=base_svc \
-    urls="$TEST/a","$TEST/b","$TEST/c" \
+    urls="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \
     namespace=multimod-test-ns
 
 dgate-cli -Vf route create name=base_rt \
@@ -71,7 +71,7 @@ dgate-cli -Vf route create name=base_rt \
     namespace=multimod-test-ns
 
 
-curl -s --fail-with-body ${PROXY_URL}/ -H Host:multimod-test.example.com
-curl -s --fail-with-body ${PROXY_URL}/multimod-test -H Host:multimod-test.example.com
+curl -sf ${PROXY_URL}/ -H Host:multimod-test.example.com
+curl -sf ${PROXY_URL}/multimod-test -H Host:multimod-test.example.com
 
 echo "Multi Module Test Passed"

functional-tests/admin_tests/performance_test_prep.sh

@@ -18,7 +18,7 @@ dgate-cli -Vf domain create \
     namespace=test-ns1 priority:=100
 
 dgate-cli -Vf service create \
-    name=test-svc urls:="$TEST" \
+    name=test-svc urls:="$TEST_URL" \
     namespace=test-ns1 retries:=3 retryTimeout=50ms
 
 MOD_B64="$(base64 < $DIR/performance_test_prep.ts)"
@@ -53,9 +53,9 @@ dgate-cli -Vf route create \
     namespace=test-ns1
 
 
-curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:performance.example.com
+curl -sf ${PROXY_URL}/svctest -H Host:performance.example.com
 
-curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:performance.example.com
+curl -sf ${PROXY_URL}/modtest -H Host:performance.example.com
 
 curl -s ${PROXY_URL}/blank -H Host:performance.example.com
 

functional-tests/admin_tests/url_shortener_test.sh

@@ -39,6 +39,6 @@ JSON_RESP=$(curl -fsG -X POST \
 
 URL_ID=$(echo $JSON_RESP | jq -r '.id')
 
-curl -s --fail-with-body \
+curl -sf \
     ${PROXY_URL}/$URL_ID \
     -H Host:url_shortener.example.com

functional-tests/raft_tests/raft_test.sh

@@ -4,6 +4,7 @@ set -eo xtrace
 
 ADMIN_URL1=${ADMIN_URL1:-"http://localhost:9081"}
 PROXY_URL1=${PROXY_URL1:-"http://localhost:81"}
+TEST_URL1=${TEST_URL1:-"http://localhost:8081"}
 
 ADMIN_URL2=${ADMIN_URL2:-"http://localhost:9082"}
 PROXY_URL2=${PROXY_URL2:-"http://localhost:82"}
@@ -24,8 +25,11 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL1
 
-
-id=$(uuid)
+if ! command -v uuid > /dev/null; then
+    id=X$RANDOM-$RANDOM-$RANDOM
+else
+    id=$(uuid)
+fi
 
 dgate-cli -Vf namespace create name=ns-$id
 
@@ -34,7 +38,7 @@ dgate-cli -Vf domain create name=dm-$id \
 
 dgate-cli -Vf service create \
     name=svc-$id namespace=ns-$id \
-    urls="http://localhost:8081/$RANDOM"
+    urls="$TEST_URL1/$RANDOM"
 
 dgate-cli -Vf route create \
     name=rt-$id \
@@ -45,12 +49,12 @@ dgate-cli -Vf route create \
     preserveHost:=false \
     stripPath:=false
 
-curl -f $ADMIN_URL1/readyz
+curl -sf $ADMIN_URL1/readyz
 
 for i in {1..1}; do
     for j in {1..3}; do
         proxy_url=PROXY_URL$i
-        curl -f ${!proxy_url}/$id/$RANDOM-$j -H Host:$id.example.com
+        curl -sf ${!proxy_url}/$id/$RANDOM-$j -H Host:$id.example.com
     done
 done
 

internal/admin/routes/service_routes.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	urllib "net/url"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -50,13 +51,34 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C
 			util.JsonError(w, http.StatusBadRequest, "retry timeout must be greater than 0")
 			return
 		}
+		if len(svc.URLs) == 0 {
+			util.JsonError(w, http.StatusBadRequest, "urls are required")
+			return
+		} else {
+			for i, url := range svc.URLs {
+				errPrefix := fmt.Sprintf(`error on urls["%d"]: `, i)
+				if url, err := urllib.Parse(url); err != nil {
+					util.JsonError(w, http.StatusBadRequest, errPrefix+err.Error())
+				} else {
+					if url.Scheme == "" {
+						util.JsonError(w, http.StatusBadRequest, errPrefix+"url scheme cannot be empty")
+						return
+					}
+					if url.Host == "" {
+						util.JsonError(w, http.StatusBadRequest, errPrefix+"url host cannot be empty")
+						return
+					}
+				}
+			}
+		}
 		if svc.NamespaceName == "" {
 			if appConfig.DisableDefaultNamespace {
 				util.JsonError(w, http.StatusBadRequest, "namespace is required")
 				return
 			}
 			svc.NamespaceName = spec.DefaultNamespace.Name
 		}
+
 		cl := spec.NewChangeLog(&svc, svc.NamespaceName, spec.AddServiceCommand)
 		err = cs.ApplyChangeLog(cl)
 		if err != nil {

internal/proxy/change_log.go

@@ -74,17 +74,29 @@ func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) (
 				})
 			}
 		}()
-		if err = ps.processResource(cl); err != nil {
-			ps.logger.Error("decoding or processing change log", zap.Error(err))
-			return
+		if cl.Cmd.Resource() == spec.Documents {
+			if err = ps.processDocument(cl.Item.(*spec.Document), cl, store); err != nil {
+				ps.logger.Error("error processing document change log", zap.Error(err))
+				return
+			}
+		} else {
+			if err = ps.processResource(cl); err != nil {
+				ps.logger.Error("error processing change log", zap.Error(err))
+				return
+			}
 		}
 	}
 
 	// apply state changes to the proxy
 	if reload {
 		overrideReload := cl.Cmd.IsNoop() || ps.pendingChanges
 		if overrideReload || cl.Cmd.Resource().IsRelatedTo(spec.Routes) {
-			ps.logger.Debug("Reloading change log at...", zap.String("id", cl.ID))
+			ps.logger.Debug("Storing cached documents", zap.String("id", cl.ID))
+			if err := ps.storeCachedDocuments(); err != nil {
+				ps.logger.Error("error storing cached documents", zap.Error(err))
+				return err
+			}
+			ps.logger.Debug("Reloading change log", zap.String("id", cl.ID))
 			if err = ps.reconfigureState(cl); err != nil {
 				ps.logger.Error("Error registering change log", zap.Error(err))
 				return
@@ -150,11 +162,6 @@ func (ps *ProxyState) processResource(cl *spec.ChangeLog) (err error) {
 		if item, err = decode[spec.Collection](cl.Item); err == nil {
 			err = ps.processCollection(&item, cl)
 		}
-	case spec.Documents:
-		var item spec.Document
-		if item, err = decode[spec.Document](cl.Item); err == nil {
-			err = ps.processDocument(&item, cl)
-		}
 	case spec.Secrets:
 		var item spec.Secret
 		if item, err = decode[spec.Secret](cl.Item); err == nil {
@@ -253,17 +260,44 @@ func (ps *ProxyState) processCollection(col *spec.Collection, cl *spec.ChangeLog
 	return err
 }
 
-func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog) (err error) {
+var docCache = []*spec.Document{}
+
+func (ps *ProxyState) storeCachedDocuments() error {
+	err := ps.store.StoreDocuments(docCache)
+	if err != nil {
+		return err
+	}
+	docCache = []*spec.Document{}
+	return nil
+}
+
+func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog, store bool) (err error) {
 	if doc.NamespaceName == "" {
 		doc.NamespaceName = cl.Namespace
 	}
-	switch cl.Cmd.Action() {
-	case spec.Add:
-		err = ps.store.StoreDocument(doc)
-	case spec.Delete:
-		err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName)
-	default:
-		err = fmt.Errorf("unknown command: %s", cl.Cmd)
+	if store {
+		switch cl.Cmd.Action() {
+		case spec.Add:
+			err = ps.store.StoreDocument(doc)
+		case spec.Delete:
+			err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName)
+		default:
+			err = fmt.Errorf("unknown command: %s", cl.Cmd)
+		}
+	} else {
+		switch cl.Cmd.Action() {
+		case spec.Add:
+			docCache = append(docCache, doc)
+		case spec.Delete:
+			deletedIndex := sliceutil.BinarySearch(docCache, doc, func(doc1 *spec.Document, doc2 *spec.Document) bool {
+				return doc1.ID < doc2.ID
+			})
+			if deletedIndex >= 0 {
+				docCache = append(docCache[:deletedIndex], docCache[deletedIndex+1:]...)
+			}
+		default:
+			err = fmt.Errorf("unknown command: %s", cl.Cmd)
+		}
 	}
 	return err
 }
@@ -291,10 +325,10 @@ func (ps *ProxyState) restoreFromChangeLogs(directApply bool) error {
 		ps.logger.Info("restoring state change logs from storage", zap.Int("count", len(logs)))
 		// we might need to sort the change logs by timestamp
 		for _, cl := range logs {
-			// ps.logger.Debug("restoring change log",
-			// 	zap.Int("index", i),
-			// 	zap.Stringer("changeLog", cl.Cmd),
-			// )
+			// skip documents as they are persisted in the store
+			if cl.Cmd.Resource() == spec.Documents {
+				continue
+			}
 			if err = ps.processChangeLog(cl, false, false); err != nil {
 				return err
 			} else {