Combination of fixes/debug for e2e flakiness

* Retries on UDP packet sending
* Failing on UDP packet sending dumps gameserver details
* Drop parallelism on e2e tests from 32 to 16.

build/e2e-image/e2e.sh

@@ -20,5 +20,5 @@ set -e
 echo "installing current release"
 DOCKER_RUN= make install FEATURE_GATES='"'$FEATURES'"'
 echo "starting e2e test"
-DOCKER_RUN= make test-e2e ARGS=-parallel=32 FEATURE_GATES='"'$FEATURES'"'
+DOCKER_RUN= make test-e2e ARGS=-parallel=16 FEATURE_GATES='"'$FEATURES'"'
 echo "completed e2e test"

test/e2e/fleet_test.go

@@ -1155,7 +1155,7 @@ func TestFleetRecreateGameServers(t *testing.T) {
 			for _, gs := range list.Items {
 				gs := gs
 				var reply string
-				reply, err := e2e.SendGameServerUDP(&gs, "EXIT")
+				reply, err := framework.SendGameServerUDP(t, &gs, "EXIT")
 				if err != nil {
 					t.Fatalf("Could not message GameServer: %v", err)
 				}
@@ -1167,7 +1167,7 @@ func TestFleetRecreateGameServers(t *testing.T) {
 			for _, gs := range list.Items {
 				gs := gs
 				var reply string
-				reply, err := e2e.SendGameServerUDP(&gs, "UNHEALTHY")
+				reply, err := framework.SendGameServerUDP(t, &gs, "UNHEALTHY")
 				if err != nil {
 					t.Fatalf("Could not message GameServer: %v", err)
 				}
@@ -1332,7 +1332,7 @@ func TestFleetAggregatedPlayerStatus(t *testing.T) {
 		totalCapacity += capacity
 
 		msg := fmt.Sprintf("PLAYER_CAPACITY %d", capacity)
-		reply, err := e2e.SendGameServerUDP(gs, msg)
+		reply, err := framework.SendGameServerUDP(t, gs, msg)
 		if err != nil {
 			t.Fatalf("Could not message GameServer: %v", err)
 		}
@@ -1344,7 +1344,7 @@ func TestFleetAggregatedPlayerStatus(t *testing.T) {
 			logrus.WithField("msg", msg).WithField("gs", gs.ObjectMeta.Name).Info("Sending Player Connect")
 			// retry on failure. Will stop flakiness of UDP packets being sent/received.
 			err := wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
-				reply, err := e2e.SendGameServerUDP(gs, msg)
+				reply, err := framework.SendGameServerUDP(t, gs, msg)
 				if err != nil {
 					logrus.WithError(err).Warn("error with udp packet")
 					return false, nil

test/e2e/framework/framework.go

@@ -466,23 +466,24 @@ func (f *Framework) CreateAndApplyAllocation(t *testing.T, flt *agonesv1.Fleet)
 // SendGameServerUDP sends a message to a gameserver and returns its reply
 // finds the first udp port from the spec to send the message to,
 // returns error if no Ports were allocated
-func SendGameServerUDP(gs *agonesv1.GameServer, msg string) (string, error) {
+func (f *Framework) SendGameServerUDP(t *testing.T, gs *agonesv1.GameServer, msg string) (string, error) {
 	if len(gs.Status.Ports) == 0 {
 		return "", errors.New("Empty Ports array")
 	}
 
 	// use first udp port
 	for _, p := range gs.Spec.Ports {
 		if p.Protocol == corev1.ProtocolUDP {
-			return SendGameServerUDPToPort(gs, p.Name, msg)
+			return f.SendGameServerUDPToPort(t, gs, p.Name, msg)
 		}
 	}
 	return "", errors.New("No UDP ports")
 }
 
 // SendGameServerUDPToPort sends a message to a gameserver at the named port and returns its reply
 // returns error if no Ports were allocated or a port of the specified name doesn't exist
-func SendGameServerUDPToPort(gs *agonesv1.GameServer, portName string, msg string) (string, error) {
+func (f *Framework) SendGameServerUDPToPort(t *testing.T, gs *agonesv1.GameServer, portName string, msg string) (string, error) {
+	log := TestLogger(t)
 	if len(gs.Status.Ports) == 0 {
 		return "", errors.New("Empty Ports array")
 	}
@@ -493,42 +494,59 @@ func SendGameServerUDPToPort(gs *agonesv1.GameServer, portName string, msg strin
 		}
 	}
 	address := fmt.Sprintf("%s:%d", gs.Status.Address, port.Port)
-	return SendUDP(address, msg)
-}
+	reply, err := f.SendUDP(t, address, msg)
 
-// SendUDP sends a message to an address, and returns its reply if
-// it returns one in 30 seconds
-func SendUDP(address, msg string) (string, error) {
-	conn, err := net.Dial("udp", address)
 	if err != nil {
-		return "", errors.Wrap(err, "could not dial GameServer address")
+		log.WithField("gs", gs.ObjectMeta.Name).WithField("status", fmt.Sprintf("%+v", gs.Status)).Info("Failed to send UDP packet to GameServer. Dumping Events!")
+		f.LogEvents(t, log, gs.ObjectMeta.Namespace, gs)
 	}
-	defer func() {
-		err = conn.Close()
-	}()
 
+	return reply, err
+}
+
+// SendUDP sends a message to an address, and returns its reply if
+// it returns one in 10 seconds. Will retry 5 times, in case UDP packets drop.
+func (f *Framework) SendUDP(t *testing.T, address, msg string) (string, error) {
+	log := TestLogger(t).WithField("address", address)
+	b := make([]byte, 1024)
+	var n int
 	// sometimes we get I/O timeout, so let's do a retry
-	err = wait.PollImmediate(time.Second, 5*time.Second, func() (bool, error) {
-		_, err := conn.Write([]byte(msg))
+	err := wait.PollImmediate(time.Second, 5*time.Second, func() (bool, error) {
+
+		conn, err := net.Dial("udp", address)
+		if err != nil {
+			log.WithError(err).Info("could not dial address")
+			return false, nil
+		}
+
+		defer func() {
+			err = conn.Close()
+		}()
+
+		_, err = conn.Write([]byte(msg))
 		if err != nil {
-			logrus.WithError(err).Error("could not write message to GameServer")
+			log.WithError(err).Info("could not write message to address")
+			return false, nil
+		}
+
+		err = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
+		if err != nil {
+			log.WithError(err).Info("Could not set read deadline")
+			return false, nil
+		}
+
+		n, err = conn.Read(b)
+		if err != nil {
+			log.WithError(err).Info("Could not read from address")
 		}
+
 		return err == nil, nil
 	})
-	if err != nil {
-		return "", errors.Wrap(err, "could not send message to GameServer after retries")
-	}
-
-	b := make([]byte, 1024)
 
-	err = conn.SetReadDeadline(time.Now().Add(30 * time.Second))
-	if err != nil {
-		return "", errors.Wrap(err, "could not set read deadline")
-	}
-	n, err := conn.Read(b)
 	if err != nil {
-		return "", errors.Wrap(err, "could not read response from the GameServer")
+		return "", errors.Wrap(err, "timed out attempting to send UDP packet to address")
 	}
+
 	return string(b[:n]), nil
 }