Launch smoke tests via ECS (#35)

* feat(cd-manager): rename task id param * feat(cd-manager): launch ecs task for smoke tests * feat(cd-manager): remove delay check from smoke test * feat: use job param id const * feat: timeout smoke tests after 15 mins * fix: update smoke test family prefix * clean: remove ECS prefix from variables * ci: add dagger update workflow * fix(ci): add job and steps
3box · Nov 23, 2022 · 34007c1 · 34007c1
1 parent 156413b
commit 34007c1
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 14 deletions.
diff --git a/cd/manager/jobmanager/jobManager.go b/cd/manager/jobmanager/jobManager.go
@@ -529,7 +529,7 @@ func (m *JobManager) prepareJob(jobState manager.JobState) (manager.Job, error)
 	case manager.JobType_TestE2E:
 		job = jobs.E2eTestJob(m.db, m.d, m.notifs, jobState)
 	case manager.JobType_TestSmoke:
-		job = jobs.SmokeTestJob(m.db, m.apiGw, m.notifs, jobState)
+		job = jobs.SmokeTestJob(m.db, m.d, m.notifs, jobState)
 	default:
 		genErr = fmt.Errorf("prepareJob: unknown job type: %s", manager.PrintJob(jobState))
 	}

diff --git a/cd/manager/jobs/anchor.go b/cd/manager/jobs/anchor.go
@@ -9,8 +9,8 @@ import (
 	"github.com/3box/pipeline-tools/cd/manager"
 )
 
+// Allow up to 3 hours for anchor workers to run
 const AnchorStalledTime = 3 * time.Hour
-const TaskIdParam = "id"
 
 var _ manager.Job = &anchorJob{}
 
@@ -41,11 +41,11 @@ func (a anchorJob) AdvanceJob() (manager.JobState, error) {
 		} else {
 			// Update the job stage and spawned task identifier
 			a.state.Stage = manager.JobStage_Started
-			a.state.Params[TaskIdParam] = id
+			a.state.Params[manager.JobParam_Id] = id
 			a.state.Params[manager.JobParam_Start] = time.Now().UnixMilli()
 		}
 	} else if a.state.Stage == manager.JobStage_Started {
-		if running, err := a.d.CheckTask("ceramic-"+a.env+"-cas", "", true, false, a.state.Params[TaskIdParam].(string)); err != nil {
+		if running, err := a.d.CheckTask("ceramic-"+a.env+"-cas", "", true, false, a.state.Params[manager.JobParam_Id].(string)); err != nil {
 			a.state.Stage = manager.JobStage_Failed
 			a.state.Params[manager.JobParam_Error] = err.Error()
 			log.Printf("anchorJob: error checking task running status: %v, %s", err, manager.PrintJob(a.state))
@@ -60,7 +60,7 @@ func (a anchorJob) AdvanceJob() (manager.JobState, error) {
 			return a.state, nil
 		}
 	} else if a.state.Stage == manager.JobStage_Waiting {
-		if stopped, err := a.d.CheckTask("ceramic-"+a.env+"-cas", "", false, false, a.state.Params[TaskIdParam].(string)); err != nil {
+		if stopped, err := a.d.CheckTask("ceramic-"+a.env+"-cas", "", false, false, a.state.Params[manager.JobParam_Id].(string)); err != nil {
 			a.state.Stage = manager.JobStage_Failed
 			a.state.Params[manager.JobParam_Error] = err.Error()
 			log.Printf("anchorJob: error checking task stopped status: %v, %s", err, manager.PrintJob(a.state))

diff --git a/cd/manager/jobs/smoke.go b/cd/manager/jobs/smoke.go
@@ -9,33 +9,71 @@ import (
 	"github.com/3box/pipeline-tools/cd/manager"
 )
 
+// Allow up to 15 minutes for smoke tests to run
+const SmokeTestFailureTime = 15 * time.Minute
+
+const ClusterName = "ceramic-qa-tests"
+const FamilyPrefix = "ceramic-qa-tests-smoke--"
+const ContainerName = "ceramic-qa-tests-smoke"
+const NetworkConfigurationParameter = "/ceramic-qa-tests-smoke/network_configuration"
+
 var _ manager.Job = &smokeTestJob{}
 
 type smokeTestJob struct {
 	state  manager.JobState
 	db     manager.Database
-	api    manager.ApiGw
+	d      manager.Deployment
 	notifs manager.Notifs
+	env    string
 }
 
-func SmokeTestJob(db manager.Database, api manager.ApiGw, notifs manager.Notifs, jobState manager.JobState) manager.Job {
-	return &smokeTestJob{jobState, db, api, notifs}
+func SmokeTestJob(db manager.Database, d manager.Deployment, notifs manager.Notifs, jobState manager.JobState) manager.Job {
+	return &smokeTestJob{jobState, db, d, notifs, os.Getenv("ENV")}
 }
 
 func (s smokeTestJob) AdvanceJob() (manager.JobState, error) {
 	if s.state.Stage == manager.JobStage_Queued {
-		resourceId := os.Getenv("SMOKE_TEST_RESOURCE_ID")
-		restApiId := os.Getenv("SMOKE_TEST_REST_API_ID")
-		if _, err := s.api.Invoke("GET", resourceId, restApiId, ""); err != nil {
+		// Launch smoke test
+		if id, err := s.d.LaunchTask(
+			ClusterName,
+			FamilyPrefix+s.env,
+			ContainerName,
+			NetworkConfigurationParameter,
+			nil); err != nil {
 			s.state.Stage = manager.JobStage_Failed
 			s.state.Params[manager.JobParam_Error] = err.Error()
-			log.Printf("smokeTestJob: error starting tests: %v, %s", err, manager.PrintJob(s.state))
+			log.Printf("smokeTestJob: error starting task: %v, %s", err, manager.PrintJob(s.state))
 		} else {
+			// Update the job stage and spawned task identifier
 			s.state.Stage = manager.JobStage_Started
+			s.state.Params[manager.JobParam_Id] = id
+			s.state.Params[manager.JobParam_Start] = time.Now().UnixMilli()
 		}
+	} else if manager.IsTimedOut(s.state, SmokeTestFailureTime) { // Smoke test did not finish in time
+		s.state.Stage = manager.JobStage_Failed
+		s.state.Params[manager.JobParam_Error] = manager.Error_Timeout
+		log.Printf("smokeTestJob: job run timed out: %s", manager.PrintJob(s.state))
 	} else if s.state.Stage == manager.JobStage_Started {
-		if time.Now().Add(-manager.DefaultWaitTime).After(s.state.Ts) {
-			// Since we're not monitoring for smoke test completion, give the tests some time to complete.
+		if running, err := s.d.CheckTask(ClusterName, "", true, false, s.state.Params[manager.JobParam_Id].(string)); err != nil {
+			s.state.Stage = manager.JobStage_Failed
+			s.state.Params[manager.JobParam_Error] = err.Error()
+			log.Printf("smokeTestJob: error checking task running status: %v, %s", err, manager.PrintJob(s.state))
+		} else if running {
+			s.state.Stage = manager.JobStage_Waiting
+		} else if manager.IsTimedOut(s.state, manager.DefaultWaitTime) { // Smoke test did not start running in time
+			s.state.Stage = manager.JobStage_Failed
+			s.state.Params[manager.JobParam_Error] = manager.Error_Timeout
+			log.Printf("smokeTestJob: job startup timed out: %s", manager.PrintJob(s.state))
+		} else {
+			// Return so we come back again to check
+			return s.state, nil
+		}
+	} else if s.state.Stage == manager.JobStage_Waiting {
+		if stopped, err := s.d.CheckTask(ClusterName, "", false, false, s.state.Params[manager.JobParam_Id].(string)); err != nil {
+			s.state.Stage = manager.JobStage_Failed
+			s.state.Params[manager.JobParam_Error] = err.Error()
+			log.Printf("smokeTestJob: error checking task stopped status: %v, %s", err, manager.PrintJob(s.state))
+		} else if stopped {
 			s.state.Stage = manager.JobStage_Completed
 		} else {
 			// Return so we come back again to check

diff --git a/cd/manager/models.go b/cd/manager/models.go
@@ -67,6 +67,7 @@ const (
 
 const (
 	JobParam_Component string = "component"
+	JobParam_Id        string = "id"
 	JobParam_Sha       string = "sha"
 	JobParam_Error     string = "error"
 	JobParam_Layout    string = "layout"