From 3cb98dc83ca20afb4cac536e79c4ee6c9db1fd35 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 29 Jul 2024 15:27:01 +0100 Subject: [PATCH 1/3] temp commit --- cli/pkg/spec/spec.go | 2 + dgctl/cmd/exec.go | 89 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index d1ec726d7..528ebcb55 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -200,7 +200,9 @@ func RunSpecManualCommand( commentUpdater := comment_summary.NoopCommentUpdater{} // TODO: do not require conversion to gh service + log.Printf("<========= DIGGER RUNNING IN MANUAL MODE =========>") allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, false, false, commentId, currentDir) + log.Printf("<========= DIGGER COMPLETED =========>") if err != nil || allAppliesSuccess == false { usage.ReportErrorAndExit(spec.VCS.RepoOwner, "Terraform execution failed", 1) } diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index 16db8d7a5..caf5765e3 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -76,6 +76,22 @@ func getRepoFullname() (string, error) { return repoFullname, nil } +func GetUrlContents(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("%v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("%v", err) + } + + content := string(body) + return content, nil +} + func GetSpec(diggerUrl string, authToken string, command string, actor string, projectMarshalled string, diggerConfigMarshalled string, repoFullName string, defaultBanch string, prBranch string) ([]byte, error) { payload := spec.GetSpecPayload{ Command: command, @@ -132,32 +148,59 @@ func pushToBranch(prBranch string) error { return err } -func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, repoName string, diggerJobID string) (*int64, *string, error) { +func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, repoName string, diggerJobID string) (*int64, *int64, *string, error) { timeFilter := time.Now().Add(-5 * time.Minute) runs, _, err := client.Actions.ListRepositoryWorkflowRuns(context.Background(), repoOwner, repoName, &github.ListWorkflowRunsOptions{ Created: ">=" + timeFilter.Format(time.RFC3339), }) if err != nil { - return nil, nil, fmt.Errorf("error listing workflow runs %v", err) + return nil, nil, nil, fmt.Errorf("error listing workflow runs %v", err) } for _, workflowRun := range runs.WorkflowRuns { workflowjobs, _, err := client.Actions.ListWorkflowJobs(context.Background(), repoOwner, repoName, *workflowRun.ID, nil) if err != nil { - return nil, nil, fmt.Errorf("error listing workflow jobs for run %v %v", workflowRun.ID, err) + return nil, nil, nil, fmt.Errorf("error listing workflow jobs for run %v %v", workflowRun.ID, err) } for _, workflowjob := range workflowjobs.Jobs { for _, step := range workflowjob.Steps { + //log.Printf("workflowRun: %v workflowJob %v workflowjobName: %v workflowStep %v", *workflowRun.ID, *workflowjob.ID, *workflowjob.Name, *step.Name) + if strings.Contains(*step.Name, diggerJobID) { url := fmt.Sprintf("https://github.com/%v/%v/actions/runs/%v", repoOwner, repoName, *workflowRun.ID) - return workflowRun.ID, &url, nil + return workflowRun.ID, workflowjob.ID, &url, nil } } } } - return nil, nil, fmt.Errorf("workflow not found") + return nil, nil, nil, fmt.Errorf("workflow not found") +} + +func cleanupDiggerOutput(output string) string { + + startingDelimeter := "<========= DIGGER RUNNING IN MANUAL MODE =========>" + endingDelimiter := "<========= DIGGER COMPLETED =========>" + + startPos := 0 + endPos := len(output) + // removes output of terraform -version command that terraform-exec executes on every run + i := strings.Index(output, startingDelimeter) + if i != -1 { + startPos = i + } + + e := strings.Index(output, endingDelimiter) + if e != -1 { + endPos = e + } + + // This should not happen but in case we get here we avoid slice bounds out of range exception by resetting endPos + if endPos <= startPos { + endPos = len(output) + } + return output[startPos:endPos] } // validateCmd represents the validate command @@ -169,6 +212,12 @@ var execCmd = &cobra.Command{ var execConfig execConfig viperExec.Unmarshal(&execConfig) log.Printf("%v - %v ", execConfig.Project, execConfig.Command) + + if execConfig.Command != "digger plan" { + log.Printf("ERROR: currently only 'digger plan' supported with exec command") + os.Exit(1) + } + config, _, _, err := digger_config.LoadDiggerConfig("./", true, nil) if err != nil { log.Printf("Invalid digger config file: %v. Exiting.", err) @@ -266,16 +315,42 @@ var execCmd = &cobra.Command{ repoOwner, repoName, _ := strings.Cut(repoFullname, "/") var logsUrl *string var runId *int64 + var jobId *int64 for { - runId, logsUrl, err = GetWorkflowIdAndUrlFromDiggerJobId(client, repoOwner, repoName, spec.JobId) + runId, jobId, logsUrl, err = GetWorkflowIdAndUrlFromDiggerJobId(client, repoOwner, repoName, spec.JobId) + log.Printf("=============") if err == nil { break } time.Sleep(time.Second * 1) } - log.Printf("logs url: %v runId %v", *logsUrl, *runId) + log.Printf("waiting for logs to be available, you can view job in this url: %v runId %v", *logsUrl, *runId) + log.Printf("......") + + for { + j, _, err := client.Actions.GetWorkflowJobByID(context.Background(), repoOwner, repoName, *jobId) + if err != nil { + log.Printf("GetWorkflowJobByID error: %v", err) + } + log.Printf("job status: %v", *j.Status) + if *j.Status == "completed" { + break + } + time.Sleep(time.Second * 1) + } + + logs, _, err := client.Actions.GetWorkflowJobLogs(context.Background(), repoOwner, repoName, *jobId, 1) + + log.Printf("stereaming logs from: %v || %v", logs, err) + logsContent, err := GetUrlContents(logs.String()) + + if err != nil { + log.Printf("error while fetching logs: %v", err) + os.Exit(1) + } + log.Printf("logsContent is: %v", logsContent) }, } From c3bd972101fa2f996bdf828c2684983737c00596 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 29 Jul 2024 15:40:43 +0100 Subject: [PATCH 2/3] logs streaming from dgctl --- dgctl/cmd/exec.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index caf5765e3..6a7725e83 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -188,7 +188,7 @@ func cleanupDiggerOutput(output string) string { // removes output of terraform -version command that terraform-exec executes on every run i := strings.Index(output, startingDelimeter) if i != -1 { - startPos = i + startPos = i + len(startingDelimeter) } e := strings.Index(output, endingDelimiter) @@ -318,7 +318,6 @@ var execCmd = &cobra.Command{ var jobId *int64 for { runId, jobId, logsUrl, err = GetWorkflowIdAndUrlFromDiggerJobId(client, repoOwner, repoName, spec.JobId) - log.Printf("=============") if err == nil { break } @@ -331,9 +330,9 @@ var execCmd = &cobra.Command{ for { j, _, err := client.Actions.GetWorkflowJobByID(context.Background(), repoOwner, repoName, *jobId) if err != nil { - log.Printf("GetWorkflowJobByID error: %v", err) + log.Printf("GetWorkflowJobByID error: %v please view the logs in the job directly", err) + os.Exit(1) } - log.Printf("job status: %v", *j.Status) if *j.Status == "completed" { break } @@ -342,15 +341,15 @@ var execCmd = &cobra.Command{ logs, _, err := client.Actions.GetWorkflowJobLogs(context.Background(), repoOwner, repoName, *jobId, 1) - log.Printf("stereaming logs from: %v || %v", logs, err) + log.Printf("streaming logs from remote job:") logsContent, err := GetUrlContents(logs.String()) if err != nil { log.Printf("error while fetching logs: %v", err) os.Exit(1) } - - log.Printf("logsContent is: %v", logsContent) + cleanedLogs := cleanupDiggerOutput(logsContent) + log.Printf("logsContent is: %v", cleanedLogs) }, } From 114c982f4d34526431d6f33c1489f898c8818671 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 29 Jul 2024 15:43:36 +0100 Subject: [PATCH 3/3] remove logs line --- dgctl/cmd/exec.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/dgctl/cmd/exec.go b/dgctl/cmd/exec.go index 6a7725e83..0c4dcae26 100644 --- a/dgctl/cmd/exec.go +++ b/dgctl/cmd/exec.go @@ -165,8 +165,6 @@ func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, for _, workflowjob := range workflowjobs.Jobs { for _, step := range workflowjob.Steps { - //log.Printf("workflowRun: %v workflowJob %v workflowjobName: %v workflowStep %v", *workflowRun.ID, *workflowjob.ID, *workflowjob.Name, *step.Name) - if strings.Contains(*step.Name, diggerJobID) { url := fmt.Sprintf("https://github.com/%v/%v/actions/runs/%v", repoOwner, repoName, *workflowRun.ID) return workflowRun.ID, workflowjob.ID, &url, nil