Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

spec refactor: use plan policy provider #1594

Merged
merged 1 commit into from
Jun 28, 2024
Merged

spec refactor: use plan policy provider #1594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions cli/cmd/digger/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ import (
core_backend "github.com/diggerhq/digger/cli/pkg/core/backend"
core_policy "github.com/diggerhq/digger/cli/pkg/core/policy"
"github.com/diggerhq/digger/cli/pkg/digger"
"github.com/diggerhq/digger/cli/pkg/storage"
"github.com/diggerhq/digger/cli/pkg/usage"
"github.com/diggerhq/digger/libs/comment_utils/reporting"
"github.com/diggerhq/digger/libs/comment_utils/summary"
"github.com/diggerhq/digger/libs/digger_config"
core_locking "github.com/diggerhq/digger/libs/locking"
orchestrator "github.com/diggerhq/digger/libs/orchestrator"
"github.com/diggerhq/digger/libs/storage"
"log"
"os"
)
Expand All @@ -26,7 +26,12 @@ func exec(actor string, projectName string, repoNamespace string, command string

}

planStorage := storage.NewPlanStorage("", "", "", actor, nil)
planStorage, err := storage.NewPlanStorage("", "", "", nil)
if err != nil {

usage.ReportErrorAndExit(actor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)

}

changedFiles, err := prService.GetChangedFiles(prNumber)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions cli/cmd/digger/run_spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ var runSpecCmd = &cobra.Command{
lib_spec.ReporterProvider{},
lib_spec.BackendApiProvider{},
lib_spec.PolicyProvider{},
lib_spec.PlanStorageProvider{},
comment_summary.CommentUpdaterProviderBasic{},
)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cli/pkg/core/execution/execution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"github.com/diggerhq/digger/libs/comment_utils/utils"
"github.com/diggerhq/digger/libs/locking"
"github.com/diggerhq/digger/libs/storage"
"github.com/diggerhq/digger/libs/terraform_utils"
"github.com/samber/lo"
"log"
Expand All @@ -14,7 +15,6 @@ import (
"strings"

"github.com/diggerhq/digger/cli/pkg/core/runners"
"github.com/diggerhq/digger/cli/pkg/core/storage"
"github.com/diggerhq/digger/cli/pkg/core/terraform"
"github.com/diggerhq/digger/libs/comment_utils/reporting"
configuration "github.com/diggerhq/digger/libs/digger_config"
Expand Down
2 changes: 1 addition & 1 deletion cli/pkg/digger/digger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
comment_updater "github.com/diggerhq/digger/libs/comment_utils/summary"
coreutils "github.com/diggerhq/digger/libs/comment_utils/utils"
locking2 "github.com/diggerhq/digger/libs/locking"
"github.com/diggerhq/digger/libs/storage"
"log"
"os"
"path"
Expand All @@ -17,7 +18,6 @@ import (
"github.com/diggerhq/digger/cli/pkg/core/execution"
"github.com/diggerhq/digger/cli/pkg/core/policy"
"github.com/diggerhq/digger/cli/pkg/core/runners"
"github.com/diggerhq/digger/cli/pkg/core/storage"
"github.com/diggerhq/digger/cli/pkg/core/terraform"
"github.com/diggerhq/digger/cli/pkg/usage"
utils "github.com/diggerhq/digger/cli/pkg/utils"
Expand Down
19 changes: 14 additions & 5 deletions cli/pkg/github/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/diggerhq/digger/cli/pkg/digger"
"github.com/diggerhq/digger/cli/pkg/drift"
github_models "github.com/diggerhq/digger/cli/pkg/github/models"
"github.com/diggerhq/digger/cli/pkg/storage"
"github.com/diggerhq/digger/cli/pkg/usage"
"github.com/diggerhq/digger/cli/pkg/utils"
"github.com/diggerhq/digger/libs/comment_utils/reporting"
Expand All @@ -20,6 +19,7 @@ import (
"github.com/diggerhq/digger/libs/orchestrator"
dg_github "github.com/diggerhq/digger/libs/orchestrator/github"
"github.com/diggerhq/digger/libs/orchestrator/scheduler"
"github.com/diggerhq/digger/libs/storage"
"github.com/google/go-github/v61/github"
"github.com/samber/lo"
"gopkg.in/yaml.v3"
Expand Down Expand Up @@ -185,7 +185,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
commentUpdater.UpdateComment(serializedBatch.Jobs, serializedBatch.PrNumber, &githubPrService, commentId64)
digger.UpdateAggregateStatus(serializedBatch, &githubPrService)

planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, jobSpec.PullRequestNumber)
planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, jobSpec.PullRequestNumber)
if err != nil {
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
}

if err != nil {
serializedBatch, reportingError := backendApi.ReportProjectJobStatus(repoName, jobSpec.ProjectName, inputs.Id, "failed", time.Now(), nil, "", "")
Expand Down Expand Up @@ -267,7 +270,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh

stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)

planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, nil)
planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, nil)
if err != nil {
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
}

jobs := orchestrator.Job{
ProjectName: project,
Expand All @@ -285,7 +291,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
StateEnvVars: stateEnvVars,
CommandEnvVars: commandEnvVars,
}
err := digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir)
err = digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir)
if err != nil {
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to run commands. %s", err), 8)
}
Expand Down Expand Up @@ -383,7 +389,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to set job output. Exiting. %s", err), 4)
}

planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, &prNumber)
planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, &prNumber)
if err != nil {
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
}

reporter := &reporting.CiReporter{
CiService: &githubPrService,
Expand Down
9 changes: 4 additions & 5 deletions cli/pkg/integration/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"github.com/diggerhq/digger/libs/locking"
"github.com/diggerhq/digger/libs/locking/aws"
"github.com/diggerhq/digger/libs/storage"
"log"
"math/rand"
"os"
Expand All @@ -15,8 +16,6 @@ import (
"github.com/diggerhq/digger/cli/pkg/core/terraform"
"github.com/diggerhq/digger/cli/pkg/digger"
"github.com/diggerhq/digger/cli/pkg/github/models"
"github.com/diggerhq/digger/cli/pkg/storage"
"github.com/diggerhq/digger/cli/pkg/utils"
"github.com/diggerhq/digger/libs/comment_utils/reporting"
configuration "github.com/diggerhq/digger/libs/digger_config"
dg_github "github.com/diggerhq/digger/libs/orchestrator/github"
Expand Down Expand Up @@ -398,7 +397,7 @@ func TestHappyPath(t *testing.T) {
event := ghEvent.(github.PullRequestEvent)
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig)
assert.NoError(t, err)
zipManager := utils.Zipper{}
zipManager := storage.Zipper{}
planStorage := &storage.GithubPlanStorage{
Client: github.NewTokenClient(context.Background(), ghToken),
Owner: repoOwner,
Expand Down Expand Up @@ -556,7 +555,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
assert.NoError(t, err)

zipManager := utils.Zipper{}
zipManager := storage.Zipper{}
planStorage := &storage.GithubPlanStorage{
Client: github.NewTokenClient(context.Background(), ghToken),
Owner: repoOwner,
Expand Down Expand Up @@ -777,7 +776,7 @@ workflows:
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
assert.NoError(t, err)

zipManager := utils.Zipper{}
zipManager := storage.Zipper{}
planStorage := &storage.GithubPlanStorage{
Client: github.NewTokenClient(context.Background(), ghToken),
Owner: repoOwner,
Expand Down
9 changes: 6 additions & 3 deletions cli/pkg/spec/spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package spec
import (
"fmt"
"github.com/diggerhq/digger/cli/pkg/digger"
storage2 "github.com/diggerhq/digger/cli/pkg/storage"
"github.com/diggerhq/digger/cli/pkg/usage"
comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary"
"github.com/diggerhq/digger/libs/digger_config"
Expand All @@ -23,6 +22,7 @@ func RunSpec(
reporterProvider spec.ReporterProvider,
backedProvider spec.BackendApiProvider,
policyProvider spec.PolicyProvider,
PlanStorageProvider spec.PlanStorageProvider,
commentUpdaterProvider comment_summary.CommentUpdaterProvider,
) error {

Expand Down Expand Up @@ -72,8 +72,11 @@ func RunSpec(
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get comment updater: %v", err), 8)
}

planStorage := storage2.NewPlanStorage("", "", "", "", nil)

planStorage, err := PlanStorageProvider.GetPlanStorage(spec.VCS.RepoOwner, spec.VCS.RepoName, *spec.Job.PullRequestNumber)
if err != nil {
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get plan storage: %v", err), 8)
}

jobs := []orchestrator.Job{job}

fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
Expand Down
8 changes: 4 additions & 4 deletions cli_e2e/plan_storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package cli_e2e
import (
"fmt"
"github.com/diggerhq/digger/libs/locking/gcp"
storage2 "github.com/diggerhq/digger/libs/storage"
"log"
"os"
"strings"
"testing"

"github.com/diggerhq/digger/cli/pkg/storage"
"github.com/stretchr/testify/assert"
)

Expand All @@ -28,7 +28,7 @@ func TestGCPPlanStorageStorageAndRetrieval(t *testing.T) {
fmt.Printf("getting bucket")
bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET"))
bucket := client.Bucket(bucketName)
planStorage := &storage.PlanStorageGcp{
planStorage := &storage2.PlanStorageGcp{
Client: client,
Bucket: bucket,
Context: ctx,
Expand Down Expand Up @@ -60,13 +60,13 @@ func TestAWSPlanStorageStorageAndRetrieval(t *testing.T) {
defer os.Remove(file.Name())

log.Printf("getting AWS S3 client")
ctx, client, err := storage.GetAWSStorageClient()
ctx, client, err := storage2.GetAWSStorageClient()
if err != nil {
t.Errorf("failed to get AWS storage client: %v", err)
}
bucketName := strings.ToLower(os.Getenv("AWS_S3_BUCKET"))
log.Printf("using AWS S3 bucket found by env var 'AWS_S3_BUCKET': %s", bucketName)
planStorage := &storage.PlanStorageAWS{
planStorage := &storage2.PlanStorageAWS{
Client: client,
Bucket: bucketName,
Context: ctx,
Expand Down
1 change: 1 addition & 0 deletions ee/cli/cmd/digger/run_spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ var runSpecCmd = &cobra.Command{
lib_spec.ReporterProvider{},
lib_spec.BackendApiProvider{},
lib_spec.PolicyProvider{},
lib_spec.PlanStorageProvider{},
comment_summary.CommentUpdaterProviderBasic{},
)
if err != nil {
Expand Down
14 changes: 13 additions & 1 deletion libs/spec/providers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/diggerhq/digger/libs/locking/gcp"
"github.com/diggerhq/digger/libs/orchestrator"
"github.com/diggerhq/digger/libs/orchestrator/github"
storage2 "github.com/diggerhq/digger/libs/storage"
"github.com/samber/lo"
"log"
"net/http"
Expand Down Expand Up @@ -173,7 +174,7 @@ func (v VCSProvider) GetPrService(vcsSpec VcsSpec) (orchestrator.PullRequestServ
case "github":
token := os.Getenv("GITHUB_TOKEN")
if token == "" {
return nil, fmt.Errorf("failed to get githbu service: GITHUB_TOKEN not specified")
return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified")
}
return github.GithubServiceProviderBasic{}.NewService(token, vcsSpec.RepoName, vcsSpec.RepoOwner)
default:
Expand All @@ -198,3 +199,14 @@ func (p PolicyProvider) GetPolicyProvider(policySpec PolicySpec, diggerHost stri
return nil, fmt.Errorf("could not find ")
}
}

type PlanStorageProvider struct{}

func (p PlanStorageProvider) GetPlanStorage(repoOwner string, repositoryName string, prNumber int) (storage2.PlanStorage, error) {
token := os.Getenv("GITHUB_TOKEN")
if token == "" {
return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified")
}
storage, err := storage2.NewPlanStorage(token, repoOwner, repositoryName, &prNumber)
return storage, err
}
0 cli/pkg/storage/aws_plan_storage.go → libs/storage/aws_plan_storage.go
View file
Open in desktop
File renamed without changes.
0 cli/pkg/storage/aws_plan_storage_test.go → libs/storage/aws_plan_storage_test.go
View file
Open in desktop
File renamed without changes.
0 cli/pkg/storage/gcp_plan_storage.go → libs/storage/gcp_plan_storage.go
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion cli/pkg/utils/io.go → libs/storage/io.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package utils
package storage

import (
"archive/zip"
Expand Down
2 changes: 1 addition & 1 deletion cli/pkg/utils/io_test.go → libs/storage/io_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package utils
package storage

import (
"archive/zip"
Expand Down
21 changes: 9 additions & 12 deletions cli/pkg/storage/plan_storage.go → libs/storage/plan_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/diggerhq/digger/cli/pkg/core/storage"
"github.com/diggerhq/digger/cli/pkg/usage"
"github.com/diggerhq/digger/libs/locking/gcp"
"io"
"log"
Expand All @@ -16,8 +14,6 @@ import (
"os/exec"
"strings"

"github.com/diggerhq/digger/cli/pkg/utils"

"github.com/google/go-github/v61/github"
)

Expand All @@ -26,7 +22,7 @@ type GithubPlanStorage struct {
Owner string
RepoName string
PullRequestNumber int
ZipManager utils.Zipper
ZipManager Zipper
}

func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName string, storedPlanFilePath string) error {
Expand Down Expand Up @@ -208,13 +204,14 @@ func getLatestArtifactWithName(artifacts []*github.Artifact, name string) *githu
return latest
}

func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, requestedBy string, prNumber *int) storage.PlanStorage {
var planStorage storage.PlanStorage
// TODO: refactor this function to make it fully parametarised and no reliance on env variables
func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, prNumber *int) (PlanStorage, error) {
var planStorage PlanStorage

uploadDestination := strings.ToLower(os.Getenv("PLAN_UPLOAD_DESTINATION"))
switch {
case uploadDestination == "github":
zipManager := utils.Zipper{}
zipManager := Zipper{}
planStorage = &GithubPlanStorage{
Client: github.NewTokenClient(context.Background(), ghToken),
Owner: ghRepoOwner,
Expand All @@ -226,7 +223,7 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string,
ctx, client := gcp.GetGoogleStorageClient()
bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET"))
if bucketName == "" {
usage.ReportErrorAndExit(requestedBy, "GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET is not defined", 9)
return nil, fmt.Errorf("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET is not defined")
}
bucket := client.Bucket(bucketName)
planStorage = &PlanStorageGcp{
Expand All @@ -237,11 +234,11 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string,
case uploadDestination == "aws":
ctx, client, err := GetAWSStorageClient()
if err != nil {
usage.ReportErrorAndExit(requestedBy, fmt.Sprintf("Failed to create AWS storage client: %s", err), 9)
return nil, fmt.Errorf(fmt.Sprintf("Failed to create AWS storage client: %s", err))
}
bucketName := strings.ToLower(os.Getenv("AWS_S3_BUCKET"))
if bucketName == "" {
usage.ReportErrorAndExit(requestedBy, "AWS_S3_BUCKET is not defined", 9)
return nil, fmt.Errorf("AWS_S3_BUCKET is not defined")
}
planStorage = &PlanStorageAWS{
Context: ctx,
Expand All @@ -252,5 +249,5 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string,
//TODO implement me
}

return planStorage
return planStorage, nil
}
0 cli/pkg/core/storage/plan_storage.go → libs/storage/storage.go
View file
Open in desktop
File renamed without changes.
Loading