BCDA-7900: Set CCLF/Opt Out imports in dev/test to only import specif…

…ic subfolders (#974)

## 🎫 Ticket

https://jira.cms.gov/browse/BCDA-7900

## 🛠 Changes

- Update dev/test CCLF and Opt Out S3 Importers to only import specific
subfolders

## ℹ️ Context

As part of improving integration tests, it was observed that dev and
test were competing for the same files in BFD Test S3. This changes the
app code for those environments to avoid competition and only look for
files in environment-relevant subfolders (e.g. `bcda/in/test/` instead
of `bcda/in/`.)

## 🧪 Validation

See #963 for successful tests, yay!

bcda/cclf/s3_fileprocessor.go

@@ -33,6 +33,12 @@ func (processor *S3FileProcessor) LoadCclfFiles(path string) (cclfMap map[string
 			continue
 		}
 
+		// ignore files for different environments, and don't add it to the skipped count
+		if !optout.IsForCurrentEnv(*obj.Key) {
+			processor.Handler.Infof("Skipping file for different environment: %s/%s", bucket, *obj.Key)
+			continue
+		}
+
 		// validate the top level zipped folder
 		cmsID, err := getCMSID(*obj.Key)
 		if err != nil {

bcda/cclf/s3_fileprocessor_test.go

@@ -89,6 +89,20 @@ func (s *S3ProcessorTestSuite) TestLoadCclfFiles() {
 	}
 }
 
+func (s *S3ProcessorTestSuite) TestLoadCclfFiles_SkipOtherEnvs() {
+	cleanupEnvVars := testUtils.SetEnvVars(s.T(), []testUtils.EnvVar{{Name: "ENV", Value: "someenv"}})
+	defer cleanupEnvVars()
+
+	s3Bucket, cleanupS3 := testUtils.CreateZipsInS3(s.T(), testUtils.ZipInput{ZipName: "blah/not-someenv/T.BCD.A0001.ZCY18.D181120.T1000000", CclfNames: []string{"test", "zip"}})
+	defer cleanupS3()
+
+	cclfMap, skipped, failure, err := s.processor.LoadCclfFiles(s3Bucket)
+	assert.Nil(s.T(), err)
+	assert.Equal(s.T(), 0, skipped)
+	assert.Equal(s.T(), 0, failure)
+	assert.Empty(s.T(), cclfMap)
+}
+
 func (s *S3ProcessorTestSuite) TestLoadCclfFiles_SingleFile() {
 	cmsID, key := "A0001", metadataKey{perfYear: 18, fileType: models.FileTypeDefault}
 	tests := []struct {

optout/utils.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/CMSgov/bcda-app/conf"
 	"github.com/pkg/errors"
 )
 
@@ -31,6 +33,13 @@ func ParseMetadata(filename string) (OptOutFilenameMetadata, error) {
 		return metadata, err
 	}
 
+	// ignore files for different environments
+	if !IsForCurrentEnv(filename) {
+		fmt.Printf("Skipping file for different environment: %s.\n", filename)
+		err := fmt.Errorf("Skipping file for different environment: %s", filename)
+		return metadata, err
+	}
+
 	filenameDate := matches[3]
 	t, err := time.Parse("D060102.T150405", filenameDate)
 	if err != nil || t.IsZero() {
@@ -104,3 +113,16 @@ func ConvertDt(s string) (time.Time, error) {
 	}
 	return t, nil
 }
+
+// Checks if the given S3 filePath is for the current environment; this is necessary for lower environments
+// since they share a single BFD S3 bucket and will upload files under a subdirectory for the given env.
+func IsForCurrentEnv(filePath string) bool {
+	env := conf.GetEnv("ENV")
+
+	// We do not expect or require subdirectories for local dev or production; always return true.
+	if env != "dev" && env != "test" {
+		return true
+	}
+
+	return strings.Contains(filePath, fmt.Sprintf("/%s/", env))
+}

optout/utils_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/CMSgov/bcda-app/conf"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 )
@@ -13,6 +14,10 @@ type OptOutTestSuite struct {
 	suite.Suite
 }
 
+func TestOptOutTestSuite(t *testing.T) {
+	suite.Run(t, new(OptOutTestSuite))
+}
+
 func (s *OptOutTestSuite) TestParseMetadata() {
 	assert := assert.New(s.T())
 
@@ -31,6 +36,39 @@ func (s *OptOutTestSuite) TestParseMetadata() {
 	assert.Nil(err)
 }
 
+func (s *OptOutTestSuite) TestParseMetadata_CorrectEnv() {
+	assert := assert.New(s.T())
+
+	originalValue := conf.GetEnv("ENV")
+	if err := conf.SetEnv(s.T(), "ENV", "someenv"); err != nil {
+		assert.Failf("Error %s env value %s to %s\n", err.Error(), "ENV", "someenv")
+	}
+	defer func() {
+		conf.SetEnv(s.T(), "ENV", originalValue)
+	}()
+
+	expTime, _ := time.Parse(time.RFC3339, "2019-12-20T21:09:42Z")
+	metadata, err := ParseMetadata("blah/someenv/T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420")
+	assert.Equal("T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420", metadata.Name)
+	assert.Equal(expTime.Format("010203040506"), metadata.Timestamp.Format("010203040506"))
+	assert.Nil(err)
+}
+
+func (s *OptOutTestSuite) TestParseMetadata_DifferentEnv() {
+	assert := assert.New(s.T())
+
+	originalValue := conf.GetEnv("ENV")
+	if err := conf.SetEnv(s.T(), "ENV", "someenv"); err != nil {
+		assert.Failf("Error %s env value %s to %s\n", err.Error(), "ENV", "someenv")
+	}
+	defer func() {
+		conf.SetEnv(s.T(), "ENV", originalValue)
+	}()
+
+	_, err := ParseMetadata("blah/not-someenv/T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420")
+	assert.EqualError(err, "Skipping file for different environment: blah/not-someenv/T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420")
+}
+
 func (s *OptOutTestSuite) TestParseMetadata_InvalidData() {
 	assert := assert.New(s.T())