diff --git a/bcda/cclf/s3_fileprocessor_test.go b/bcda/cclf/s3_fileprocessor_test.go index 92c2b4886..c69e1987a 100644 --- a/bcda/cclf/s3_fileprocessor_test.go +++ b/bcda/cclf/s3_fileprocessor_test.go @@ -89,6 +89,47 @@ func (s *S3ProcessorTestSuite) TestLoadCclfFiles() { } } +func (s *S3ProcessorTestSuite) TestLoadCclfFiles_SingleFile() { + cmsID, key := "A0001", metadataKey{perfYear: 18, fileType: models.FileTypeDefault} + tests := []struct { + path string + filename string + numCCLFFiles int // Expected count for the cmsID, perfYear above + skipped int + failure int + numCCLF0 int // Expected count for the cmsID, perfYear above + numCCLF8 int // Expected count for the cmsID, perfYear above + }{ + {"cclf/archives/valid2/", "T.BCD.A0001.ZCY18.D181120.T1000000", 2, 0, 0, 1, 1}, + } + + for _, tt := range tests { + s.T().Run(tt.path, func(t *testing.T) { + bucketName, cleanup := testUtils.CopyToS3(s.T(), filepath.Join(s.basePath, tt.path)) + defer cleanup() + + cclfMap, skipped, failure, err := s.processor.LoadCclfFiles(filepath.Join(bucketName, tt.path, tt.filename)) + cclfFiles := cclfMap[cmsID][key] + assert.NoError(t, err) + assert.Equal(t, tt.skipped, skipped) + assert.Equal(t, tt.failure, failure) + assert.Equal(t, tt.numCCLFFiles, len(cclfFiles)) + var numCCLF0, numCCLF8 int + for _, cclfFile := range cclfFiles { + if cclfFile.cclfNum == 0 { + numCCLF0++ + } else if cclfFile.cclfNum == 8 { + numCCLF8++ + } else { + assert.Fail(t, "Unexpected CCLF num received %d", cclfFile.cclfNum) + } + } + assert.Equal(t, tt.numCCLF0, numCCLF0) + assert.Equal(t, tt.numCCLF8, numCCLF8) + }) + } +} + func (s *S3ProcessorTestSuite) TestLoadCclfFiles_InvalidPath() { cclfMap, skipped, failure, err := s.processor.LoadCclfFiles("foo") assert.ErrorContains(s.T(), err, "NoSuchBucket: The specified bucket does not exist") diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 24272aea5..e22df4589 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -55,8 +55,10 @@ func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, e return "", err } - dir := bcdaaws.ParseS3Directory(e.S3.Bucket.Name, e.S3.Object.Key) - return handleCclfImport(s3AssumeRoleArn, dir) + // Send the entire filepath into the CCLF Importer so we are only + // importing the one file that was sent in the trigger. + filepath := fmt.Sprintf("%s/%s", e.S3.Bucket.Name, e.S3.Object.Key) + return handleCclfImport(s3AssumeRoleArn, filepath) } } diff --git a/bcda/lambda/cclf/main_test.go b/bcda/lambda/cclf/main_test.go index 2f2137770..e6189740e 100644 --- a/bcda/lambda/cclf/main_test.go +++ b/bcda/lambda/cclf/main_test.go @@ -23,7 +23,7 @@ func TestCclfImportMainSuite(t *testing.T) { } func (s *CclfImportMainSuite) TestImportCCLFDirectory() { - targetACO := "A0002" + targetACO := "A0001" assert := assert.New(s.T()) env := uuid.NewUUID() @@ -42,14 +42,15 @@ func (s *CclfImportMainSuite) TestImportCCLFDirectory() { type test struct { path string + filename string err error expectedLogs []string } tests := []test{ - {path: "../../../shared_files/cclf/archives/valid2/", expectedLogs: []string{"Successfully imported 2 files.", "Failed to import 0 files.", "Skipped 0 files."}}, - {path: "../../../shared_files/cclf/archives/invalid_bcd/", err: errors.New("one or more files failed to import correctly"), expectedLogs: []string{}}, - {path: "../../../shared_files/cclf/archives/skip/", expectedLogs: []string{"Successfully imported 0 files.", "Failed to import 0 files.", "Skipped 0 files."}}, + {path: "../../../shared_files/cclf/archives/valid2/", filename: "cclf/archives/valid2/T.BCD.A0001.ZCY18.D181120.T1000000", expectedLogs: []string{"Successfully imported 2 files.", "Failed to import 0 files.", "Skipped 0 files."}}, + {path: "../../../shared_files/cclf/archives/invalid_bcd/", filename: "cclf/archives/invalid_bcd/P.BCD.A0009.ZCY18.D181120.T0001000", err: errors.New("one or more files failed to import correctly"), expectedLogs: []string{}}, + {path: "../../../shared_files/cclf/archives/skip/", filename: "cclf/archives/skip/T.BCD.ACOB.ZC0Y18.D181120.T0001000", expectedLogs: []string{"Successfully imported 0 files.", "Failed to import 0 files.", "Skipped 0 files."}}, } for _, tc := range tests { @@ -59,7 +60,7 @@ func (s *CclfImportMainSuite) TestImportCCLFDirectory() { path, cleanup := testUtils.CopyToS3(s.T(), tc.path) defer cleanup() - res, err := cclfImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), path, "fake_filename")) + res, err := cclfImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), path, tc.filename)) if tc.err == nil { assert.Nil(err) diff --git a/shared_files/cclf/archives/valid2/T.BCD.A0001.ZCY18.D181120.T1000000 b/shared_files/cclf/archives/valid2/T.BCD.A0001.ZCY18.D181120.T1000000 new file mode 100644 index 000000000..754f0e7f4 Binary files /dev/null and b/shared_files/cclf/archives/valid2/T.BCD.A0001.ZCY18.D181120.T1000000 differ diff --git a/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181120.T1000000 b/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181120.T1000000 deleted file mode 100644 index 7cb54f806..000000000 Binary files a/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181120.T1000000 and /dev/null differ diff --git a/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181121.T1000000 b/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181121.T1000000 deleted file mode 100644 index d7d9d31ee..000000000 Binary files a/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181121.T1000000 and /dev/null differ diff --git a/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181122.T1000000 b/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181122.T1000000 deleted file mode 100644 index 77cfe00aa..000000000 Binary files a/shared_files/cclf/archives/valid2/T.BCD.A0002.ZCY18.D181122.T1000000 and /dev/null differ