Skip to content

Commit

Permalink
Making rename dir limit test package parallell (#1850)
Browse files Browse the repository at this point in the history 
* making rename dir limit test parallell

* setup test dir

* lint tests fix

* small fix

* formating

* creating global testDir variable

* rebasing and cleanup at the end

* removing cleanup

* review comment

* passing destDir directly to the method
  • Loading branch information
@Tulsishah
Tulsishah committed Apr 19, 2024
1 parent d5eedac commit ede0d0f
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 87 deletions.
2 changes: 1 addition & 1 deletion perfmetrics/scripts/run_e2e_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ TEST_DIR_PARALLEL=(
"gzip"
"write_large_files"
"list_large_dir"
"rename_dir_limit"
"read_large_files"
)
# These tests never become parallel as it is changing bucket permissions.
Expand All @@ -44,7 +45,6 @@ TEST_DIR_NON_PARALLEL_GROUP_2=(
"explicit_dir"
"implicit_dir"
"operations"
"rename_dir_limit"
)

TEST_DIR_HNS_GROUP=(
Expand Down
53 changes: 23 additions & 30 deletions tools/integration_tests/rename_dir_limit/move_dir_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,6 @@ func checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDirPath string, t *test
// srcMoveDir/srcMoveFile -- File
// srcMoveDir/subSrcMoveDir -- Dir
func createSrcDirectoryWithObjectsForMoveDirTest(dirPath string, t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

// testBucket/srcMoveDir
err := os.Mkdir(dirPath, setup.FilePermission_0600)
if err != nil {
Expand Down Expand Up @@ -133,11 +130,12 @@ func checkIfMovedDirectoryHasCorrectData(destDir string, t *testing.T) {
// destMoveDir/srcMoveFile -- File
// destMoveDir/subSrcMoveDir -- Dir
func TestMoveDirectoryInNonExistingDirectory(t *testing.T) {
srcDir := path.Join(setup.MntDir(), SrcMoveDirectory)
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
srcDir := path.Join(testDir, SrcMoveDirectory)

createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t)

destDir := path.Join(setup.MntDir(), DestMoveDirectoryNotExist)
destDir := path.Join(testDir, DestMoveDirectoryNotExist)

err := operations.Move(srcDir, destDir)
if err != nil {
Expand All @@ -158,13 +156,14 @@ func TestMoveDirectoryInNonExistingDirectory(t *testing.T) {
// destMoveDir/srcMoveDir/srcMoveFile -- File
// destMoveDir/srcMoveDir/subSrcMoveDir -- Dir
func TestMoveDirectoryInEmptyDirectory(t *testing.T) {
srcDir := path.Join(setup.MntDir(), SrcMoveDirectory)
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
srcDir := path.Join(testDir, SrcMoveDirectory)

createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t)

// Create below directory
// destMoveDir -- Dir
destDir := path.Join(setup.MntDir(), DestMoveDirectory)
destDir := path.Join(testDir, DestMoveDirectory)
err := os.Mkdir(destDir, setup.FilePermission_0600)
if err != nil {
t.Errorf("Error in creating directory: %v", err)
Expand Down Expand Up @@ -193,23 +192,23 @@ func TestMoveDirectoryInEmptyDirectory(t *testing.T) {
checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t)
}

func createDestNonEmptyDirectoryForMoveTest(t *testing.T) {
destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory)
operations.CreateDirectoryWithNFiles(0, destDir, "", t)
func createDestNonEmptyDirectoryForMoveTest(dirPath string, t *testing.T) {
operations.CreateDirectoryWithNFiles(0, dirPath, "", t)

destSubDir := path.Join(destDir, SubDirInNonEmptyDestMoveDirectory)
destSubDir := path.Join(dirPath, SubDirInNonEmptyDestMoveDirectory)
operations.CreateDirectoryWithNFiles(0, destSubDir, "", t)
}

func TestMoveDirectoryInNonEmptyDirectory(t *testing.T) {
srcDir := path.Join(setup.MntDir(), SrcMoveDirectory)
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
srcDir := path.Join(testDir, SrcMoveDirectory)

createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t)

// Create below directory
// destMoveDir -- Dir
destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory)
createDestNonEmptyDirectoryForMoveTest(t)
destDir := path.Join(testDir, DestNonEmptyMoveDirectory)
createDestNonEmptyDirectoryForMoveTest(destDir, t)

err := operations.Move(srcDir, destDir)
if err != nil {
Expand Down Expand Up @@ -269,17 +268,15 @@ func checkIfMovedEmptyDirectoryHasNoData(destSrc string, t *testing.T) {
// destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory
// destNonEmptyMoveDirectory/emptySrcDirectoryMoveTest
func TestMoveEmptyDirectoryInNonEmptyDirectory(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest)
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
srcDir := path.Join(testDir, EmptySrcDirectoryMoveTest)
operations.CreateDirectoryWithNFiles(0, srcDir, "", t)

// Create below directory
// destNonEmptyMoveDirectory -- Dir
// destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory -- Dir
destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory)
createDestNonEmptyDirectoryForMoveTest(t)
destDir := path.Join(testDir, DestNonEmptyMoveDirectory)
createDestNonEmptyDirectoryForMoveTest(destDir, t)

err := operations.Move(srcDir, destDir)
if err != nil {
Expand Down Expand Up @@ -325,15 +322,13 @@ func TestMoveEmptyDirectoryInNonEmptyDirectory(t *testing.T) {
// Output
// destEmptyMoveDirectory/emptySrcDirectoryMoveTest
func TestMoveEmptyDirectoryInEmptyDirectory(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest)
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
srcDir := path.Join(testDir, EmptySrcDirectoryMoveTest)
operations.CreateDirectoryWithNFiles(0, srcDir, "", t)

// Create below directory
// destMoveDir -- Dir
destDir := path.Join(setup.MntDir(), DestEmptyMoveDirectory)
destDir := path.Join(testDir, DestEmptyMoveDirectory)
operations.CreateDirectoryWithNFiles(0, destDir, "", t)

err := operations.Move(srcDir, destDir)
Expand Down Expand Up @@ -371,14 +366,12 @@ func TestMoveEmptyDirectoryInEmptyDirectory(t *testing.T) {
// Output
// destMoveDirectoryNotExist
func TestMoveEmptyDirectoryInNonExistingDirectory(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest)
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
srcDir := path.Join(testDir, EmptySrcDirectoryMoveTest)
operations.CreateDirectoryWithNFiles(0, srcDir, "", t)

// destMoveDirectoryNotExist -- Dir
destDir := path.Join(setup.MntDir(), DestMoveDirectoryNotExist)
destDir := path.Join(testDir, DestMoveDirectoryNotExist)

_, err := os.Stat(destDir)
if err == nil {
Expand Down
1 change: 1 addition & 0 deletions tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
)

const DirForRenameDirLimitTests = "dirForRenameDirLimitTests"
const DirectoryWithThreeFiles = "directoryWithThreeFiles"
const DirectoryWithTwoFiles = "directoryWithTwoFiles"
const DirectoryWithFourFiles = "directoryWithFourFiles"
Expand Down
98 changes: 42 additions & 56 deletions tools/integration_tests/rename_dir_limit/rename_dir_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,17 @@ import (
// As --rename-directory-limit = 3, and the number of objects in the directory is three,
// which is equal to the limit, the operation should get successful.
func TestRenameDirectoryWithThreeFiles(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
// Create directory structure
// testBucket/directoryWithThreeFiles -- Dir
// testBucket/directoryWithThreeFiles/temp1.txt -- File
// testBucket/directoryWithThreeFiles/temp2.txt -- File
// testBucket/directoryWithThreeFiles/temp3.txt -- File
dirPath := path.Join(setup.MntDir(), DirectoryWithThreeFiles)
// testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles -- Dir
// testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles/temp1.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles/temp2.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithThreeFiles/temp3.txt -- File
dirPath := path.Join(testDir, DirectoryWithThreeFiles)
operations.CreateDirectoryWithNFiles(3, dirPath, PrefixTempFile, t)

oldDirPath := path.Join(setup.MntDir(), DirectoryWithThreeFiles)
newDirPath := path.Join(setup.MntDir(), RenamedDirectory)
oldDirPath := path.Join(testDir, DirectoryWithThreeFiles)
newDirPath := path.Join(testDir, RenamedDirectory)

// Cleaning the directory before renaming.
operations.RemoveDir(newDirPath)
Expand All @@ -54,19 +52,17 @@ func TestRenameDirectoryWithThreeFiles(t *testing.T) {
// As --rename-directory-limit = 3, and the number of objects in the directory is two,
// which is less than the limit, the operation should get successful.
func TestRenameDirectoryWithTwoFiles(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
// Create directory structure
// testBucket/directoryWithTwoFiles -- Dir
// testBucket/directoryWithTwoFiles/temp1.txt -- File
// testBucket/directoryWithTwoFiles/temp2.txt -- File
dirPath := path.Join(setup.MntDir(), DirectoryWithTwoFiles)
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFiles -- Dir
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFiles/temp1.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFiles/temp2.txt -- File
dirPath := path.Join(testDir, DirectoryWithTwoFiles)

operations.CreateDirectoryWithNFiles(2, dirPath, PrefixTempFile, t)

oldDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFiles)
newDirPath := path.Join(setup.MntDir(), RenamedDirectory)
oldDirPath := path.Join(testDir, DirectoryWithTwoFiles)
newDirPath := path.Join(testDir, RenamedDirectory)

// Cleaning the directory before renaming.
operations.RemoveDir(newDirPath)
Expand All @@ -81,25 +77,19 @@ func TestRenameDirectoryWithTwoFiles(t *testing.T) {
// As --rename-directory-limit = 3, and the number of objects in the directory is two,
// which is greater than the limit, the operation should get fail.
func TestRenameDirectoryWithFourFiles(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
// Creating directory structure
// testBucket/directoryWithFourFiles -- Dir
// testBucket/directoryWithFourFiles/temp1.txt -- File
// testBucket/directoryWithFourFiles/temp2.txt -- File
// testBucket/directoryWithFourFiles/temp3.txt -- File
// testBucket/directoryWithFourFiles/temp4.txt -- File
dirPath := path.Join(setup.MntDir(), DirectoryWithFourFiles)

operations.CreateDirectoryWithNFiles(4, dirPath, PrefixTempFile, t)

oldDirPath := path.Join(setup.MntDir(), DirectoryWithFourFiles)
newDirPath := path.Join(setup.MntDir(), RenamedDirectory)
// testBucket/dirForRenameDirLimitTests/directoryWithFourFiles -- Dir
// testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp1.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp2.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp3.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithFourFiles/temp4.txt -- File
oldDirPath := path.Join(testDir, DirectoryWithFourFiles)
operations.CreateDirectoryWithNFiles(4, oldDirPath, PrefixTempFile, t)
newDirPath := path.Join(testDir, RenamedDirectory)

// Cleaning the directory before renaming.
operations.RemoveDir(newDirPath)

err := os.Rename(oldDirPath, newDirPath)

if err == nil {
Expand All @@ -110,22 +100,20 @@ func TestRenameDirectoryWithFourFiles(t *testing.T) {
// As --rename-directory-limit = 3, and the number of objects in the directory is three,
// which is equal to limit, the operation should get successful.
func TestRenameDirectoryWithTwoFilesAndOneEmptyDirectory(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
// Creating directory structure
// testBucket/directoryWithTwoFilesOneEmptyDirectory -- Dir
// testBucket/directoryWithTwoFilesOneEmptyDirectory/a.txt -- File
// testBucket/directoryWithTwoFilesOneEmptyDirectory/b.txt -- File
// testBucket/directoryWithTwoFilesOneEmptyDirectory/emptySubDirectory -- Dir
dirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneEmptyDirectory)
subDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneEmptyDirectory, EmptySubDirectory)
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory -- Dir
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory/a.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory/b.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneEmptyDirectory/emptySubDirectory -- Dir
dirPath := path.Join(testDir, DirectoryWithTwoFilesOneEmptyDirectory)
subDirPath := path.Join(testDir, DirectoryWithTwoFilesOneEmptyDirectory, EmptySubDirectory)

operations.CreateDirectoryWithNFiles(2, dirPath, PrefixTempFile, t)
operations.CreateDirectoryWithNFiles(0, subDirPath, PrefixTempFile, t)

oldDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneEmptyDirectory)
newDirPath := path.Join(setup.MntDir(), RenamedDirectory)
oldDirPath := path.Join(testDir, DirectoryWithTwoFilesOneEmptyDirectory)
newDirPath := path.Join(testDir, RenamedDirectory)

// Cleaning the directory before renaming.
operations.RemoveDir(newDirPath)
Expand All @@ -140,24 +128,22 @@ func TestRenameDirectoryWithTwoFilesAndOneEmptyDirectory(t *testing.T) {
// As --rename-directory-limit = 3, and the number of objects in the directory is Four,
// which is greater than the limit, the operation should get fail.
func TestRenameDirectoryWithTwoFilesAndOneNonEmptyDirectory(t *testing.T) {
// Clean the mountedDirectory before running test.
setup.CleanMntDir()

testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
// Creating directory structure
// testBucket/directoryWithTwoFilesOneNonEmptyDirectory -- Dir
// testBucket/directoryWithTwoFilesOneNonEmptyDirectory/temp1.txt -- File
// testBucket/directoryWithTwoFilesOneNonEmptyDirectory/temp2.txt -- File
// testBucket/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory -- Dir
// testBucket/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory/temp3.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory -- Dir
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/temp1.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/temp2.txt -- File
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory -- Dir
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory/NonEmptySubDirectory/temp3.txt -- File

dirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneNonEmptyDirectory)
dirPath := path.Join(testDir, DirectoryWithTwoFilesOneNonEmptyDirectory)
subDirPath := path.Join(dirPath, NonEmptySubDirectory)

operations.CreateDirectoryWithNFiles(2, dirPath, PrefixTempFile, t)
operations.CreateDirectoryWithNFiles(1, subDirPath, PrefixTempFile, t)

oldDirPath := path.Join(setup.MntDir(), DirectoryWithTwoFilesOneNonEmptyDirectory)
newDirPath := path.Join(setup.MntDir(), RenamedDirectory)
oldDirPath := path.Join(testDir, DirectoryWithTwoFilesOneNonEmptyDirectory)
newDirPath := path.Join(testDir, RenamedDirectory)

// Cleaning the directory before renaming.
operations.RemoveDir(newDirPath)
Expand Down

0 comments on commit ede0d0f

Please sign in to comment.