From 91be13187ee9cd9ca1d074c79a9a36fcbd7d178b Mon Sep 17 00:00:00 2001 From: HARPER Jon Date: Wed, 26 Jun 2024 17:32:59 +0200 Subject: [PATCH] Allow to import folders Signed-off-by: HARPER Jon --- .../conversion/test/export/CgmesExportTest.java | 4 +++- .../powsybl/commons/datasource/DataSource.java | 4 ++-- .../commons/datasource/DataSourceUtil.java | 3 +++ .../commons/datasource/FileDataSource.java | 12 +++++++++--- .../commons/datasource/FileDataSourceTest.java | 16 ++++++++-------- .../sensitivity/SensitivityAnalysisToolTest.java | 2 +- .../ShortCircuitAnalysisToolTest.java | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesExportTest.java b/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesExportTest.java index be488373e0d..dcebc7ad182 100644 --- a/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesExportTest.java +++ b/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesExportTest.java @@ -258,6 +258,7 @@ void testDoNotExportFictitiousSwitchesCreatedForDisconnectedTerminals() throws I assertEquals("true", fictitiousSwitch.getProperty(Conversion.PROPERTY_IS_CREATED_FOR_DISCONNECTED_TERMINAL)); String exportFolder = "/test-terminal-disconnected-fictitious-switch"; + String exportFolder1 = "/test-terminal-disconnected-fictitious-switch-closed"; try (FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) { // Export to CGMES and add boundary EQ for reimport Path tmpDir = Files.createDirectory(fs.getPath(exportFolder)); @@ -283,7 +284,8 @@ void testDoNotExportFictitiousSwitchesCreatedForDisconnectedTerminals() throws I // And the fictitious switch is not crated when re-importing fictitiousSwitch.setOpen(false); String baseName1 = "testTerminalDisconnectedFictitiousSwitchClosedExported"; - ReadOnlyDataSource exportedCgmes1 = exportAndAddBoundaries(network, tmpDir, baseName1, ds); + Path tmpDir1 = Files.createDirectory(fs.getPath(exportFolder1)); + ReadOnlyDataSource exportedCgmes1 = exportAndAddBoundaries(network, tmpDir1, baseName1, ds); CgmesModel cgmes1 = CgmesModelFactory.create(exportedCgmes1, TripleStoreFactory.defaultImplementation()); assertTrue(cgmes1.isNodeBreaker()); assertFalse(cgmes1.switches().stream().anyMatch(sw -> sw.getId("Switch").equals(fictitiousSwitchId))); diff --git a/commons/src/main/java/com/powsybl/commons/datasource/DataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/DataSource.java index d21556dd9dd..2d0c9687e9f 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/DataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/DataSource.java @@ -26,8 +26,8 @@ public interface DataSource extends ReadOnlyDataSource { static DataSource fromPath(Path file) { Objects.requireNonNull(file); - if (!Files.isRegularFile(file)) { - throw new PowsyblException("File " + file + " does not exist or is not a regular file"); + if (!Files.exists(file)) { + throw new PowsyblException("File " + file + " does not exist"); } Path absFile = file.toAbsolutePath(); return fromPath(absFile.getParent(), absFile.getFileName().toString()); diff --git a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java index 81300618f3b..59dfecc6f8a 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java @@ -7,6 +7,7 @@ */ package com.powsybl.commons.datasource; +import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -96,6 +97,8 @@ static DataSource createDataSource(Path directory, String fileNameOrBaseName, Da } else if (fileNameOrBaseName.endsWith(".bz2")) { String fileNameWithoutCompressionExtension = fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4); return new Bzip2FileDataSource(directory, getBaseName(fileNameWithoutCompressionExtension), getBaseExtension(fileNameWithoutCompressionExtension), observer); + } else if (Files.isDirectory(directory.resolve(fileNameOrBaseName))) { + return new FileDataSource(directory.resolve(fileNameOrBaseName), getBaseName(fileNameOrBaseName), "", observer); } else { return new FileDataSource(directory, getBaseName(fileNameOrBaseName), getBaseExtension(fileNameOrBaseName), observer); } diff --git a/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java index cf22669f4e8..6640b758d6a 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java @@ -120,11 +120,17 @@ public Set listNames(String regex) throws IOException { Pattern p = Pattern.compile(regex); int maxDepth = 1; try (Stream paths = Files.walk(directory, maxDepth)) { - return paths + + var filenames = paths .filter(Files::isRegularFile) .map(Path::getFileName) - .map(Path::toString) - .filter(name -> name.startsWith(baseName)) + .map(Path::toString); + // For a file in a directory, we filter by basename to get similar files only because other files may be unrelated. + // For a directory, we list all files because we assume everything is related. And this allows to still have the directory name as the basename (for example for writing new files named like the directory) + var isListingAllDirectory = baseExtension.isEmpty(); // we assume all files have extensions so if we have a mainExtension, this means we are not in a clean separate directory + var maybeBaseNameFilteredFilenames = isListingAllDirectory ? filenames + : filenames.filter(name -> name.startsWith(baseName)); + return maybeBaseNameFilteredFilenames // Return names after removing the compression extension .map(name -> name.replace(getCompressionExt(), "")) .filter(s -> p.matcher(s).matches()) diff --git a/commons/src/test/java/com/powsybl/commons/datasource/FileDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/FileDataSourceTest.java index 99e6a7d0e72..5a3e1f072b3 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/FileDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/FileDataSourceTest.java @@ -42,16 +42,16 @@ void listNamesTest() throws IOException { } // A file data source created using the complete filename does not return other filenames - Set names = new FileDataSource(testDir, getBaseName() + ".txt").listNames(".*"); + Set names = new FileDataSource(testDir, getBaseName(), ".txt").listNames(".*"); assertEquals(1, names.size()); assertTrue(names.contains(validFilename)); assertFalse(names.contains(otherFilename)); - // A file data source created using the test folder and the basename sees only the right file + // A file data source created using the test folder and the basename sees all the files names = new FileDataSource(testDir, getBaseName()).listNames(".*"); - assertEquals(1, names.size()); + assertEquals(2, names.size()); assertTrue(names.contains(validFilename)); - assertFalse(names.contains(otherFilename)); + assertTrue(names.contains(otherFilename)); } @Test @@ -81,11 +81,11 @@ void createNewFilesTest() throws IOException { assertTrue(ds.exists("dummy.txt")); assertTrue(ds.exists("dummy2.txt")); - // but only the files that contain the basename can be accessed through list names + // all the files can be accessed through list names for basename datasource (no extension) Set names = ds.listNames(".*"); - assertEquals(1, names.size()); + assertEquals(3, names.size()); assertTrue(names.contains(getBaseName() + suffix + "." + ext)); - assertFalse(names.contains("dummy.txt")); - assertFalse(names.contains("dummy2.txt")); + assertTrue(names.contains("dummy.txt")); + assertTrue(names.contains("dummy2.txt")); } } diff --git a/sensitivity-analysis-api/src/test/java/com/powsybl/sensitivity/SensitivityAnalysisToolTest.java b/sensitivity-analysis-api/src/test/java/com/powsybl/sensitivity/SensitivityAnalysisToolTest.java index b30b0df5b15..004914d2ebc 100644 --- a/sensitivity-analysis-api/src/test/java/com/powsybl/sensitivity/SensitivityAnalysisToolTest.java +++ b/sensitivity-analysis-api/src/test/java/com/powsybl/sensitivity/SensitivityAnalysisToolTest.java @@ -177,7 +177,7 @@ void checkFailsWhenNetworkFileNotFound() { "--case-file", "wrongFile.xiidm", "--factors-file", "factors.json", "--output-file", "output.csv"}, - "com.powsybl.commons.PowsyblException: File wrongFile.xiidm does not exist or is not a regular file"); + "com.powsybl.commons.PowsyblException: File wrongFile.xiidm does not exist"); } @Test diff --git a/shortcircuit-api/src/test/java/com/powsybl/shortcircuit/ShortCircuitAnalysisToolTest.java b/shortcircuit-api/src/test/java/com/powsybl/shortcircuit/ShortCircuitAnalysisToolTest.java index 8d65a020545..36f78a6267d 100644 --- a/shortcircuit-api/src/test/java/com/powsybl/shortcircuit/ShortCircuitAnalysisToolTest.java +++ b/shortcircuit-api/src/test/java/com/powsybl/shortcircuit/ShortCircuitAnalysisToolTest.java @@ -73,7 +73,7 @@ void checkFailsWhenInputFileNotFound() { @Test void checkFailsWhenNetworkFileNotFound() { - assertCommandErrorMatch(new String[] {COMMAND_NAME, "--input-file", "input.txt", "--case-file", "wrongFile.uct"}, "com.powsybl.commons.PowsyblException: File wrongFile.uct does not exist or is not a regular file"); + assertCommandErrorMatch(new String[] {COMMAND_NAME, "--input-file", "input.txt", "--case-file", "wrongFile.uct"}, "com.powsybl.commons.PowsyblException: File wrongFile.uct does not exist"); } @Test