SCANGRADLE-148 Gradle scanner scans some files in hidden folders (#228)

integrationTests/src/test/java/org/sonarqube/gradle/GradleTest.java

@@ -354,11 +354,14 @@ public void testScanAllOnMultiModuleWithSubModulesProjectCollectsTheExpectedSour
         baseDir.resolve("module/submodule/submoduleScript.sh").toString(),
         baseDir.resolve("gradlew").toString(),
         baseDir.resolve("gradlew.bat").toString(),
-        baseDir.resolve("gradle/wrapper/gradle-wrapper.properties").toString()
+        baseDir.resolve("gradle/wrapper/gradle-wrapper.properties").toString(),
+        baseDir.resolve(".hiddenDir/file.txt").toString(),
+        baseDir.resolve(".hiddenConfig").toString()
       ).doesNotContain(
         baseDir.resolve("skippedModule/build.gradle.kts").toString(),
         baseDir.resolve("skippedModule/skippedSubmodule/skippedSubmoduleScript.sh").toString(),
-        baseDir.resolve("skippedModule/skippedSubmodule/build.gradle.kts").toString()
+        baseDir.resolve("skippedModule/skippedSubmodule/build.gradle.kts").toString(),
+        baseDir.resolve(".hiddenDir/script.py").toString()
       );
 
     assertThat(props.getProperty(":module.sonar.sources")).isEqualTo(baseDir.resolve("module/src/main/java").toString());

src/main/java/org/sonarqube/gradle/SonarPropertyComputer.java

@@ -220,7 +220,13 @@ private static void computeScanAllProperties(Project project, Map<String, Object
 
     Set<Path> excludedFiles = computeReportPaths(properties);
 
-    SourceCollector visitor = new SourceCollector(allModulesExistingSources, skippedDirs, excludedFiles, false);
+    SourceCollector visitor = SourceCollector.builder()
+      .setRoot(project.getProjectDir().toPath())
+      .setExistingSources(allModulesExistingSources)
+      .setExcludedFiles(excludedFiles)
+      .setDirectoriesToIgnore(skippedDirs)
+      .build();
+
 
     try {
       Files.walkFileTree(project.getProjectDir().toPath(), visitor);

src/main/java/org/sonarqube/gradle/SourceCollector.java

@@ -35,6 +35,17 @@
 public class SourceCollector implements FileVisitor<Path> {
   private static final Set<String> EXCLUDED_DIRECTORIES = new HashSet<>(
     Arrays.asList(
+      ".cache",
+      ".env",
+      ".git",
+      ".gradle",
+      ".jruby",
+      ".m2",
+      ".node_modules",
+      ".npm",
+      ".pycache",
+      ".pytest_cache",
+      ".venv",
       "bin",
       "build",
       "dist",
@@ -81,6 +92,48 @@ public class SourceCollector implements FileVisitor<Path> {
     ".kt")).map(ext -> ext.toLowerCase(Locale.ROOT))
     .collect(Collectors.toSet());
 
+  private static final Set<String> INCLUDE_EXTENSIONS_FOR_HIDDEN_FILES = Set.of(
+    ".bash",
+    ".bat",
+    ".cnf",
+    ".config",
+    ".db",
+    ".env",
+    ".htpasswd",
+    ".json",
+    ".ksh",
+    ".properties",
+    ".ps1",
+    ".settings",
+    ".sh",
+    ".txt",
+    ".xml",
+    ".yaml",
+    ".yml",
+    ".zsh"
+  );
+
+  private static final Set<String> INCLUDE_HIDDEN_FILES_KEYWORDS = Set.of(
+    ".env.",
+    "access",
+    "cfg",
+    "config",
+    "credential",
+    "history",
+    "id_dsa",
+    "id_ecdsa",
+    "id_ed25519",
+    "id_rsa",
+    "key",
+    "password",
+    "private",
+    "pwd",
+    "secret",
+    "sessions",
+    "token"
+  );
+
+  private final Path root;
   private final Set<Path> existingSources;
   private final Set<Path> directoriesToIgnore;
   private final Set<Path> excludedFiles;
@@ -92,20 +145,23 @@ public Set<Path> getCollectedSources() {
 
   private final Set<Path> collectedSources = new HashSet<>();
 
-  public SourceCollector(Set<Path> existingSources, Set<Path> directoriesToIgnore, Set<Path> excludedFiles, boolean shouldCollectJavaAndKotlinSources) {
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  private SourceCollector(Path root, Set<Path> existingSources, Set<Path> directoriesToIgnore, Set<Path> excludedFiles, boolean shouldCollectJavaAndKotlinSources) {
+    this.root = root;
     this.existingSources = existingSources;
     this.directoriesToIgnore = directoriesToIgnore;
     this.excludedFiles = excludedFiles;
     this.excludedExtensions = shouldCollectJavaAndKotlinSources ? EXCLUDED_EXTENSIONS_WITH_JAVA_AND_KOTLIN : EXCLUDED_EXTENSIONS_WITHOUT_JAVA_AND_KOTLIN;
   }
 
   @Override
-  public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
-    if (
-      isHidden(path) ||
-      isExcludedDirectory(path) ||
-      isCoveredByExistingSources(path)
-    ) {
+  public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) {
+    boolean isHiddenAndTooFarDownTheTree = isHidden(path) && !isChildOrGrandChildOfRoot(path);
+
+    if (isHiddenAndTooFarDownTheTree || isExcludedDirectory(path) || isCoveredByExistingSources(path)) {
       return FileVisitResult.SKIP_SUBTREE;
     }
     return FileVisitResult.CONTINUE;
@@ -116,6 +172,10 @@ private static boolean isHidden(Path path) {
       .anyMatch(token -> token.toString().startsWith("."));
   }
 
+  private boolean isChildOrGrandChildOfRoot(Path path) {
+    return root.equals(path.getParent()) || root.equals(path.getParent().getParent());
+  }
+
   private boolean isExcludedDirectory(Path path) {
     String pathAsString = path.getFileName().toString().toLowerCase(Locale.ROOT);
     return EXCLUDED_DIRECTORIES.contains(pathAsString) || directoriesToIgnore.contains(path);
@@ -129,10 +189,18 @@ private boolean isCoveredByExistingSources(Path path) {
   public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) {
     if (!basicFileAttributes.isSymbolicLink() && !excludedFiles.contains(path) && existingSources.stream().noneMatch(path::equals)) {
       String lowerCaseFileName = path.getFileName().toString().toLowerCase(Locale.ROOT);
-      if (excludedExtensions.stream().noneMatch(lowerCaseFileName::endsWith)) {
+
+      if (isHidden(path)) {
+        boolean isHiddenFileToCollect = INCLUDE_HIDDEN_FILES_KEYWORDS.stream().anyMatch(lowerCaseFileName::contains)
+          || INCLUDE_EXTENSIONS_FOR_HIDDEN_FILES.stream().anyMatch(lowerCaseFileName::endsWith);
+        if (isHiddenFileToCollect) {
+          collectedSources.add(path);
+        }
+      } else if (excludedExtensions.stream().noneMatch(lowerCaseFileName::endsWith)) {
         collectedSources.add(path);
       }
     }
+
     return FileVisitResult.CONTINUE;
   }
 
@@ -145,4 +213,46 @@ public FileVisitResult visitFileFailed(Path path, IOException e) throws IOExcept
   public FileVisitResult postVisitDirectory(Path path, IOException e) throws IOException {
     return FileVisitResult.CONTINUE;
   }
+
+  public static class Builder {
+    private Path root = null;
+    private Set<Path> existingSources = new HashSet<>();
+    private Set<Path> directoriesToIgnore = new HashSet<>();
+    private Set<Path> excludedFiles = new HashSet<>();
+    private boolean shouldCollectJavaAndKotlinSources = false;
+
+    private Builder() { }
+
+    public Builder setRoot(Path root) {
+      this.root = root;
+      return this;
+    }
+
+    public Builder setExistingSources(Set<Path> existingSources) {
+      this.existingSources = existingSources;
+      return this;
+    }
+
+    public Builder setDirectoriesToIgnore(Set<Path> directoriesToIgnore) {
+      this.directoriesToIgnore = directoriesToIgnore;
+      return this;
+    }
+
+    public Builder setExcludedFiles(Set<Path> excludedFiles) {
+      this.excludedFiles = excludedFiles;
+      return this;
+    }
+
+    public Builder setShouldCollectJavaAndKotlinSources(boolean shouldCollectJavaAndKotlinSources) {
+      this.shouldCollectJavaAndKotlinSources = shouldCollectJavaAndKotlinSources;
+      return this;
+    }
+
+    public SourceCollector build() {
+      if (root == null) {
+        throw new IllegalStateException("Root path must be set");
+      }
+      return new SourceCollector(root, existingSources, directoriesToIgnore, excludedFiles, shouldCollectJavaAndKotlinSources);
+    }
+  }
 }

src/test/groovy/org/sonarqube/gradle/FunctionalTests.groovy

@@ -505,7 +505,7 @@ class FunctionalTests extends Specification {
                 .withArguments('sonar', '--info',
                         '-Dsonar.gradle.scanAll=true',
                         '-Dsonar.coverageReportPaths=my-first-coverage-report.xml,my-second-coverage-report.xml',
-                        '-Dsonar.coverage.jacoco.xmlReportPaths=' + thirdCoverageReport.toAbsolutePath().toString(),
+                        '-Dsonar.coverage.jacoco.xmlReportPaths=' + thirdCoverageReport.toRealPath().toString(),
                         '-Dsonar.scanner.dumpToFile=' + outFile.toAbsolutePath())
                 .withPluginClasspath()
                 .withDebug(true)
@@ -518,7 +518,7 @@ class FunctionalTests extends Specification {
         props.load(outFile.newDataInputStream())
         props."sonar.gradle.scanAll" == "true"
         props."sonar.coverageReportPaths" == "my-first-coverage-report.xml,my-second-coverage-report.xml"
-        props."sonar.coverage.jacoco.xmlReportPaths" == thirdCoverageReport.toAbsolutePath().toString()
+        props."sonar.coverage.jacoco.xmlReportPaths" == thirdCoverageReport.toRealPath().toString()
         result.output.contains("Parameter sonar.gradle.scanAll is enabled. The scanner will attempt to collect additional sources.")
 
         // Assert that the extra files (empty script and reports) exist on disk

src/test/groovy/org/sonarqube/gradle/SonarQubePluginTest.groovy

@@ -1011,6 +1011,13 @@ class SonarQubePluginTest extends Specification {
 
     then:
     def expectedSources = """
+                .hidden/.hidden-file.txt
+                .hidden/file.txt
+                .hidden/folder/.hidden-nested-file.txt
+                .hidden/folder/nested-file.txt
+                .hidden/folder/public-id_rsa-prod
+                .hidden/folder/test-config.config
+                .hidden/prod-token
                 build.gradle.kts
                 module1/build.gradle.kts
                 module1/extras/pyScriptM1.py
@@ -1024,7 +1031,7 @@ class SonarQubePluginTest extends Specification {
     assert properties[":module2.:module2:submodule.sonar.sources"] == null
   }
 
-  def "scan all detects scripts in all modules"() {
+  def "scan all detects files in all modules"() {
     def dir = new File("src/test/projects/java-multi-nested-modules")
     def parent = ProjectBuilder.builder()
       .withName("java-multi-nested-modules")
@@ -1076,6 +1083,13 @@ class SonarQubePluginTest extends Specification {
 
     then:
     def expectedSources = """
+                .hidden/.hidden-file.txt
+                .hidden/file.txt
+                .hidden/folder/.hidden-nested-file.txt
+                .hidden/folder/nested-file.txt
+                .hidden/folder/public-id_rsa-prod
+                .hidden/folder/test-config.config
+                .hidden/prod-token
                 build.gradle.kts
                 module1/build.gradle.kts
                 module1/extras/pyScriptM1.py
@@ -1085,7 +1099,8 @@ class SonarQubePluginTest extends Specification {
                 module2/settings.gradle.kts
                 module2/submodule/build.gradle.kts
                 module2/submodule/scriptM2S.sh
-                settings.gradle.kts""".stripIndent().trim()
+                settings.gradle.kts
+                """.stripIndent().trim()
     assert sources == expectedSources
 
     assert normalizePathArray(module1Sources).contains(normalizePathString("src/test/projects/java-multi-nested-modules/module1/src/main/java"))