fix fs info reporting negative available size

Signed-off-by: panguixin <panguixin@bytedance.com>
opensearch-project · Dec 11, 2023 · cf2596f · cf2596f
1 parent 6bb53de
commit cf2596f
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 0 deletions.
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
@@ -82,6 +82,10 @@ public FsInfo stats(FsInfo previous) throws IOException {
                 paths[i].fileCacheReserved = adjustForHugeFilesystems(dataLocations[i].fileCacheReservedSize.getBytes());
                 paths[i].fileCacheUtilized = adjustForHugeFilesystems(fileCache.usage().usage());
                 paths[i].available -= (paths[i].fileCacheReserved - paths[i].fileCacheUtilized);
+                // occurs if reserved file cache space is occupied by other files, like local indices
+                if (paths[i].available < 0) {
+                    paths[i].available = 0;
+                }
             }
         }
         FsInfo.IoStats ioStats = null;

diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
@@ -45,9 +45,12 @@
 import org.opensearch.index.store.remote.filecache.FileCacheFactory;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.nio.file.FileStore;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.FileAttributeView;
 import java.nio.file.attribute.FileStoreAttributeView;
 import java.util.Arrays;
@@ -58,6 +61,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.opensearch.monitor.fs.FsProbe.adjustForHugeFilesystems;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.greaterThan;
@@ -162,6 +166,59 @@ public void testFsCacheInfo() throws IOException {
         }
     }
 
+    public void testFsInfoWhenFileCacheOccupied() throws IOException {
+        Settings settings = Settings.builder().putList("node.roles", "search", "data").build();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            final long usableSpace = adjustForHugeFilesystems(env.fileCacheNodePath().fileStore.getUsableSpace());
+            ByteSizeValue gbByteSizeValue = new ByteSizeValue(usableSpace - 10, ByteSizeUnit.BYTES);
+            env.fileCacheNodePath().fileCacheReservedSize = gbByteSizeValue;
+            FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(
+                gbByteSizeValue.getBytes(),
+                16,
+                new NoopCircuitBreaker(CircuitBreaker.REQUEST)
+            );
+
+            // write a temp file to occupy some file cache space
+            Path tempFile = createTempFile();
+            try (BufferedWriter bufferedWriter = Files.newBufferedWriter(tempFile, StandardOpenOption.APPEND)) {
+                bufferedWriter.write(randomAlphaOfLength(100));
+                bufferedWriter.flush();
+            }
+
+            FsProbe probe = new FsProbe(env, fileCache);
+            FsInfo stats = probe.stats(null);
+            assertNotNull(stats);
+            assertTrue(stats.getTimestamp() > 0L);
+            FsInfo.Path total = stats.getTotal();
+            assertNotNull(total);
+            assertTrue(total.total > 0L);
+            assertTrue(total.free > 0L);
+            assertTrue(total.fileCacheReserved > 0L);
+            if (env.nodePaths().length > 1) {
+                assertTrue(total.available > 0L);
+            } else {
+                assertEquals(0L, total.available);
+            }
+            assertTrue((total.free - total.available) < total.fileCacheReserved);
+
+            for (FsInfo.Path path : stats) {
+                assertNotNull(path);
+                assertFalse(path.getPath().isEmpty());
+                assertFalse(path.getMount().isEmpty());
+                assertFalse(path.getType().isEmpty());
+                assertTrue(path.total > 0L);
+                assertTrue(path.free > 0L);
+
+                if (path.fileCacheReserved > 0L) {
+                    assertEquals(0L, path.available);
+                    assertTrue(path.free - path.available < path.fileCacheReserved);
+                } else {
+                    assertTrue(path.available > 0L);
+                }
+            }
+        }
+    }
+
     public void testFsInfoOverflow() throws Exception {
         final FsInfo.Path pathStats = new FsInfo.Path(
             "/foo/bar",