diff --git a/bench/src/main/scala/sjsonnet/MainBenchmark.scala b/bench/src/main/scala/sjsonnet/MainBenchmark.scala index bd3ab6d9..7d541de5 100644 --- a/bench/src/main/scala/sjsonnet/MainBenchmark.scala +++ b/bench/src/main/scala/sjsonnet/MainBenchmark.scala @@ -8,9 +8,12 @@ import org.openjdk.jmh.infra._ object MainBenchmark { val mainArgs = Array[String]( - "../../universe2/rulemanager/deploy/rulemanager.jsonnet", - "-J", "../../universe2", - "-J", "../../universe2/mt-shards/dev/az-westus-c2", + "../../universe/rulemanager/deploy/rulemanager.jsonnet", + // "../../universe/kubernetes/admission-controller/gatekeeper/deploy/gatekeeper.jsonnet", + "-J", "../../universe", + "-J", "../../universe/mt-shards/dev/az-westus-c2", + "-J", "../../universe/bazel-bin", + "--ext-code", "isKubecfg=false" ) def findFiles(): (IndexedSeq[(Path, String)], EvalScope) = { @@ -28,7 +31,7 @@ object MainBenchmark { parseCache = parseCache ) val renderer = new Renderer(new StringWriter, indent = 3) - interp.interpret0(interp.resolver.read(path).get, path, renderer).getOrElse(???) + interp.interpret0(interp.resolver.read(path).get.readString(), path, renderer).getOrElse(???) (parseCache.keySet.toIndexedSeq, interp.evaluator) } diff --git a/bench/src/main/scala/sjsonnet/RunProfiler.scala b/bench/src/main/scala/sjsonnet/RunProfiler.scala index 85177b7a..536e5d3f 100644 --- a/bench/src/main/scala/sjsonnet/RunProfiler.scala +++ b/bench/src/main/scala/sjsonnet/RunProfiler.scala @@ -24,7 +24,7 @@ object RunProfiler extends App { def run(): Long = { val renderer = new Renderer(new StringWriter, indent = 3) - val start = interp.resolver.read(path).get + val start = interp.resolver.read(path).get.readString() val t0 = System.nanoTime() interp.interpret0(start, path, renderer).getOrElse(???) System.nanoTime() - t0 diff --git a/sjsonnet/src-js/sjsonnet/Platform.scala b/sjsonnet/src-js/sjsonnet/Platform.scala index 01b47a32..35064a07 100644 --- a/sjsonnet/src-js/sjsonnet/Platform.scala +++ b/sjsonnet/src-js/sjsonnet/Platform.scala @@ -6,10 +6,10 @@ object Platform { def gzipString(s: String): String = { throw new Exception("GZip not implemented in Scala.js") } - def xzBytes(s: Array[Byte]): String = { + def xzBytes(s: Array[Byte], compressionLevel: Option[Int]): String = { throw new Exception("XZ not implemented in Scala.js") } - def xzString(s: String): String = { + def xzString(s: String, compressionLevel: Option[Int]): String = { throw new Exception("XZ not implemented in Scala.js") } def yamlToJson(s: String): String = { diff --git a/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala index f88ab814..6170676d 100644 --- a/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala @@ -24,8 +24,8 @@ object SjsonnetMain { case null => None case s => Some(JsVirtualPath(s)) } - def read(path: Path): Option[String] = - Option(importLoader(path.asInstanceOf[JsVirtualPath].path)) + def read(path: Path): Option[ResolvedFile] = + Option(StaticResolvedFile(importLoader(path.asInstanceOf[JsVirtualPath].path))) }, parseCache = new DefaultParseCache, new Settings(preserveOrder = preserveOrder), @@ -57,4 +57,4 @@ case class JsVirtualPath(path: String) extends Path{ def renderOffsetStr(offset: Int, loadedFileContents: mutable.HashMap[Path, Array[Int]]): String = { path + ":" + offset } -} \ No newline at end of file +} diff --git a/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala b/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala new file mode 100644 index 00000000..4eb68cc1 --- /dev/null +++ b/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala @@ -0,0 +1,91 @@ +package sjsonnet + +import java.io.{BufferedInputStream, File, FileInputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.zip.CRC32 +import fastparse.ParserInput + +/** + * A class that encapsulates a resolved import. This is used to cache the result of + * resolving an import. If the import is deemed too large (IE it's a large file), then we will avoid keeping it in + * memory and instead will re-read it from disk. + * + * @param resolvedImportPath The path of the file on disk that was resolved + * @param memoryLimitBytes The maximum size of a file that we will resolve. This is not the size of + * the buffer, but a mechanism to fail when being asked to resolve (and downstream parse) a file + * that is beyond this limit. + */ +class CachedResolvedFile(val resolvedImportPath: OsPath, memoryLimitBytes: Long) extends ResolvedFile { + + private val jFile: File = resolvedImportPath.p.toIO + + assert(jFile.exists(), s"Resolved import path ${resolvedImportPath} does not exist") + // Assert that the file is less than limit + assert(jFile.length() <= memoryLimitBytes, s"Resolved import path ${resolvedImportPath} is too large: ${jFile.length()} bytes > ${memoryLimitBytes} bytes") + + private[this] val resolvedImportContent: StaticResolvedFile = { + if (jFile.length() > 1024 * 1024) { + // If the file is too large, then we will just read it from disk + null + } else { + StaticResolvedFile(readString(jFile)) + } + } + + private[this] def readString(jFile: File): String = { + new String(Files.readAllBytes(jFile.toPath), StandardCharsets.UTF_8); + } + + /** + * A method that will return a reader for the resolved import. If the import is too large, then this will return + * a reader that will read the file from disk. Otherwise, it will return a reader that reads from memory. + */ + def getParserInput(): ParserInput = { + if (resolvedImportContent == null) { + FileParserInput(jFile) + } else { + resolvedImportContent.getParserInput() + } + } + + override def readString(): String = { + if (resolvedImportContent == null) { + // If the file is too large, then we will just read it from disk + readString(jFile) + } else { + // Otherwise, we will read it from memory + resolvedImportContent.readString() + } + } + + private def crcHashFile(file: File): Long = { + val buffer = new Array[Byte](8192) + val crc = new CRC32() + + val fis = new FileInputStream(file) + val bis = new BufferedInputStream(fis) + + try { + var bytesRead = bis.read(buffer) + while (bytesRead != -1) { + crc.update(buffer, 0, bytesRead) + bytesRead = bis.read(buffer) + } + } finally { + bis.close() + fis.close() + } + + crc.getValue() + } + + override lazy val contentHash: String = { + if (resolvedImportContent == null) { + // If the file is too large, then we will just read it from disk + crcHashFile(jFile).toString + } else { + resolvedImportContent.contentHash + } + } +} diff --git a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala index 018d34d7..6a59856a 100644 --- a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala @@ -22,8 +22,9 @@ object SjsonnetMain { .find(os.exists) .flatMap(p => try Some(OsPath(p)) catch{case NonFatal(_) => None}) - def read(path: Path): Option[String] = - try Some(os.read(path.asInstanceOf[OsPath].p)) catch { case NonFatal(_) => None } + def read(path: Path): Option[ResolvedFile] = { + readPath(path) + } } def main(args: Array[String]): Unit = { @@ -205,8 +206,9 @@ object SjsonnetMain { case Some(i) => new Importer { def resolve(docBase: Path, importName: String): Option[Path] = i(docBase, importName).map(OsPath) - def read(path: Path): Option[String] = - try Some(os.read(path.asInstanceOf[OsPath].p)) catch { case NonFatal(_) => None } + def read(path: Path): Option[ResolvedFile] = { + readPath(path) + } } case None => resolveImport(config.jpaths.map(os.Path(_, wd)).map(OsPath(_)), allowedInputs) }, @@ -291,4 +293,18 @@ object SjsonnetMain { } } + + /** + * Read a path into a [[ResolvedFile]] if it exists and is a file. A resolved file acts as a layer + * of caching on top of the underlying file system. Small files are read into memory, while large + * files are read from disk. + */ + private[this] def readPath(path: Path): Option[ResolvedFile] = { + val osPath = path.asInstanceOf[OsPath].p + if (os.exists(osPath) && os.isFile(osPath)) { + Some(new CachedResolvedFile(path.asInstanceOf[OsPath], memoryLimitBytes = Int.MaxValue.toLong)) + } else { + None + } + } } diff --git a/sjsonnet/src-jvm/sjsonnet/Platform.scala b/sjsonnet/src-jvm/sjsonnet/Platform.scala index 5537ba20..21e5e6bc 100644 --- a/sjsonnet/src-jvm/sjsonnet/Platform.scala +++ b/sjsonnet/src-jvm/sjsonnet/Platform.scala @@ -23,18 +23,26 @@ object Platform { def gzipString(s: String): String = { gzipBytes(s.getBytes()) } - def xzBytes(b: Array[Byte]): String = { + + /** + * Valid compression levels are 0 (no compression) to 9 (maximum compression). + */ + def xzBytes(b: Array[Byte], compressionLevel: Option[Int]): String = { val outputStream: ByteArrayOutputStream = new ByteArrayOutputStream(b.length) - val xz: XZOutputStream = new XZOutputStream(outputStream, new LZMA2Options()) + // Set compression to specified level + val level = compressionLevel.getOrElse(LZMA2Options.PRESET_DEFAULT) + val xz: XZOutputStream = new XZOutputStream(outputStream, new LZMA2Options(level)) xz.write(b) xz.close() val xzedBase64: String = Base64.getEncoder.encodeToString(outputStream.toByteArray) outputStream.close() xzedBase64 } - def xzString(s: String): String = { - xzBytes(s.getBytes()) + + def xzString(s: String, compressionLevel: Option[Int]): String = { + xzBytes(s.getBytes(), compressionLevel) } + def yamlToJson(yamlString: String): String = { val yaml: java.util.LinkedHashMap[String, Object] = new Yaml(new Constructor(classOf[java.util.LinkedHashMap[String, Object]])).load(yamlString) new JSONObject(yaml).toString() diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index bbd06664..7ca52497 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -6,10 +6,10 @@ object Platform { def gzipString(s: String): String = { throw new Exception("GZip not implemented in Scala Native") } - def xzBytes(s: Array[Byte]): String = { + def xzBytes(s: Array[Byte], compressionLevel: Option[Int]): String = { throw new Exception("XZ not implemented in Scala Native") } - def xzString(s: String): String = { + def xzString(s: String, compressionLevel: Option[Int]): String = { throw new Exception("XZ not implemented in Scala Native") } def yamlToJson(s: String): String = { diff --git a/sjsonnet/src/sjsonnet/Error.scala b/sjsonnet/src/sjsonnet/Error.scala index 0e380e3b..edbe18d2 100644 --- a/sjsonnet/src/sjsonnet/Error.scala +++ b/sjsonnet/src/sjsonnet/Error.scala @@ -103,7 +103,7 @@ trait EvalErrorScope { def prettyIndex(pos: Position): Option[(Int, Int)] = { importer.read(pos.currentFile).map { s => val Array(line, col) = - new IndexedParserInput(s).prettyIndex(pos.offset).split(':') + s.getParserInput().prettyIndex(pos.offset).split(':') (line.toInt, col.toInt) } } diff --git a/sjsonnet/src/sjsonnet/Evaluator.scala b/sjsonnet/src/sjsonnet/Evaluator.scala index be1e1086..d9f1d1ea 100644 --- a/sjsonnet/src/sjsonnet/Evaluator.scala +++ b/sjsonnet/src/sjsonnet/Evaluator.scala @@ -300,7 +300,7 @@ class Evaluator(resolver: CachedResolver, } def visitImportStr(e: ImportStr)(implicit scope: ValScope): Val.Str = - Val.Str(e.pos, importer.resolveAndReadOrFail(e.value, e.pos)._2) + Val.Str(e.pos, importer.resolveAndReadOrFail(e.value, e.pos)._2.readString()) def visitImport(e: Import)(implicit scope: ValScope): Val = { val (p, str) = importer.resolveAndReadOrFail(e.value, e.pos) diff --git a/sjsonnet/src/sjsonnet/Importer.scala b/sjsonnet/src/sjsonnet/Importer.scala index 60f341ba..3051bf92 100644 --- a/sjsonnet/src/sjsonnet/Importer.scala +++ b/sjsonnet/src/sjsonnet/Importer.scala @@ -1,20 +1,26 @@ package sjsonnet +import java.io.{BufferedInputStream, BufferedReader, ByteArrayInputStream, File, FileInputStream, FileReader, InputStream, RandomAccessFile, Reader, StringReader} +import java.nio.file.Files +import java.util.zip.CRC32 +import java.security.MessageDigest import scala.collection.mutable +import fastparse.{IndexedParserInput, Parsed, ParserInput} + +import java.nio.charset.StandardCharsets -import fastparse.Parsed /** Resolve and read imported files */ abstract class Importer { def resolve(docBase: Path, importName: String): Option[Path] - def read(path: Path): Option[String] + def read(path: Path): Option[ResolvedFile] - def resolveAndRead(docBase: Path, importName: String): Option[(Path, String)] = for { + def resolveAndRead(docBase: Path, importName: String): Option[(Path, ResolvedFile)] = for { path <- resolve(docBase, importName) txt <- read(path) } yield (path, txt) - def resolveAndReadOrFail(value: String, pos: Position)(implicit ev: EvalErrorScope): (Path, String) = + def resolveAndReadOrFail(value: String, pos: Position)(implicit ev: EvalErrorScope): (Path, ResolvedFile) = resolveAndRead(pos.fileScope.currentFile.parent(), value) .getOrElse(Error.fail("Couldn't import file: " + pprint.Util.literalize(value), pos)) } @@ -22,16 +28,146 @@ abstract class Importer { object Importer { val empty: Importer = new Importer { def resolve(docBase: Path, importName: String): Option[Path] = None - def read(path: Path): Option[String] = None + def read(path: Path): Option[ResolvedFile] = None + } +} + +case class FileParserInput(file: File) extends ParserInput { + + private[this] val bufferedFile = new BufferedRandomAccessFile(file.getAbsolutePath, 1024 * 8) + + private lazy val fileLength = file.length.toInt + + override def apply(index: Int): Char = { + bufferedFile.readChar(index) + } + + override def dropBuffer(index: Int): Unit = {} + + override def slice(from: Int, until: Int): String = { + bufferedFile.readString(from, until) + } + + override def length: Int = fileLength + + override def innerLength: Int = length + + override def isReachable(index: Int): Boolean = index < length + + override def checkTraceable(): Unit = {} + + private[this] lazy val lineNumberLookup: Array[Int] = { + val lines = mutable.ArrayBuffer[Int]() + val bufferedStream = new BufferedInputStream(new FileInputStream(file)) + var byteRead: Int = 0 + var currentPosition = 0 + + while ({ byteRead = bufferedStream.read(); byteRead != -1 }) { + if (byteRead == '\n') { + lines += currentPosition + 1 + } + currentPosition += 1 + } + + bufferedStream.close() + + lines.toArray + } + + def prettyIndex(index: Int): String = { + val line = lineNumberLookup.indexWhere(_ > index) match { + case -1 => lineNumberLookup.length - 1 + case n => math.max(0, n - 1) + } + val col = index - lineNumberLookup(line) + s"${line + 1}:${col + 1}" + } +} + +class BufferedRandomAccessFile(fileName: String, bufferSize: Int) { + + // The file is opened in read-only mode + private val file = new RandomAccessFile(fileName, "r") + + private val buffer = new Array[Byte](bufferSize) + + private var bufferStart: Long = -1 + + private var bufferEnd: Long = -1 + + private val fileLength: Long = file.length() + + private def fillBuffer(position: Long): Unit = { + if (file.getFilePointer() != position) { + file.seek(position) + } + val bytesRead = file.read(buffer, 0, bufferSize) + bufferStart = position + bufferEnd = position + bytesRead + } + + def readChar(index: Long): Char = { + if (index >= fileLength) { + throw new IndexOutOfBoundsException(s"Index $index is out of bounds for file of length $fileLength") + } + if (index < bufferStart || index >= bufferEnd) { + fillBuffer(index) + } + buffer((index - bufferStart).toInt).toChar } + + def readString(from: Long, until: Long): String = { + if (!(from < fileLength && until <= fileLength && from <= until)) { + throw new IndexOutOfBoundsException(s"Invalid range: $from-$until for file of length $fileLength") + } + val length = (until - from).toInt + + if (from >= bufferStart && until <= bufferEnd) { + // Range is within the buffer + new String(buffer, (from - bufferStart).toInt, length, StandardCharsets.UTF_8) + } else { + // Range is outside the buffer + val stringBytes = new Array[Byte](length) + file.seek(from) + file.readFully(stringBytes, 0, length) + new String(stringBytes, StandardCharsets.UTF_8) + } + } + + def close(): Unit = { + file.close() + } +} + +trait ResolvedFile { + /** + * Get an efficient parser input for this resolved file. Large files will be read from disk + * (buffered reads), while small files will be served from memory. + */ + def getParserInput(): ParserInput + + // Use this to read the file as a string. This is generally used for `importstr` + def readString(): String + + // Get a content hash of the file suitable for detecting changes in a given file. + def contentHash(): String +} + +case class StaticResolvedFile(content: String) extends ResolvedFile { + def getParserInput(): ParserInput = IndexedParserInput(content) + + def readString(): String = content + + // We just cheat, the content hash can be the content itself for static imports + lazy val contentHash: String = content } class CachedImporter(parent: Importer) extends Importer { - val cache = mutable.HashMap.empty[Path, String] + val cache = mutable.HashMap.empty[Path, ResolvedFile] def resolve(docBase: Path, importName: String): Option[Path] = parent.resolve(docBase, importName) - def read(path: Path): Option[String] = cache.get(path) match { + def read(path: Path): Option[ResolvedFile] = cache.get(path) match { case s @ Some(x) => if(x == null) None else s case None => @@ -46,9 +182,9 @@ class CachedResolver( val parseCache: ParseCache, strictImportSyntax: Boolean) extends CachedImporter(parentImporter) { - def parse(path: Path, txt: String)(implicit ev: EvalErrorScope): Either[Error, (Expr, FileScope)] = { - parseCache.getOrElseUpdate((path, txt), { - val parsed = fastparse.parse(txt, new Parser(path, strictImportSyntax).document(_)) match { + def parse(path: Path, content: ResolvedFile)(implicit ev: EvalErrorScope): Either[Error, (Expr, FileScope)] = { + parseCache.getOrElseUpdate((path, content.contentHash.toString), { + val parsed = fastparse.parse(content.getParserInput(), new Parser(path, strictImportSyntax).document(_)) match { case f @ Parsed.Failure(_, _, _) => val traced = f.trace() val pos = new Position(new FileScope(path), traced.index) diff --git a/sjsonnet/src/sjsonnet/Interpreter.scala b/sjsonnet/src/sjsonnet/Interpreter.scala index 45701e3f..c3608f75 100644 --- a/sjsonnet/src/sjsonnet/Interpreter.scala +++ b/sjsonnet/src/sjsonnet/Interpreter.scala @@ -34,7 +34,7 @@ class Interpreter(extVars: Map[String, String], def parseVar(k: String, v: String) = { - resolver.parse(wd / s"<$k>", v)(evaluator).fold(throw _, _._1) + resolver.parse(wd / s"<$k>", StaticResolvedFile(v))(evaluator).fold(throw _, _._1) } lazy val evaluator: Evaluator = createEvaluator( @@ -76,9 +76,10 @@ class Interpreter(extVars: Map[String, String], } def evaluate(txt: String, path: Path): Either[Error, Val] = { - resolver.cache(path) = txt + val resolvedImport = StaticResolvedFile(txt) + resolver.cache(path) = resolvedImport for{ - res <- resolver.parse(path, txt)(evaluator) + res <- resolver.parse(path, resolvedImport)(evaluator) (parsed, _) = res res0 <- handleException(evaluator.visitExpr(parsed)(ValScope.empty)) res = res0 match{ diff --git a/sjsonnet/src/sjsonnet/Std.scala b/sjsonnet/src/sjsonnet/Std.scala index 56889251..2ec01b38 100644 --- a/sjsonnet/src/sjsonnet/Std.scala +++ b/sjsonnet/src/sjsonnet/Std.scala @@ -1089,10 +1089,19 @@ class Std { } }, - builtin("xz", "v"){ (pos, ev, v: Val) => - v match{ - case Val.Str(_, value) => Platform.xzString(value) - case arr: Val.Arr => Platform.xzBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray) + builtinWithDefaults("xz", "v" -> null, "compressionLevel" -> Val.Null(dummyPos)){ (args, pos, ev) => + val compressionLevel: Option[Int] = args(1) match { + case Val.Null(_) => + // Use default compression level if the user didn't set one + None + case Val.Num(_, n) => + Some(n.toInt) + case x => + Error.fail("Cannot xz encode with compression level " + x.prettyName) + } + args(0) match { + case Val.Str(_, value) => Platform.xzString(value, compressionLevel) + case arr: Val.Arr => Platform.xzBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray, compressionLevel) case x => Error.fail("Cannot xz encode " + x.prettyName) } }, diff --git a/sjsonnet/test/src-jvm/sjsonnet/BufferedRandomAccessFileTests.scala b/sjsonnet/test/src-jvm/sjsonnet/BufferedRandomAccessFileTests.scala new file mode 100644 index 00000000..0ad1d221 --- /dev/null +++ b/sjsonnet/test/src-jvm/sjsonnet/BufferedRandomAccessFileTests.scala @@ -0,0 +1,285 @@ +package sjsonnet + +import utest._ +import java.io.{File, FileWriter} +import java.nio.file.Files +import scala.util.Random + +object BufferedRandomAccessFileTests extends TestSuite { + // Utility function to create a temporary file with known content + def createTempFile(content: String): File = { + val tempFile = Files.createTempFile(null, null).toFile + val writer = new FileWriter(tempFile) + try { + writer.write(content) + } finally { + writer.close() + } + tempFile + } + + // Test content and large test content + val testContent = "Hello, World! This is a test file with various content to thoroughly test the BufferedRandomAccessFile." + val largeTestContent = Random.alphanumeric.take(100000).mkString // 100k characters + val tempFile = createTempFile(testContent) + val largeTempFile = createTempFile(largeTestContent) + + val tests = Tests { + test("readChar") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Normal operation + assert(bufferedFile.readChar(0) == 'H') + assert(bufferedFile.readChar(7) == 'W') + + // Boundary conditions + assert(bufferedFile.readChar(testContent.length - 1) == '.') + intercept[IndexOutOfBoundsException] { + bufferedFile.readChar(testContent.length) + } + + bufferedFile.close() + } + + test("readString") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Normal operation + assert(bufferedFile.readString(0, 5) == "Hello") + assert(bufferedFile.readString(7, 12) == "World") + + // String within buffer + assert(bufferedFile.readString(0, 10) == "Hello, Wor") + + // String across buffer boundary + assert(bufferedFile.readString(5, 15) == ", World! T") + + // Boundary conditions + assert(bufferedFile.readString(testContent.length - 5, testContent.length) == "File.") + assert(bufferedFile.readString(0, testContent.length) == testContent) + intercept[IndexOutOfBoundsException] { + bufferedFile.readString(0, testContent.length + 1) + } + + bufferedFile.close() + } + + test("readChar") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 5) + + // Testing buffer reloading + assert(bufferedFile.readChar(4) == 'o') // Last char in initial buffer + assert(bufferedFile.readChar(5) == ',') // Triggers buffer reload + + // Testing reading same character, buffer should not reload + assert(bufferedFile.readChar(5) == ',') + + bufferedFile.close() + } + + test("readString") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Test reading across multiple buffer reloads + assert(bufferedFile.readString(5, 25).contains("World! This is a")) + + bufferedFile.close() + } + + test("bufferManagement") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 4) + + // Test reading with buffer boundary exactly at string end + assert(bufferedFile.readString(0, 4) == "Hell") + + // Test reading over the buffer boundary + assert(bufferedFile.readString(3, 8) == "lo, W") + + bufferedFile.close() + } + + test("errorHandling") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Test reading with invalid indices + intercept[java.io.IOException] { + bufferedFile.readChar(-1) + } + intercept[java.io.IOException] { + bufferedFile.readString(-1, 2) + } + intercept[IndexOutOfBoundsException] { + bufferedFile.readString(2, 1) // 'from' is greater than 'until' + } + + bufferedFile.close() + } + + test("varyingBufferSizes") { + val smallBufferFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 2) + val mediumBufferFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 15) + val largeBufferFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 50) + + // Small buffer + assert(smallBufferFile.readChar(1) == 'e') + assert(smallBufferFile.readString(0, 4) == "Hell") + + // Medium buffer + assert(mediumBufferFile.readString(7, 22) == "World! This is ") + + // Large buffer + assert(largeBufferFile.readString(14, 40).contains("This is a test file with v")) + + smallBufferFile.close() + mediumBufferFile.close() + largeBufferFile.close() + } + + test("edgeCases") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Read string at the very end of the file + assert(bufferedFile.readString(testContent.length - 6, testContent.length) == "sFile.") + + // Read single character at the end of the file + assert(bufferedFile.readChar(testContent.length - 1) == '.') + + bufferedFile.close() + } + + test("sequentialReads") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 20) + + // Sequential reads to test buffer usage + assert(bufferedFile.readChar(0) == 'H') + assert(bufferedFile.readString(1, 10) == "ello, Wor") + assert(bufferedFile.readString(10, 20) == "ld! This i") + + bufferedFile.close() + } + + test("invalidFile") { + // Attempting to create a BufferedRandomAccessFile with a non-existent file + intercept[Exception] { + val invalidFile = new BufferedRandomAccessFile("nonexistent.txt", 10) + invalidFile.close() + } + } + + // Test content + val testContent = "Hello, World! This is a test file with various content to thoroughly test the BufferedRandomAccessFile." + + test("bufferReloadsAndEdgeReads") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 15) + + // Reading across buffer reloads + assert(bufferedFile.readString(14, 29) == "This is a test ") + + // Edge case: Reading exactly at the buffer end + assert(bufferedFile.readChar(14) == 'T') // Last char of the first buffer load + + bufferedFile.close() + } + + test("readFullBuffer") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Read the full buffer length + assert(bufferedFile.readString(0, 10) == "Hello, Wor") + + bufferedFile.close() + } + + test("readBeyondBuffer") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Read beyond buffer length, should trigger multiple buffer loads + assert(bufferedFile.readString(0, 20).contains("Hello, World! This i")) + + bufferedFile.close() + } + + test("zeroLengthString") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Test for zero-length string + assert(bufferedFile.readString(10, 10) == "") + + bufferedFile.close() + } + + test("concurrentAccess") { + val bufferedFile1 = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + val bufferedFile2 = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + + // Concurrent access by two different instances + assert(bufferedFile1.readChar(0) == 'H') + assert(bufferedFile2.readString(0, 5) == "Hello") + + bufferedFile1.close() + bufferedFile2.close() + } + + test("readAfterClose") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 10) + bufferedFile.close() + + // Attempt to read after closing the file + intercept[Exception] { + bufferedFile.readChar(0) + } + } + + test("largeFileReads") { + val bufferedFile = new BufferedRandomAccessFile(largeTempFile.getAbsolutePath, 1024) + + // Read from various positions in a large file + assert(bufferedFile.readString(50000, 50010).length == 10) + assert(bufferedFile.readString(99990, 100000).length == 10) + + bufferedFile.close() + } + + test("characterEncoding") { + // This test assumes UTF-8 encoding, but the class may need modifications to handle different encodings + val testString = "こんにちは" // "Hello" in Japanese + val tempFileWithEncoding = createTempFile(testString) + val bufferedFile = new BufferedRandomAccessFile(tempFileWithEncoding.getAbsolutePath, 10) + + // Read a string with non-ASCII characters + assert(bufferedFile.readString(0, testString.getBytes("UTF-8").length) == testString) + + bufferedFile.close() + tempFileWithEncoding.delete() + } + + test("randomAccessReads") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 20) + + // Perform random access reads + val positions = List(10, 30, 15, 5, 25) + for (pos <- positions) { + bufferedFile.readChar(pos) // Just to trigger reads at random positions + } + + bufferedFile.close() + } + + test("sequentialAndRandomMixedReads") { + val bufferedFile = new BufferedRandomAccessFile(tempFile.getAbsolutePath, 15) + + // Perform sequential and random reads mixed + assert(bufferedFile.readString(0, 5) == "Hello") + assert(bufferedFile.readChar(20) == 's') + assert(bufferedFile.readString(10, 15) == "ld! T") + + bufferedFile.close() + } + } + + // Clean up the temporary files after tests + override def utestAfterAll(): Unit = { + tempFile.delete() + largeTempFile.delete() + } +} diff --git a/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala b/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala new file mode 100644 index 00000000..9b86918d --- /dev/null +++ b/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala @@ -0,0 +1,21 @@ +package sjsonnet + +import utest._ +import TestUtils.eval + +object StdXzTests extends TestSuite { + val tests = Tests { + test("xz"){ + eval("""std.xz([1, 2])""") + eval("""std.xz("hi")""") + eval("""std.xz([1, 2], compressionLevel = 0)""") + eval("""std.xz("hi", compressionLevel = 1)""") + val ex = intercept[Exception] { + // Compression level 10 is invalid + eval("""std.xz("hi", 10)""") + } + assert(ex.getMessage.contains("Unsupported preset: 10")) + } + } +} + diff --git a/sjsonnet/test/src/sjsonnet/Std0150FunctionsTests.scala b/sjsonnet/test/src/sjsonnet/Std0150FunctionsTests.scala index 66d04b98..a8e1cc79 100644 --- a/sjsonnet/test/src/sjsonnet/Std0150FunctionsTests.scala +++ b/sjsonnet/test/src/sjsonnet/Std0150FunctionsTests.scala @@ -121,8 +121,8 @@ object Std0150FunctionsTests extends TestSuite { override def resolve(docBase: Path, importName: String): Option[Path] = importName match{ case "bar.json" => Some(DummyPath("bar")) } - override def read(path: Path): Option[String] = path match{ - case DummyPath("bar") => Some("""{"x": "y"}""") + override def read(path: Path): Option[ResolvedFile] = path match{ + case DummyPath("bar") => Some(StaticResolvedFile("""{"x": "y"}""")) } }, parseCache = new DefaultParseCache,