Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Key functions #40

Merged
merged 18 commits into from
Oct 9, 2019
Merged

Key functions #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c1687b4
Regex support
Sep 24, 2019
87da69e
All tests work except quoteMeta - RE2 is not public
Sep 24, 2019
64ce644
Regex added to Std lib
Sep 25, 2019
ec710b3
Moved RE2 code into the Platform
Sep 30, 2019
fb7f671
re2.js support
Sep 30, 2019
5dab570
Should be re2, not re2.js
Sep 30, 2019
fe7a5ca
Support for RE2-style regular expressions
Oct 3, 2019
6bf9747
SetMember with key function
Oct 6, 2019
b43fb86
Update Std.scala
Oct 6, 2019
15ff82c
Sort with key function
Oct 6, 2019
6f64f62
More Set functions
Oct 6, 2019
f0afa78
setUnion implemented
Oct 7, 2019
12446d7
Optimized setMember
Oct 7, 2019
d52a317
Even simpler setMember
Oct 7, 2019
7f0d142
setInter and setDiff
Oct 7, 2019
2ee0e01
Removed old code
Oct 7, 2019
148b1dd
Update Platform.scala
Oct 9, 2019
4bc6745
Removed Regex for PR
Oct 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.sc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,4 +136,4 @@ class SjsonnetModule(val crossScalaVersion: String) extends Module {
}
}

}
}
321 changes: 216 additions & 105 deletions sjsonnet/src/sjsonnet/Std.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@ import java.io.StringWriter
import java.util.Base64

import sjsonnet.Expr.Member.Visibility
import sjsonnet.Expr.Params
import sjsonnet.Expr.{BinaryOp, False, Params}

import scala.collection.mutable.ArrayBuffer
import scala.collection.compat._
import sjsonnet.Std.builtinWithDefaults
import ujson.Value

import util.control.Breaks._

/**
* The Jsonnet standard library, `std`, with each builtin function implemented
Expand Down Expand Up @@ -478,135 +482,159 @@ object Std {
builtin("base64DecodeBytes", "s"){ (ev, fs, s: String) =>
Val.Arr(Base64.getDecoder().decode(s).map(i => Val.Lazy(Val.Num(i))))
},
builtin("sort", "arr"){ (ev, fs, arr: Val) =>
arr match{
case Val.Arr(vs) =>
Val.Arr(

if (vs.forall(_.force.isInstanceOf[Val.Str])){
vs.map(_.force.cast[Val.Str]).sortBy(_.value).map(Val.Lazy(_))
}else if (vs.forall(_.force.isInstanceOf[Val.Num])){
vs.map(_.force.cast[Val.Num]).sortBy(_.value).map(Val.Lazy(_))
}else {
???
}
)
case Val.Str(s) => Val.Arr(s.sorted.map(c => Val.Lazy(Val.Str(c.toString))))
case x => throw new Error.Delegate("Cannot sort " + x.prettyName)
}
},
builtin("uniq", "arr"){ (ev, fs, arr: Val.Arr) =>
val ujson.Arr(vs) = Materializer(arr)(ev)
val out = collection.mutable.Buffer.empty[ujson.Value]
for(v <- vs) if (out.isEmpty || out.last != v) out.append(v)

Val.Arr(out.map(v => Val.Lazy(Materializer.reverse(v))).toSeq)
},
builtin("set", "arr"){ (ev, fs, arr: Val.Arr) =>
val ujson.Arr(vs0) = Materializer(arr)(ev)
val vs =
if (vs0.forall(_.isInstanceOf[ujson.Str])){
vs0.map(_.asInstanceOf[ujson.Str]).sortBy(_.value)
}else if (vs0.forall(_.isInstanceOf[ujson.Num])){
vs0.map(_.asInstanceOf[ujson.Num]).sortBy(_.value)
}else {
throw new Error.Delegate("Every element of the input must be of the same type, string or number")
}

val out = collection.mutable.Buffer.empty[ujson.Value]
for(v <- vs) if (out.isEmpty || out.last != v) out.append(v)
builtinWithDefaults("uniq", "arr" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>
val arr = args("arr")
val keyF = args("keyF")

Val.Arr(out.map(v => Val.Lazy(Materializer.reverse(v))).toSeq)
uniqArr(ev, arr, keyF)
},
builtin("setUnion", "a", "b"){ (ev, fs, a: Val.Arr, b: Val.Arr) =>
builtinWithDefaults("sort", "arr" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>
val arr = args("arr")
val keyF = args("keyF")

val ujson.Arr(vs1) = Materializer(a)(ev)
val ujson.Arr(vs2) = Materializer(b)(ev)
val vs0 = vs1 ++ vs2
val vs =
if (vs0.forall(_.isInstanceOf[ujson.Str])){
vs0.map(_.asInstanceOf[ujson.Str]).sortBy(_.value)
}else if (vs0.forall(_.isInstanceOf[ujson.Num])){
vs0.map(_.asInstanceOf[ujson.Num]).sortBy(_.value)
}else {
throw new Error.Delegate("Every element of the input must be of the same type, string or number")
}
sortArr(ev, arr, keyF)
},

val out = collection.mutable.Buffer.empty[ujson.Value]
for(v <- vs) if (out.isEmpty || out.last != v) out.append(v)
builtinWithDefaults("set", "arr" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>
uniqArr(ev, sortArr(ev, args("arr"), args("keyF")), args("keyF"))
},
builtinWithDefaults("setUnion", "a" -> None, "b" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>
val a = args("a") match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Arguments must be either arrays or strings")
}
val b = args("b") match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Arguments must be either arrays or strings")
}

Val.Arr(out.map(v => Val.Lazy(Materializer.reverse(v))).toSeq)
val concat = Val.Arr(a ++ b)
uniqArr(ev, sortArr(ev, concat, args("keyF")), args("keyF"))
},
builtin("setInter", "a", "b"){ (ev, fs, a: Val, b: Val.Arr) =>
val vs1 = Materializer(a)(ev) match{
case ujson.Arr(vs1) => vs1
case x => Seq(x)
builtinWithDefaults("setInter", "a" -> None, "b" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>
val a = args("a") match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Arguments must be either arrays or strings")
}
val b = args("b") match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Arguments must be either arrays or strings")
}
val ujson.Arr(vs2) = Materializer(b)(ev)

val keyF = args("keyF")
val out = collection.mutable.Buffer.empty[Val.Lazy]

val vs0 = vs1.to(collection.mutable.LinkedHashSet)
.intersect(vs2.to(collection.mutable.LinkedHashSet))
.toSeq
val vs =
if (vs0.forall(_.isInstanceOf[ujson.Str])){
vs0.map(_.asInstanceOf[ujson.Str]).sortBy(_.value)
}else if (vs0.forall(_.isInstanceOf[ujson.Num])){
vs0.map(_.asInstanceOf[ujson.Num]).sortBy(_.value)
}else {
throw new Error.Delegate("Every element of the input must be of the same type, string or number")
for (v <- a) {
if (keyF == Val.False) {
val mv = Materializer.apply(v.force)(ev)
if (b.exists(value => {
val mValue = Materializer.apply(value.force)(ev)
mValue == mv
}) && !out.exists(value => {
val mValue = Materializer.apply(value.force)(ev)
mValue == mv
})) {
out.append(v)
}
} else {
val keyFFunc = keyF.asInstanceOf[Val.Func]
val keyFApplyer = Applyer(keyFFunc, ev, null)
val appliedX = keyFApplyer.apply(v)

if (b.exists(value => {
val appliedValue = keyFApplyer.apply(value)
appliedValue == appliedX
}) && !out.exists(value => {
val mValue = keyFApplyer.apply(value)
mValue == appliedX
})) {
out.append(v)
}
}
}

val out = collection.mutable.Buffer.empty[ujson.Value]
for(v <- vs) if (out.isEmpty || out.last != v) out.append(v)

Val.Arr(out.map(v => Val.Lazy(Materializer.reverse(v))).toSeq)
sortArr(ev, Val.Arr(out.toSeq), keyF)
},
builtin("setDiff", "a", "b"){ (ev, fs, a: Val.Arr, b: Val.Arr) =>
val ujson.Arr(vs1) = Materializer(a)(ev)
val ujson.Arr(vs2) = Materializer(b)(ev)
builtinWithDefaults("setDiff", "a" -> None, "b" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>

val a = args("a") match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Arguments must be either arrays or strings")
}
val b = args("b") match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Arguments must be either arrays or strings")
}

val vs0 = vs1.to(collection.mutable.LinkedHashSet)
.diff(vs2.to(collection.mutable.LinkedHashSet))
.toSeq
val vs =
if (vs0.forall(_.isInstanceOf[ujson.Str])){
vs0.map(_.asInstanceOf[ujson.Str]).sortBy(_.value)
}else if (vs0.forall(_.isInstanceOf[ujson.Num])){
vs0.map(_.asInstanceOf[ujson.Num]).sortBy(_.value)
}else {
throw new Error.Delegate("Every element of the input must be of the same type, string or number")
}

val out = collection.mutable.Buffer.empty[ujson.Value]
for(v <- vs) if (out.isEmpty || out.last != v) out.append(v)
val keyF = args("keyF")
val out = collection.mutable.Buffer.empty[Val.Lazy]

Val.Arr(out.map(v => Val.Lazy(Materializer.reverse(v))).toSeq)
for (v <- a) {
if (keyF == Val.False) {
val mv = Materializer.apply(v.force)(ev)
if (!b.exists(value => {
val mValue = Materializer.apply(value.force)(ev)
mValue == mv
}) && !out.exists(value => {
val mValue = Materializer.apply(value.force)(ev)
mValue == mv
})) {
out.append(v)
}
} else {
val keyFFunc = keyF.asInstanceOf[Val.Func]
val keyFApplyer = Applyer(keyFFunc, ev, null)
val appliedX = keyFApplyer.apply(v)

if (!b.exists(value => {
val appliedValue = keyFApplyer.apply(value)
appliedValue == appliedX
}) && !out.exists(value => {
val mValue = keyFApplyer.apply(value)
mValue == appliedX
})) {
out.append(v)
}
}
}

sortArr(ev, Val.Arr(out.toSeq), keyF)
},
builtinWithDefaults("setMember", "x" -> None, "arr" -> None, "keyF" -> Some(Expr.False(0))) { (args, ev) =>
val keyF = args("keyF")

if (keyF == Val.False) {
val ujson.Arr(mArr) = Materializer(args("arr"))(ev)
val mx = Materializer(args("x"))(ev)
mArr.contains(mx)
} else {
val x = Val.Lazy(args("x"))
val arr = args("arr").asInstanceOf[Val.Arr].value
val keyFFunc = keyF.asInstanceOf[Val.Func]
val keyFApplyer = Applyer(keyFFunc, ev, null)
val appliedX = keyFApplyer.apply(x)
arr.exists(value => {
val appliedValue = keyFApplyer.apply(value)
appliedValue == appliedX
})
}
},
builtin("setMember", "x", "arr"){ (ev, fs, x: Val, arr: Val.Arr) =>
val vs1 = Materializer(x)(ev)
val ujson.Arr(vs2) = Materializer(arr)(ev)
vs2.contains(vs1)
},

builtin("split", "str", "c"){ (ev, fs, str: String, c: String) =>
Val.Arr(str.split(java.util.regex.Pattern.quote(c), -1).map(s => Val.Lazy(Val.Str(s))))
},
builtin("splitLimit", "str", "c", "maxSplits"){ (ev, fs, str: String, c: String, maxSplits: Int) =>
Val.Arr(str.split(java.util.regex.Pattern.quote(c), maxSplits + 1).map(s => Val.Lazy(Val.Str(s))))
},
builtin("stringChars", "str"){ (ev, fs, str: String) =>

var offset = 0
val output = collection.mutable.Buffer.empty[String]
while (offset < str.length) {
val codepoint = str.codePointAt(offset)
output.append(new String(Character.toChars(codepoint)))
offset += Character.charCount(codepoint)
}
Val.Arr(output.map(s => Val.Lazy(Val.Str(s))).toSeq)

stringChars(str)
},
builtin("parseInt", "str"){ (ev, fs, str: String) =>
str.toInt
Expand Down Expand Up @@ -676,6 +704,7 @@ object Std {
scope.bindings(1).get.force
}
),

"extVar" -> Val.Func(
None,
Params(Array(("x", None, 0))),
Expand Down Expand Up @@ -791,4 +820,86 @@ object Std {
None, None, None, Array(Val.Lazy(Std)).padTo(size, null)
)
}

def uniqArr(ev: EvalScope, arr: Val, keyF: Val) = {
val arrValue = arr match {
case arr: Val.Arr => arr.value
case str: Val.Str => stringChars(str.value).value
case _ => throw new Error.Delegate("Argument must be either array or string")
}

val out = collection.mutable.Buffer.empty[Val.Lazy]
for (v <- arrValue) {
if (out.isEmpty) {
out.append(v)
} else if (keyF == Val.False) {
val ol = Materializer.apply(out.last.force)(ev)
val mv = Materializer.apply(v.force)(ev)
if (ol != mv) {
out.append(v)
}
} else if (keyF != Val.False) {
val keyFFunc = keyF.asInstanceOf[Val.Func]
val keyFApplyer = Applyer(keyFFunc, ev, null)

val o1Key = keyFApplyer.apply(v)
val o2Key = keyFApplyer.apply(out.last)
val o1KeyExpr = Materializer.toExpr(Materializer.apply(o1Key)(ev))
val o2KeyExpr = Materializer.toExpr(Materializer.apply(o2Key)(ev))

val comparisonExpr = Expr.BinaryOp(0, o1KeyExpr, BinaryOp.`!=`, o2KeyExpr)
val exprResult = ev.visitExpr(comparisonExpr)(scope(0), new FileScope(null, Map.empty))

val res = Materializer.apply(exprResult)(ev).asInstanceOf[ujson.Bool]

if (res.value) {
out.append(v)
}
}
}

Val.Arr(out.toSeq)
}

def sortArr(ev: EvalScope, arr: Val, keyF: Val) = {
arr match{
case Val.Arr(vs) =>
Val.Arr(

if (vs.forall(_.force.isInstanceOf[Val.Str])){
vs.map(_.force.cast[Val.Str]).sortBy(_.value).map(Val.Lazy(_))
}else if (vs.forall(_.force.isInstanceOf[Val.Num])) {
vs.map(_.force.cast[Val.Num]).sortBy(_.value).map(Val.Lazy(_))
}else if (vs.forall(_.force.isInstanceOf[Val.Obj])){
if (keyF == Val.False) {
throw new Error.Delegate("Unable to sort array of objects without key function")
} else {
val keyFFunc = keyF.asInstanceOf[Val.Func]
val keyFApplyer = Applyer(keyFFunc, ev, null)
vs.map(_.force.cast[Val.Obj]).sortWith((o1, o2) => {
val o1Key = keyFApplyer.apply(Val.Lazy(o1))
val o2Key = keyFApplyer.apply(Val.Lazy(o2))
val o1KeyExpr = Materializer.toExpr(Materializer.apply(o1Key)(ev))
val o2KeyExpr = Materializer.toExpr(Materializer.apply(o2Key)(ev))

val comparisonExpr = Expr.BinaryOp(0, o1KeyExpr, BinaryOp.`<`, o2KeyExpr)
val exprResult = ev.visitExpr(comparisonExpr)(scope(0), new FileScope(null, Map.empty))
val res = Materializer.apply(exprResult)(ev).asInstanceOf[ujson.Bool]
res.value
}).map(Val.Lazy(_))
}
}else {
???
}
)
case Val.Str(s) => Val.Arr(s.sorted.map(c => Val.Lazy(Val.Str(c.toString))))
case x => throw new Error.Delegate("Cannot sort " + x.prettyName)
}
}

def stringChars(str: String): Val.Arr = {
var offset = 0
val output = str.toSeq.sliding(1).toList
Val.Arr(output.map(s => Val.Lazy(Val.Str(s.toString()))).toSeq)
}
}
Loading