improvements in meta-data handling: renamed to meta & now supporting merging JSON meta-data

Author: Jonas Chapuis

README.md

@@ -20,7 +20,7 @@ Add the following lines to your `build.sbt` file:
 
  ```
  resolvers += Resolver.bintrayRepo("jchapuis", "maven")
- libraryDependencies += "com.nexthink" %% "scala-parser-combinators-completion" % "1.0.2"
+ libraryDependencies += "com.nexthink" %% "scala-parser-combinators-completion" % "1.0.3"
  ```
 
 ## Completing on a grammar
@@ -228,8 +228,8 @@ Completing on the same expressions (`2`, `2+`, `(10*2`) now leads to the followi
 |`%`     |defines the completions tag|<code>("+" &#124; "-") % "operators"</code>|
 |`%`     |followed by an `Int`, defines the completion tag score|<code>("+" &#124; "-") % 10</code>|
 |`%?`    |defines the description of the completions tag|<code>("+" &#124; "-") %? "arithmetic operators"</code>|
-|`%%`    |defines the completions tag meta-data (can be used to encode properties for the tag, e.g. visual decorations)|<code>("+" &#124; "-") %% "style: highlight"</code>|
-|`%-%`   |defines the completion meta-data (can be used to encode properties for each completion entry, e.g. visual decorations)|<code>("+" %-% "style: highlight" &#124; "-")</code>|   
+|`%%`    |defines the completions tag meta-data (can be used to encode properties for the tag in JSON, e.g. visual decorations)|<code>("+" &#124; "-") %% "{\\"style\\": \\"highlight\\"}"</code>|
+|`%-%`   |defines the completion meta-data (can be used to encode properties for each completion entry in JSON, e.g. visual decorations)|<code>("+" %-% "{\\"style\\": \\"highlight\\"}" &#124; "-")</code>|   
 
 ## Fuzzy completion
 

build.sbt

@@ -1,7 +1,7 @@
 name := "scala-parser-combinators-completion"
 organization := "com.nexthink"
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))
-version := "1.0.2"
+version := "1.0.3"
 scalaVersion := "2.12.2"
 bintrayRepository := "maven"
 bintrayVcsUrl := Some("jchapuis@github.com:jchapuis/scala-parser-combinators-completion")

src/main/scala/com/nexthink/utils/parsing/combinator/completion/CompletionExpansionSupport.scala

@@ -56,7 +56,7 @@ trait CompletionExpansionSupport extends RegexCompletionSupport {
       else
         completions.allSets
           .map(cSet => {
-            cSet.completions
+            cSet.entries
               .map(c => {
                 val completedInput = completeString(str, completions.position.column, c)
                 if (stop(new CharSequenceReader(completedInput)).successful) {

src/main/scala/com/nexthink/utils/parsing/combinator/completion/CompletionSupport.scala

@@ -115,9 +115,7 @@ trait CompletionSupport extends Parsers with CompletionTypes {
         this,
         in => {
           val completions = this.completions(in)
-          Completions(completions.position,
-                      completions.sets.mapValues(s =>
-                        CompletionSet(s.tag, s.completions.toList.sortBy(_.score).reverse.take(n).toSet)))
+          Completions(completions.position, completions.sets.mapValues(s => CompletionSet(s.tag, s.entries.toList.sortBy(_.score).reverse.take(n).toSet)).toSeq)
         }
       )
 
@@ -150,8 +148,7 @@ trait CompletionSupport extends Parsers with CompletionTypes {
       * @return wrapper `Parser` instance specifying completion tag
       */
     def %(tag: String, tagScore: Int, tagDescription: String): Parser[T] =
-      Parser(this,
-             in => updateCompletionsTag(this.completions(in), Some(tag), Some(tagScore), Some(tagDescription), None))
+      Parser(this, in => updateCompletionsTag(this.completions(in), Some(tag), Some(tagScore), Some(tagDescription), None))
 
     /** An operator to specify the completion tag, score, description and meta of a parser
       * @param tag the completion tag
@@ -161,19 +158,14 @@ trait CompletionSupport extends Parsers with CompletionTypes {
       * @return wrapper `Parser` instance specifying completion tag
       */
     def %(tag: String, tagScore: Int, tagDescription: String, tagKind: String): Parser[T] =
-      Parser(
-        this,
-        in =>
-          updateCompletionsTag(this.completions(in), Some(tag), Some(tagScore), Some(tagDescription), Some(tagKind)))
+      Parser(this, in => updateCompletionsTag(this.completions(in), Some(tag), Some(tagScore), Some(tagDescription), Some(tagKind)))
 
     /** An operator to specify the completion tag
       * @param tag the completion tag
       * @return wrapper `Parser` instance specifying completion tag
       */
     def %(tag: CompletionTag): Parser[T] =
-      Parser(
-        this,
-        in => updateCompletionsTag(this.completions(in), Some(tag.label), Some(tag.score), tag.description, tag.meta))
+      Parser(this, in => updateCompletionsTag(this.completions(in), Some(tag.label), Some(tag.score), tag.description, tag.meta))
 
     /** An operator to specify the completion tag description of a parser (empty by default)
       * @param tagDescription the completion description (to be used e.g. to add information to a completion entry)
@@ -182,14 +174,20 @@ trait CompletionSupport extends Parsers with CompletionTypes {
     def %?(tagDescription: String): Parser[T] =
       Parser(this, in => updateCompletionsTag(this.completions(in), None, None, Some(tagDescription), None))
 
-    /** An operator to specify the completion tag meta of a parser (empty by default)
+    /** An operator to specify the completion tag meta-data of a parser (empty by default).
+      * Note that if the meta-data is encoded in JSON, it is automatically merged when combining two equivalent tags
+      * (i.e. bearing the same label, but with a different payload). This allows for more flexibility when defining the grammar:
+      * various parsers can return the same completion tags with an additive effect on the meta-data (and the entries).
       * @param tagMeta the completion tag meta-data (to be used e.g. to specify the visual style for a completion tag in the menu)
       * @return wrapper `Parser` instance specifying the completion tag meta-data
       */
     def %%(tagMeta: String): Parser[T] =
       Parser(this, in => updateCompletionsTag(this.completions(in), None, None, None, Some(tagMeta)))
 
-    /** An operator to specify the meta for completions of a parser (empty by default)
+    /** An operator to specify the meta-data for completions of a parser (empty by default)
+      * Note that if the meta-data is encoded in JSON, it is automatically merged when combining two equivalent tags
+      * (i.e. bearing the same label, but with a different payload). This allows for more flexibility when defining the grammar:
+      * various parsers can return the same completion entries with an additive effect on the meta-data.
       * @param meta the completion meta-data (to be used e.g. to specify the visual style for a completion entry in the menu)
       * @return wrapper `Parser` instance specifying the completion meta-data
       */
@@ -222,7 +220,7 @@ trait CompletionSupport extends Parsers with CompletionTypes {
                   completions.sets.values
                     .map(updateSet)
                     .map(s => s.tag.label -> s)
-                    .toMap)
+                    .toSeq)
     }
 
     private def updateCompletionsTag(completions: Completions,
@@ -231,15 +229,14 @@ trait CompletionSupport extends Parsers with CompletionTypes {
                                      newTagDescription: Option[String],
                                      newTagKind: Option[String]) = {
       def updateSet(existingSet: CompletionSet) =
-        CompletionSet(existingSet.tag.update(newTagLabel, newTagScore, newTagDescription, newTagKind),
-                      existingSet.completions)
+        CompletionSet(existingSet.tag.update(newTagLabel, newTagScore, newTagDescription, newTagKind), existingSet.completions)
 
       updateCompletionsSets(completions, updateSet)
     }
 
     private def updateCompletions(completions: Completions, newCompletionKind: Option[String]) = {
       def updateSet(existingSet: CompletionSet) =
-        CompletionSet(existingSet.tag, existingSet.completions.map(e => e.updateKind(newCompletionKind)))
+        CompletionSet(existingSet.tag, existingSet.entries.map(e => e.updateKind(newCompletionKind)))
       updateCompletionsSets(completions, updateSet)
     }
 
@@ -898,5 +895,4 @@ trait CompletionSupport extends Parsers with CompletionTypes {
   def phrase[T](p: Parser[T]): Parser[T] =
     Parser(super.phrase(p), p.completions)
 
-
 }

src/main/scala/com/nexthink/utils/parsing/combinator/completion/CompletionTypes.scala

@@ -10,9 +10,9 @@ package com.nexthink.utils.parsing.combinator.completion
 import org.json4s
 
 import scala.util.parsing.input.{NoPosition, Position}
-import org.json4s._
 import org.json4s.JsonDSL._
 import org.json4s.native.JsonMethods._
+import scala.collection.immutable
 
 /**  Collection of data types allowing definition of structured parser completions.
   *  A `Completions` instance can contain multiple `CompletionSet`s instances. A `CompletionSet` provides a set of
@@ -73,45 +73,52 @@ trait CompletionTypes {
     * @param tag set tag
     * @param completions set of unique completion entries
     */
-  case class CompletionSet(tag: CompletionTag, completions: Set[Completion]) {
-    def label: String               = tag.label
-    def score: Int                  = tag.score
-    def description: Option[String] = tag.description
-    def meta: Option[String]        = tag.meta
-    def completionStrings: Seq[String] =
-      completions.toSeq.sorted.map(_.value.toString)
+  case class CompletionSet(tag: CompletionTag, completions: immutable.HashMap[Elems, Completion]) {
+    def label: String                  = tag.label
+    def score: Int                     = tag.score
+    def description: Option[String]    = tag.description
+    def meta: Option[String]           = tag.meta
+    def entries: Iterable[Completion]  = completions.values
+    def sortedEntries: Seq[Completion] = entries.toSeq.sorted
+    def stringEntries: Seq[String]     = sortedEntries.map(_.value.toString)
 
     private[CompletionTypes] def serializeJson =
-      ("tag" -> tag.serializeJson) ~ ("completions" -> completions.map(_.serializeJson).toList)
+      ("tag" -> tag.serializeJson) ~ ("completions" -> entries.map(_.serializeJson).toList)
 
     override def toString: String = pretty(render(serializeJson))
     def toJson: String            = compact(render(serializeJson))
   }
 
   case object CompletionSet {
+    def apply(tag: CompletionTag, completions: Seq[(Elems, Completion)]): CompletionSet =
+      CompletionSet(tag, immutable.HashMap(completions: _*))
+
     def apply(tag: String, el: Elem): CompletionSet =
-      CompletionSet(CompletionTag(tag), Set(Completion(el)))
+      CompletionSet(CompletionTag(tag), Seq(Seq(el) -> Completion(el)))
 
     def apply(tag: String, elems: Elems): CompletionSet =
-      CompletionSet(CompletionTag(tag), Set(Completion(elems)))
+      CompletionSet(CompletionTag(tag), Seq(elems -> Completion(elems)))
 
     def apply(tag: String, completion: Completion): CompletionSet =
-      CompletionSet(CompletionTag(tag), Set(completion))
+      CompletionSet(CompletionTag(tag), Seq(completion.value -> completion))
+
+    def apply(tag: CompletionTag, completions: Iterable[Completion]): CompletionSet =
+      CompletionSet(tag, completions.map(c => c.value -> c).toSeq)
 
     def apply(tag: String, completions: Iterable[Completion]): CompletionSet =
-      CompletionSet(CompletionTag(tag), completions.toSet)
+      CompletionSet(CompletionTag(tag), completions.map(c => c.value -> c).toSeq)
 
     def apply(completions: Iterable[Completion]): CompletionSet =
-      CompletionSet(CompletionTag.Default, completions.toSet)
+      CompletionSet(CompletionTag.Default, completions.map(c => c.value -> c).toSeq)
 
     def apply(completions: Completion*): CompletionSet =
-      CompletionSet(CompletionTag.Default, completions.toSet)
+      CompletionSet(CompletionTag.Default, completions.map(c => c.value -> c))
 
     def apply(el: Elem): CompletionSet =
-      CompletionSet(CompletionTag.Default, Set(Completion(el)))
+      CompletionSet(CompletionTag.Default, Seq(Seq(el) -> Completion(el)))
 
     def apply(completions: Traversable[Elems]): CompletionSet =
-      CompletionSet(CompletionTag.Default, completions.map(Completion(_)).toSet)
+      CompletionSet(CompletionTag.Default, completions.map(c => c -> Completion(c)).toSeq)
   }
 
   type Elems = Seq[Elem]
@@ -142,46 +149,45 @@ trait CompletionTypes {
     * @param position position in the input where completion entries apply
     * @param sets completion entries, grouped per tag
     */
-  case class Completions(position: Position, sets: Map[String, CompletionSet]) {
+  case class Completions(position: Position, sets: immutable.HashMap[String, CompletionSet]) {
     def isEmpty: Boolean                               = sets.isEmpty
     def nonEmpty: Boolean                              = !isEmpty
     def setWithTag(tag: String): Option[CompletionSet] = sets.get(tag)
     def allSets: Iterable[CompletionSet]               = sets.values
-    def allCompletions: Iterable[Completion]           = allSets.flatMap(_.completions)
+    def allCompletions: Iterable[Completion]           = allSets.flatMap(_.sortedEntries)
     def defaultSet: Option[CompletionSet]              = sets.get("")
 
     private def serializeJson = ("position" -> (("line" -> position.line) ~ ("column" -> position.column))) ~ ("sets" -> allSets.map(_.serializeJson))
 
     override def toString: String = pretty(render(serializeJson))
     def toJson: String            = compact(render(serializeJson))
 
-    private def unionSets(left: CompletionSet, right: CompletionSet): CompletionSet = {
-      def offsetCompletions(set: CompletionSet) = {
-        val isOffsetRequired =
-          set.completions.map(_.score).exists(_ < set.score)
-        if (isOffsetRequired)
-          set.completions.map(c => Completion(c.value, set.score + c.score, c.meta))
-        else set.completions
-      }
-      CompletionSet(
-        CompletionTag(left.tag.label, left.score.min(right.score), left.description, left.meta.orElse(right.meta)),
-        offsetCompletions(left) ++ offsetCompletions(right)
+    private def mergeMetaData(left: Option[String], right: Option[String]) = (left, right) match {
+      case (Some(l), Some(r)) =>
+        (parseOpt(l), parseOpt(r)) match {
+          case (Some(lJson), Some(rJson)) => Some(compact(render(lJson merge rJson)))
+          case _                          => Some(l + r)
+        }
+      case (Some(l), None) => Some(l)
+      case (None, Some(r)) => Some(r)
+      case (None, None)    => None
+    }
+
+    private def mergeCompletion(left: Completion, right: Completion): Completion = {
+      assert(left.value == right.value, "Attempt to merge different completion entries")
+      Completion(
+        left.value,
+        left.score.max(right.score),
+        mergeMetaData(left.meta, right.meta)
       )
     }
 
-    private def mergeCompletions(other: Completions) = {
-      val overlappingSetTags = sets.keySet.intersect(other.sets.keySet)
-      val unions =
-        overlappingSetTags.map(name => (sets(name), other.sets(name))).map {
-          case (left, right) => unionSets(left, right)
-        }
-      val leftExclusive = sets.keySet.diff(overlappingSetTags).map(sets(_))
-      val rightExclusive =
-        other.sets.keySet.diff(overlappingSetTags).map(other.sets(_))
-      Completions(position,
-                  (unions ++ leftExclusive ++ rightExclusive)
-                    .map(s => s.tag.label -> s)
-                    .toMap)
+    private def mergeSets(left: CompletionSet, right: CompletionSet): CompletionSet = {
+      assert(left.label == right.label, "Attempt to merge sets with different completion tags")
+      CompletionSet(
+        CompletionTag(left.tag.label, left.score.max(right.score), left.description.orElse(right.description), mergeMetaData(left.meta, right.meta)),
+        left.completions.merged(right.completions)((l, r) => (l._1, mergeCompletion(l._2, r._2)))
+      )
     }
 
     def |(other: Completions): Completions = {
@@ -190,7 +196,7 @@ trait CompletionTypes {
         case _ =>
           other.position match {
             case otherPos if otherPos < position  => this
-            case otherPos if otherPos == position => mergeCompletions(other)
+            case otherPos if otherPos == position => Completions(position, sets.merged(other.sets)((l, r) => (l._1, mergeSets(l._2, r._2))))
             case _                                => other
           }
       }
@@ -200,17 +206,17 @@ trait CompletionTypes {
       sets.values.toSeq
         .sortBy(_.score)
         .reverse
-        .flatMap(_.completionStrings)
+        .flatMap(_.stringEntries)
         .toList
 
     def takeTop(count: Int): Completions = {
       val allEntries = allSets
-        .flatMap(s => s.completions.map((_, s.tag)))
+        .flatMap(s => s.completions.values.map((_, s.tag)))
         .toList
       val sortedEntries =
         allEntries
           .sortBy {
-            case (Completion(_, score, meta), CompletionTag(_, tagScore, _, _)) =>
+            case (Completion(_, score, _), CompletionTag(_, tagScore, _, _)) =>
               (tagScore, score)
           }
           .reverse
@@ -219,27 +225,31 @@ trait CompletionTypes {
         .groupBy { case (_, tag) => tag }
         .map {
           case (groupTag, completions) =>
-            CompletionSet(groupTag, completions.map(_._1).toSet)
+            CompletionSet(groupTag, completions.map(c => c._1))
         }
-      copy(sets = regroupedSets.map(s => (s.tag.label, s)).toMap)
+      Completions(position, regroupedSets.map(s => s.tag.label -> s).toSeq)
     }
 
     def setsScoredWithMaxCompletion(): Completions = {
-      Completions(position, sets.mapValues(s => CompletionSet(s.tag.copy(score = s.completions.map(_.score).max), s.completions)))
+      Completions(position, sets.mapValues(s => CompletionSet(s.tag.copy(score = s.completions.values.map(_.score).max), s.completions)).toSeq)
     }
   }
 
   case object Completions {
+    def apply(position: Position, completionSets: Seq[(String, CompletionSet)]): Completions =
+      Completions(position, immutable.HashMap(completionSets: _*))
     def apply(position: Position, completionSet: CompletionSet): Completions =
-      Completions(position, Map(completionSet.tag.label -> completionSet))
+      Completions(position, Seq(completionSet.tag.label -> completionSet))
     def apply(position: Position, completions: Traversable[Elems]): Completions =
       Completions(position, CompletionSet(completions))
     def apply(completionSet: CompletionSet): Completions =
       Completions(NoPosition, completionSet)
+    def apply(position: Position, completionSets: Iterable[CompletionSet]): Completions =
+      Completions(position, completionSets.map(s => s.tag.label -> s).toSeq)
     def apply(completionSets: Iterable[CompletionSet]): Completions =
-      Completions(NoPosition, completionSets.map(s => s.tag.label -> s).toMap)
+      Completions(NoPosition, completionSets.map(s => s.tag.label -> s).toSeq)
 
-    val empty = Completions(NoPosition, Map[String, CompletionSet]())
+    val empty = Completions(NoPosition, immutable.HashMap[String, CompletionSet]())
   }
 
 }