multiple optimizations in TermsParsers

Author: jchapuis

build.sbt

@@ -1,7 +1,7 @@
 name := "scala-parser-combinators-completion"
 organization := "com.nexthink"
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))
-version := "1.0.6"
+version := "1.0.7"
 scalaVersion := "2.12.2"
 bintrayRepository := "maven"
 bintrayVcsUrl := Some("jchapuis@github.com:jchapuis/scala-parser-combinators-completion")

src/main/scala/com/nexthink/utils/parsing/combinator/completion/CompletionTypes.scala

@@ -215,7 +215,7 @@ trait CompletionTypes {
     }
 
     def completionStrings: Seq[String] =
-      sets.values.toSeq
+      sets.values.toStream
         .sortBy(_.score)
         .reverse
         .flatMap(_.stringEntries)
@@ -248,7 +248,7 @@ trait CompletionTypes {
   }
 
   private def encodeJson(meta: JValue) = compact(render(meta))
-  private def printJson(meta: JValue) = pretty(render(meta))
+  private def printJson(meta: JValue)  = pretty(render(meta))
 
   case object Completions {
     def apply(position: Position, meta: Option[String], completionSets: Seq[(String, CompletionSet)]): Completions =
@@ -261,7 +261,7 @@ trait CompletionTypes {
       Completions(position, meta, CompletionSet(completions))
     def apply(position: Position, completions: Traversable[Elems]): Completions =
       Completions(position, None, CompletionSet(completions))
-    def apply(position: Position, meta:Option[String], completionSets: Iterable[CompletionSet]): Completions =
+    def apply(position: Position, meta: Option[String], completionSets: Iterable[CompletionSet]): Completions =
       Completions(position, meta, completionSets.map(s => s.tag.label -> s).toSeq)
     def apply(position: Position, completionSets: Iterable[CompletionSet]): Completions =
       Completions(position, None, completionSets.map(s => s.tag.label -> s).toSeq)

src/main/scala/com/nexthink/utils/parsing/combinator/completion/TermsParsers.scala

@@ -18,19 +18,20 @@ import scala.util.parsing.combinator.RegexParsers
   * completion (supporting fuzzy matching)
   */
 trait TermsParsers extends RegexParsers with RegexCompletionSupport with TermsParsingHelpers {
-  val DefaultMaxCompletionsCount                  = 15  // exposed
+  val DefaultMaxCompletionsCount                  = 15 // exposed
   private val DefaultSimilarityThreshold          = 20
   private val CompletionCandidatesMultiplierRatio = 3
 
   /**
     * This defines a parser which parses any of the specified terms.
     * The parser performs a fast match by means of a trie data structure, initialized upon creation.
-    * Completions will return all available terms below the matching trie node (if any)
+    * Completions will return all available terms below the matching trie node, in alphabetical order (if any)
     * @param terms the list of terms to build the parser for
+    * @param maxCompletionsCount maximum number of completions returned by the parser
     * @return parser instance
     */
-  def oneOfTerms(terms: Seq[String]): Parser[String] = {
-    new TermsParser(terms)
+  def oneOfTerms(terms: Seq[String], maxCompletionsCount: Int = DefaultMaxCompletionsCount): Parser[String] = {
+    TermsParser(terms, maxCompletionsCount)
   }
 
   /**
@@ -59,17 +60,19 @@ trait TermsParsers extends RegexParsers with RegexCompletionSupport with TermsPa
                       similarityMeasure: (String, String) => Double = diceSorensenSimilarity,
                       similarityThreshold: Int = DefaultSimilarityThreshold,
                       maxCompletionsCount: Int = DefaultMaxCompletionsCount): Parser[String] = {
-    new FuzzyParser(terms, similarityMeasure, similarityThreshold, maxCompletionsCount)
+    FuzzyParser(terms, similarityMeasure, similarityThreshold, maxCompletionsCount)
   }
 
-  sealed private class TermsParser(terms: Seq[String]) extends Parser[String] {
-    protected def originalTerms: Seq[String]   = terms.map(_.trim()).filter(_.nonEmpty)
-    protected def normalizedTerms: Seq[String] = originalTerms.map(_.toLowerCase)
-
-    protected val trie: PrefixMap[String] = PrefixMap(normalizedTerms.zip(originalTerms).map {
-      case (normalizedTerm, originalTerm) => (normalizedTerm, originalTerm)
-    }: _*)
+  private object TermsParser {
+    def apply(terms: Seq[String], maxCompletionsCount: Int): TermsParser = {
+      val trie = PrefixMap(normalizedTerms(terms).zip(trimmedNonEmptyTerms(terms)).map {
+        case (normalizedTerm, originalTerm) => (normalizedTerm, originalTerm)
+      }: _*)
+      new TermsParser(trie, maxCompletionsCount)
+    }
+  }
 
+  sealed private class TermsParser(trie: PrefixMap[String], maxCompletionsCount: Int) extends Parser[String] {
     override def apply(in: Input): ParseResult[String] = {
       val start = dropAnyWhiteSpace(in)
       val (terms, finalPosition) =
@@ -90,57 +93,75 @@ trait TermsParsers extends RegexParsers with RegexCompletionSupport with TermsPa
         case Success(_, _) => Completions.empty
         case NoSuccess(_, _) =>
           val start = dropAnyWhiteSpace(in)
-          val terms = findAllTermsWithPrefix(start, start.offset, trie)
-          Completions(in.pos, CompletionSet(terms.map(t => Completion(t))))
+          val terms = alphabeticalCompletions(findAllTermsWithPrefix(start, start.offset, trie), maxCompletionsCount)
+          Completions(in.pos, terms)
       }
     }
   }
 
-  sealed private class FuzzyParser(terms: Seq[String], similarityMeasure: (String, String) => Double, similarityThreshold: Int, maxCompletionsCount: Int)
-      extends TermsParser(terms) {
+  private def trimmedNonEmptyTerms(terms: Seq[String]) = terms.map(_.trim()).filter(_.nonEmpty)
+  private def normalizedTerms(terms: Seq[String])      = terms.map(_.toLowerCase)
+
+  private def lazyQuicksort[A](xs: Stream[A])(implicit o: Ordering[A]): Stream[A] = {
+    import o._
+    if (xs.isEmpty) xs
+    else {
+      val (smaller, bigger) = xs.tail.partition(_ < xs.head)
+      lazyQuicksort(smaller) #::: xs.head #:: lazyQuicksort(bigger)
+    }
+  }
 
-    val ngramMap: PrefixMap[Array[String]] = {
+  private def alphabeticalCompletions(terms: Iterable[String], maxCompletionsCount: Int): CompletionSet =
+    CompletionSet(
+      lazyQuicksort(terms.toStream)
+        .take(maxCompletionsCount)
+        .reverse
+        .zipWithIndex
+        .map {
+          case (t, rank) => Completion(t, rank)
+        }
+        .toSet)
+
+  private object FuzzyParser {
+    def apply(terms: Seq[String], similarityMeasure: (String, String) => Double, similarityThreshold: Int, maxCompletionsCount: Int): FuzzyParser = {
+      val originals                 = trimmedNonEmptyTerms(terms)
+      val normalized                = normalizedTerms(terms)
+      val completionsWhenInputEmpty = alphabeticalCompletions(originals, maxCompletionsCount)
       val trigramTermPairs =
-        normalizedTerms.zip(originalTerms).par.flatMap {
+        normalized.zip(originals).par.flatMap {
           case (normalizedTerm, originalTerm) =>
             tokenizeWords(normalizedTerm).flatMap(trigramsWithAffixing).map(trigram => trigram -> originalTerm)
         }
-      PrefixMap(trigramTermPairs.groupBy(_._1).mapValues(_.map(_._2).toArray).toSeq.seq: _*)
+      val ngramMap = PrefixMap(trigramTermPairs.groupBy(_._1).mapValues(_.map(_._2).toArray).toSeq.seq: _*)
+      val trie = PrefixMap(normalized.zip(originals).map {
+        case (normalizedTerm, originalTerm) => (normalizedTerm, originalTerm)
+      }: _*)
+      new FuzzyParser(completionsWhenInputEmpty, ngramMap, trie, similarityMeasure, similarityThreshold, maxCompletionsCount)
     }
+  }
+
+  sealed private class FuzzyParser private (completionsWhenInputEmpty: CompletionSet,
+                                            ngramMap: PrefixMap[Array[String]],
+                                            trie: PrefixMap[String],
+                                            similarityMeasure: (String, String) => Double,
+                                            similarityThreshold: Int,
+                                            maxCompletionsCount: Int)
+      extends TermsParser(trie, maxCompletionsCount) {
 
     override def completions(in: Input): Completions = {
       apply(in) match {
         case Success(_, _) => Completions.empty
         case NoSuccess(_, _) =>
           val start = dropAnyWhiteSpace(in)
           if (start.atEnd) {
-            // return everything
-            alphabeticalCompletions(start)
+            Completions(in.pos, completionsWhenInputEmpty)
           } else {
             fuzzyCompletions(start)
           }
       }
     }
 
-    val maxCandidatesCount: Int = maxCompletionsCount * CompletionCandidatesMultiplierRatio
-
-    private def alphabeticalCompletions(in: Input): Completions = {
-      val matches = ngramMap
-        .withPrefix(remainder(in).toLowerCase)
-        .values
-        .flatten
-        .toList
-        .distinct
-        .sorted
-        .take(maxCompletionsCount)
-      if (matches.nonEmpty) {
-        Completions(in.pos, CompletionSet(matches.reverse.zipWithIndex.map {
-          case (t, rank) => Completion(t, rank)
-        }.toSet))
-      } else {
-        Completions.empty
-      }
-    }
+    private val maxCandidatesCount: Int = maxCompletionsCount * CompletionCandidatesMultiplierRatio
 
     private def findAndScoreNgramMatches(ngrams: Seq[String]): Map[String, Int] = {
       def iter(ngram: String, remainingNgrams: Seq[String], termsFromPreviousIter: Set[String], acc: Map[String, Int]): Map[String, Int] = {
@@ -163,15 +184,15 @@ trait TermsParsers extends RegexParsers with RegexCompletionSupport with TermsPa
     private def fuzzyCompletions(in: Input): Completions = {
       val incompleteTerm = remainder(in)
       val candidates     = findCandidateMatches(incompleteTerm)
-      val rankedCompletions = candidates
-        .map {
-          case (candidateTerm, _) =>
-            (candidateTerm, math.round(similarityMeasure(incompleteTerm, candidateTerm) * 100.0).toInt)
-        }
-        .filter { case (_, similarity) => similarity >= similarityThreshold }
-        .sortBy {
-          case (term, similarity) => (-similarity, term)
-        }
+      val rankedCompletions = lazyQuicksort(
+        candidates.toStream
+          .map {
+            case (candidateTerm, _) =>
+              (candidateTerm, math.round(similarityMeasure(incompleteTerm, candidateTerm) * 100.0).toInt)
+          }
+          .filter { case (_, similarity) => similarity >= similarityThreshold })(Ordering.by({
+        case (term, similarity) => (-similarity, term)
+      }))
         .take(maxCompletionsCount)
       if (rankedCompletions.nonEmpty) {
         Completions(in.pos, CompletionSet(rankedCompletions.map {
@@ -185,7 +206,7 @@ trait TermsParsers extends RegexParsers with RegexCompletionSupport with TermsPa
     private def findCandidateMatches(incompleteTerm: String): Seq[(String, Int)] = {
       val trigrams                        = trigramsWithAffixing(incompleteTerm.toLowerCase)
       val matchingTerms: Map[String, Int] = findAndScoreNgramMatches(trigrams)
-      matchingTerms.toSeq.sortBy(_._2).reverse.take(maxCandidatesCount)
+      matchingTerms.toSeq.sortBy(_._2).view.reverse.take(maxCandidatesCount)
     }
   }
 

src/main/scala/com/nexthink/utils/parsing/combinator/completion/TermsParsingHelpers.scala

@@ -23,25 +23,25 @@ trait TermsParsingHelpers { this: RegexParsers =>
     reader.source.subSequence(start, end).toString
   private def lastPosition[T](reader: Reader[T]): Int = reader.source.length
 
-  protected def findAllMatchingTerms(in: Input, pos: Int, map: PrefixMap[String]): (Seq[(String, Int)], Int) = {
-    def findAllMatchingTermsIter(in: Input, pos: Int, map: PrefixMap[String], prevMatches: Seq[(String, Int)]): (Seq[(String, Int)], Int) = {
+  protected def findAllMatchingTerms(in: Input, pos: Int, map: PrefixMap[String]): (Stream[(String, Int)], Int) = {
+    def findAllMatchingTermsIter(in: Input, pos: Int, map: PrefixMap[String], prevMatches: Stream[(String, Int)]): (Stream[(String, Int)], Int) = {
       lazy val nextSuffixChar = charAtPosition(in, pos)
       if (handleWhiteSpace(in.source, pos) < lastPosition(in) && map.hasSuffix(nextSuffixChar)) {
         findAllMatchingTermsIter(in, pos + 1, map.withPrefix(nextSuffixChar), prevMatches ++ map.value.map((_, pos)))
       } else {
         (prevMatches ++ map.value.map((_, pos)), pos)
       }
     }
-    findAllMatchingTermsIter(in, pos, map, Seq())
+    findAllMatchingTermsIter(in, pos, map, Stream())
   }
 
-  protected def findAllTermsWithPrefix(in: Input, pos: Int, map: PrefixMap[String]): Seq[String] = {
-    def findAllTermsWithPrefixIter(in: Input, pos: Int, map: PrefixMap[String]): Seq[String] = {
+  protected def findAllTermsWithPrefix(in: Input, pos: Int, map: PrefixMap[String]): Stream[String] = {
+    def findAllTermsWithPrefixIter(in: Input, pos: Int, map: PrefixMap[String]): Stream[String] = {
       lazy val nextSuffixChar = charAtPosition(in, pos)
       if (handleWhiteSpace(in.source, pos) < lastPosition(in) && map.hasSuffix(nextSuffixChar)) {
         findAllTermsWithPrefixIter(in, pos + 1, map.withPrefix(nextSuffixChar))
       } else {
-        map.toSeq.map { case (_, term) => term }
+        map.toStream.map { case (_, term) => term }
       }
     }
     findAllTermsWithPrefixIter(in, pos, map)

src/test/scala/com/nexthink/utils/parsing/combinator/completion/TermsParsersTest.scala

@@ -35,17 +35,17 @@ class TermsParsersTest extends PropSpec with PropertyChecks with Matchers with I
     val samples = Table(
       "skyp" -> "Skype, Skype Handsfree Support, Skype Monitor",
       "NEXT" -> "NEXThink Finder",
-      "A" -> "Activity Monitor, Adobe Acrobat"
+      "A"    -> "Activity Monitor, Adobe Acrobat"
     )
     forAll(samples) { (partial: String, options: String) =>
       val completedTerms = options.split(",").map(_.trim)
-      val completions = termsParsers$.completeString(terms, partial)
+      val completions    = termsParsers$.completeString(terms, partial)
       completions shouldBe completedTerms
     }
   }
 
   property("oneOfTerms returns correct next") {
-    val terms = termsParsers$.oneOfTerms(examples)
+    val terms  = termsParsers$.oneOfTerms(examples)
     val result = termsParsers$.parse(terms, "skype h")
     result.successful shouldBe true
     result.next.pos.column shouldBe 6
@@ -89,6 +89,34 @@ class TermsParsersTest extends PropSpec with PropertyChecks with Matchers with I
     }
   }
 
+  property("oneOfTerms with empty completes with all terms in alphabetical order") {
+    forAll(sampleTerms) { terms: List[String] =>
+      {
+        val parser      = termsParsers$.oneOfTerms(terms)
+        val completions = termsParsers$.complete(parser, " ")
+        withClue(s"terms=$terms, completions=$completions") {
+          completions.defaultSet.isDefined shouldBe true
+          terms.distinct.sorted.zipAll(completions.completionStrings, "extraCompletion", "missingCompletion").foreach {
+            case (expected, actual) => actual === expected
+          }
+        }
+      }
+    }
+  }
+
+  property("oneOfTerms with empty spaces completes at the last relevant input position") {
+    forAll(sampleTerms, Gen.chooseNum(1, 10)) { (terms: List[String], spacesCount: Int) =>
+      {
+        val spaces      = Seq.range(0, spacesCount).map(_ => " ").mkString
+        val parser      = termsParsers$.oneOfTerms(terms)
+        val completions = termsParsers$.complete(parser, spaces)
+        withClue(s"terms=$terms, completions=$completions") {
+          completions.position.column shouldBe 1
+        }
+      }
+    }
+  }
+
   property("oneOfTermsFuzzy with empty completes with all terms in alphabetical order") {
     forAll(sampleTerms) { terms: List[String] =>
       {