Merge pull request #1 from jchapuis/expansion

Expansion mechanism

Author: Jonas Chapuis

README.md

@@ -233,11 +233,11 @@ Completing on the same expressions (`2`, `2+`, `(10*2`) now leads to the followi
 
 ## Fuzzy completion
 
-This library also provides special parsers which support fuzzy completion, present in the `FuzzyParsers` trait, by means of the `oneOfTerms` method capable of fuzzing completion on the input to match a set of terms (note that parsing itself obviously requires an exact match and is really fast thanks to a prefix trie lookup on each input char). For instance, with the following dummy grammar:
+This library also provides special parsers which support fuzzy completion, present in the `TermsParsers` trait, by means of the `oneOfTermsFuzzy` method capable of fuzzing completion on the input to match a set of terms (note that parsing itself obviously requires an exact match and is fast thanks to a prefix trie lookup on each input char). For instance, with the following dummy grammar:
 
 ```scala
-object Grammar extends FuzzyParsers {
-  val fuzzyCountries = "my favourite country is " ~ oneOfTerms(Seq("United States of America", "Afghanistan", "Albania", "Algeria", "Andorra", "Angola", "Antigua & Deps", "Argentina", "Armenia", "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain", "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize", "Benin", "Bhutan", "Bolivia", "Bosnia Herzegovina", "Botswana", "Brazil", "Brunei", "Bulgaria", "Burkina", "Burma", "Burundi", "Cambodia", "Cameroon", "Canada", "Cape Verde", "Central African Rep", "Chad", "Chile", "People's Republic of China", "Republic of China", "Colombia", "Comoros", "Democratic Republic of the Congo", "Republic of the Congo", "Costa Rica,", "Croatia", "Cuba", "Cyprus", "Czech Republic", "Danzig", "Denmark", "Djibouti", "Dominica", "Dominican Republic", "East Timor", "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Ethiopia", "Fiji", "Finland", "France", "Gabon", "Gaza Strip", "The Gambia", "Georgia", "Germany", "Ghana", "Greece", "Grenada", "Guatemala", "Guinea", "Guinea-Bissau", "Guyana", "Haiti", "Holy Roman Empire", "Honduras", "Hungary", "Iceland", "India", "Indonesia", "Iran", "Iraq", "Republic of Ireland", "Israel", "Italy", "Ivory Coast", "Jamaica", "Japan", "Jonathanland", "Jordan", "Kazakhstan", "Kenya", "Kiribati", "North Korea", "South Korea", "Kosovo", "Kuwait", "Kyrgyzstan", "Laos", "Latvia", "Lebanon", "Lesotho", "Liberia", "Libya", "Liechtenstein", "Lithuania", "Luxembourg", "Macedonia", "Madagascar", "Malawi", "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Mauritania", "Mauritius", "Mexico", "Micronesia", "Moldova", "Monaco", "Mongolia", "Montenegro", "Morocco", "Mount Athos", "Mozambique", "Namibia", "Nauru", "Nepal", "Newfoundland", "Netherlands", "New Zealand", "Nicaragua", "Niger", "Nigeria", "Norway", "Oman", "Ottoman Empire", "Pakistan", "Palau", "Panama", "Papua New Guinea", "Paraguay", "Peru", "Philippines", "Poland", "Portugal", "Prussia", "Qatar", "Romania", "Rome", "Russian Federation", "Rwanda", "St Kitts & Nevis", "St Lucia", "Saint Vincent & the", "Grenadines", "Samoa", "San Marino", "Sao Tome & Principe", "Saudi Arabia", "Senegal", "Serbia", "Seychelles", "Sierra Leone", "Singapore", "Slovakia", "Slovenia", "Solomon Islands", "Somalia", "South Africa", "Spain", "Sri Lanka", "Sudan", "Suriname", "Swaziland", "Sweden", "Switzerland", "Syria", "Tajikistan", "Tanzania", "Thailand", "Togo", "Tonga", "Trinidad & Tobago", "Tunisia", "Turkey", "Turkmenistan", "Tuvalu", "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom", "Uruguay", "Uzbekistan", "Vanuatu", "Vatican City", "Venezuela", "Vietnam", "Yemen", "Zambia", "Zimbabwe"))
+object Grammar extends TermsParsers {
+  val fuzzyCountries = "my favourite country is " ~ oneOfTermsFuzzy(Seq("United States of America", "Afghanistan", "Albania", "Algeria", "Andorra", "Angola", "Antigua & Deps", "Argentina", "Armenia", "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain", "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize", "Benin", "Bhutan", "Bolivia", "Bosnia Herzegovina", "Botswana", "Brazil", "Brunei", "Bulgaria", "Burkina", "Burma", "Burundi", "Cambodia", "Cameroon", "Canada", "Cape Verde", "Central African Rep", "Chad", "Chile", "People's Republic of China", "Republic of China", "Colombia", "Comoros", "Democratic Republic of the Congo", "Republic of the Congo", "Costa Rica,", "Croatia", "Cuba", "Cyprus", "Czech Republic", "Danzig", "Denmark", "Djibouti", "Dominica", "Dominican Republic", "East Timor", "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Ethiopia", "Fiji", "Finland", "France", "Gabon", "Gaza Strip", "The Gambia", "Georgia", "Germany", "Ghana", "Greece", "Grenada", "Guatemala", "Guinea", "Guinea-Bissau", "Guyana", "Haiti", "Holy Roman Empire", "Honduras", "Hungary", "Iceland", "India", "Indonesia", "Iran", "Iraq", "Republic of Ireland", "Israel", "Italy", "Ivory Coast", "Jamaica", "Japan", "Jonathanland", "Jordan", "Kazakhstan", "Kenya", "Kiribati", "North Korea", "South Korea", "Kosovo", "Kuwait", "Kyrgyzstan", "Laos", "Latvia", "Lebanon", "Lesotho", "Liberia", "Libya", "Liechtenstein", "Lithuania", "Luxembourg", "Macedonia", "Madagascar", "Malawi", "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Mauritania", "Mauritius", "Mexico", "Micronesia", "Moldova", "Monaco", "Mongolia", "Montenegro", "Morocco", "Mount Athos", "Mozambique", "Namibia", "Nauru", "Nepal", "Newfoundland", "Netherlands", "New Zealand", "Nicaragua", "Niger", "Nigeria", "Norway", "Oman", "Ottoman Empire", "Pakistan", "Palau", "Panama", "Papua New Guinea", "Paraguay", "Peru", "Philippines", "Poland", "Portugal", "Prussia", "Qatar", "Romania", "Rome", "Russian Federation", "Rwanda", "St Kitts & Nevis", "St Lucia", "Saint Vincent & the", "Grenadines", "Samoa", "San Marino", "Sao Tome & Principe", "Saudi Arabia", "Senegal", "Serbia", "Seychelles", "Sierra Leone", "Singapore", "Slovakia", "Slovenia", "Solomon Islands", "Somalia", "South Africa", "Spain", "Sri Lanka", "Sudan", "Suriname", "Swaziland", "Sweden", "Switzerland", "Syria", "Tajikistan", "Tanzania", "Thailand", "Togo", "Tonga", "Trinidad & Tobago", "Tunisia", "Turkey", "Turkmenistan", "Tuvalu", "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom", "Uruguay", "Uzbekistan", "Vanuatu", "Vatican City", "Venezuela", "Vietnam", "Yemen", "Zambia", "Zimbabwe"))
 }
 ```
 
@@ -300,15 +300,15 @@ leads to:
 }
 ```
 
-### `oneOfTerms` parameters
+### `oneOfTermsFuzzy` parameters
 
-Below the signature of the `oneOfTerms` method:
+Below the signature of the `oneOfTermsFuzzy` method:
 
 ```scala
- def oneOfTerms(terms: Seq[String],
-                similarityMeasure: (String, String) => Double = diceSorensenSimilarity,
-                similarityThreshold: Int = DefaultSimilarityThreshold,
-                maxCompletionsCount: Int = DefaultMaxCompletionsCount)
+ def oneOfTermsFuzzy(terms: Seq[String],
+                     similarityMeasure: (String, String) => Double = diceSorensenSimilarity,
+                     similarityThreshold: Int = DefaultSimilarityThreshold,
+                     maxCompletionsCount: Int = DefaultMaxCompletionsCount)
 ```
 
  - `terms`: the list of terms to build the parser for

build.sbt

@@ -1,7 +1,7 @@
 name := "scala-parser-combinators-completion"
 organization := "com.nexthink"
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))
-version := "1.0.1"
+version := "1.0.2"
 scalaVersion := "2.12.2"
 bintrayRepository := "maven"
 bintrayVcsUrl := Some("jchapuis@github.com:jchapuis/scala-parser-combinators-completion")

src/main/scala/com/nexthink/utils/parsing/combinator/completion/CompletionExpansionSupport.scala

@@ -0,0 +1,104 @@
+/*
+ * scala-parser-combinators-completion
+ * Copyright (c) by Nexthink S.A.
+ * Lausanne, Switzerland (http://www.nexthink.com)
+ * Author: jonas.chapuis@nexthink.com
+ */
+
+package com.nexthink.utils.parsing.combinator.completion
+
+import scala.util.parsing.input.{CharSequenceReader, OffsetPosition, Position, Reader}
+
+trait CompletionExpansionSupport extends RegexCompletionSupport {
+
+  /**
+    * Adapts a parser so that completing it will list all possible expanded completions (which successfully parse)
+    * (note that if this is used within the context of a grammar allowing for infinitely growing expressions, this
+    * will trigger infinite recursion and will end up in a `StackOverflowException`)
+    * @param p the parser
+    * @param onlyAtInputEnd expansion happens only when input is positioned exactly at the end upon completion
+    * @tparam T the parser type
+    * @return a parser adapter performing completion expansion
+    */
+  def allExpandedCompletions[T](p: Parser[T], onlyAtInputEnd: Boolean = true): Parser[T] = expandedCompletions(p, p, onlyAtInputEnd)
+
+  /**
+    * Adapts a parser so that completing it will construct the list of all possible alternatives up to the point
+    * where the passed `stop` parser successfully parses the expansions.
+    * (note that if this is used within the context of a grammar allowing for infinitely growing expressions,
+    * selecting the relevant stop parser is critical to avoid infinite recursion)
+    * @param p the parser
+    * @param onlyAtInputEnd expansion happens only when input is positioned exactly at the end upon completion
+    * @param stop the parser signalling the end of exploration upon successful parse
+    * @tparam T the parser type
+    * @return a parser adapter performing completion expansion limited according to `stop` parser
+    */
+  def expandedCompletions[T](p: Parser[T], stop: Parser[Any], onlyAtInputEnd: Boolean = true): Parser[T] =
+    Parser(
+      p,
+      in => {
+        lazy val isAtInputEnd = dropAnyWhiteSpace(in).atEnd
+        if (!onlyAtInputEnd || isAtInputEnd) {
+          val Completions(_, sets) = exploreCompletions(p, stop, in)
+          Completions(OffsetPosition(in.source, handleWhiteSpace(in)), sets)
+        } else
+          p.completions(in)
+      }
+    )
+
+  private def exploreCompletions[T](p: Parser[T], stop: Parser[T], in: Input): Completions = {
+    def completeString(s: String, position: Int, c: Completion) = {
+      val input = s.substring(0, position - 1)
+      if (input.trim.isEmpty) c.value.toString() else s"$input ${c.value}"
+    }
+    def exploreCompletionsRec(str: String, completions: Completions): Completions = {
+      if (completions.isEmpty) completions
+      else
+        completions.allSets
+          .map(cSet => {
+            cSet.completions
+              .map(c => {
+                val completedInput = completeString(str, completions.position.column, c)
+                if (stop(new CharSequenceReader(completedInput)).successful) {
+                  Completions(in.pos, CompletionSet(cSet.tag, Set(Completion(completedInput, c.score, c.kind))))
+                } else {
+                  exploreCompletionsRec(completedInput, p.completions(ExplorerReader(p, completedInput)))
+                }
+              })
+              .reduce((a, b) => a | b)
+          })
+          .reduce((a, b) => a | b)
+    }
+    if (in match {
+          case ExplorerReader(exploredParser, _, _) if exploredParser == p => true
+          case _                                                           => false
+        }) {
+      // recursive parser => avoid infinite exploration
+      p.completions(in)
+    } else {
+      val inputAtPosition = if (in.atEnd) "" else in.rest.source.subSequence(in.offset, in.source.length()).toString.trim
+      exploreCompletionsRec(inputAtPosition, p.completions(ExplorerReader(p, inputAtPosition)))
+    }
+  }
+
+  private case class ExplorerReader(exploredParser: Parser[_], override val source: java.lang.CharSequence, override val offset: Int) extends Reader[Char] {
+    val charReader = new CharSequenceReader(source, offset)
+
+    def first = charReader.first
+
+    def rest: ExplorerReader =
+      if (offset < source.length) new ExplorerReader(exploredParser, source, offset + 1)
+      else this
+
+    def pos: Position = charReader.pos
+
+    def atEnd = charReader.atEnd
+
+    override def drop(n: Int): ExplorerReader = new ExplorerReader(exploredParser, source, offset + n)
+  }
+
+  private case object ExplorerReader {
+    def apply(exploredParser: Parser[_], source: java.lang.CharSequence): ExplorerReader = ExplorerReader(exploredParser, source, 0)
+  }
+
+}

src/main/scala/com/nexthink/utils/parsing/combinator/completion/CompletionSupport.scala

@@ -7,8 +7,10 @@
 
 package com.nexthink.utils.parsing.combinator.completion
 
+import java.io.StringReader
+
 import scala.util.parsing.combinator.Parsers
-import scala.util.parsing.input.Positional
+import scala.util.parsing.input.{CharSequenceReader, Positional}
 import scala.language.implicitConversions
 
 // scalastyle:off method.name
@@ -895,4 +897,6 @@ trait CompletionSupport extends Parsers with CompletionTypes {
     */
   def phrase[T](p: Parser[T]): Parser[T] =
     Parser(super.phrase(p), p.completions)
+
+
 }

src/main/scala/com/nexthink/utils/parsing/combinator/completion/RegexCompletionSupport.scala

@@ -9,7 +9,7 @@ package com.nexthink.utils.parsing.combinator.completion
 
 import scala.util.matching.Regex
 import scala.util.parsing.combinator.RegexParsers
-import scala.util.parsing.input.{CharSequenceReader, OffsetPosition, Positional, Reader}
+import scala.util.parsing.input._
 import scala.language.implicitConversions
 
 /** This component extends `RegexParsers` with completion capability. In particular,
@@ -22,7 +22,7 @@ import scala.language.implicitConversions
 trait RegexCompletionSupport extends RegexParsers with CompletionSupport {
   protected val areLiteralsCaseSensitive = false
 
-  protected def dropWhiteSpace(input: Input): Input =
+  protected def dropAnyWhiteSpace(input: Input): Input =
     input.drop(handleWhiteSpace(input.source, input.offset) - input.offset)
 
   protected def handleWhiteSpace(input: Input): Int =
@@ -36,8 +36,7 @@ trait RegexCompletionSupport extends RegexParsers with CompletionSupport {
     var sourcePos  = start
     def charsEqual(a: Char, b: Char) =
       if (areLiteralsCaseSensitive) a == b else a.toLower == b.toLower
-    while (literalPos < s.length && sourcePos < source.length && charsEqual(s.charAt(literalPos),
-                                                                            source.charAt(sourcePos))) {
+    while (literalPos < s.length && sourcePos < source.length && charsEqual(s.charAt(literalPos), source.charAt(sourcePos))) {
       literalPos += 1
       sourcePos += 1
     }
@@ -55,8 +54,7 @@ trait RegexCompletionSupport extends RegexParsers with CompletionSupport {
         literalOffset match {
           case 0 if inputAtEnd =>
             literalCompletion // whitespace, free entry possible
-          case someOffset: Int
-              if inputAtEnd & someOffset > 0 & someOffset < s.length => // partially entered literal, we are at the end
+          case someOffset: Int if inputAtEnd & someOffset > 0 & someOffset < s.length => // partially entered literal, we are at the end
             literalCompletion
           case _ => Completions.empty
         }
@@ -84,3 +82,5 @@ trait RegexCompletionSupport extends RegexParsers with CompletionSupport {
     complete(p, input).completionStrings
 
 }
+
+