zf

tokenizer.go

@@ -4,32 +4,12 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"strings"
 )
 
 const (
 	eof = rune(0)
 )
 
-type token struct {
-	kind rune
-	data string
-}
-
-func skipWhiteSpace(s io.RuneScanner) {
-	for {
-		r := read(s)
-		switch r {
-		case ' ', '\t', '\n':
-			continue
-		case eof:
-			return
-		}
-		_ = s.UnreadRune()
-		return
-	}
-}
-
 func read(s io.RuneScanner) rune {
 	r, _, err := s.ReadRune()
 	if err != nil {
@@ -75,91 +55,3 @@ func readRaw(s io.RuneScanner) *bytes.Buffer {
 
 	return &data
 }
-
-func readQuoted(s io.RuneScanner) (*token, error) {
-	var data bytes.Buffer
-
-loop:
-	for {
-		r := read(s)
-
-		switch r {
-		case eof:
-			return nil, fmt.Errorf("unexpected eof inside quoted string")
-		case '\\':
-			escaped, err := readEscaped(s)
-			if err != nil {
-				return nil, fmt.Errorf("incorrect escaping in quoted string: %v", err)
-			}
-			r = escaped
-		case '\'':
-			break loop
-		}
-
-		data.WriteRune(r)
-	}
-
-	return &token{'q', data.String()}, nil
-}
-
-func readNumberOrID(s io.RuneScanner) *token {
-	var data bytes.Buffer
-
-loop:
-	for {
-		r := read(s)
-
-		switch r {
-		case eof, ' ', '\t', '\n':
-			break loop
-		case '(', ')', ',':
-			_ = s.UnreadRune()
-			break loop
-		default:
-			data.WriteRune(r)
-		}
-	}
-
-	return &token{'s', data.String()}
-}
-
-func tokenize(s io.RuneScanner) ([]*token, error) {
-	var tokens []*token
-
-loop:
-	for {
-		var t *token
-		var err error
-
-		switch read(s) {
-		case eof:
-			break loop
-		case ' ', '\t', '\n':
-			skipWhiteSpace(s)
-			continue
-		case '(':
-			t = &token{kind: '('}
-		case ')':
-			t = &token{kind: ')'}
-		case ',':
-			t = &token{kind: ','}
-		case '\'':
-			t, err = readQuoted(s)
-			if err != nil {
-				return nil, err
-			}
-		default:
-			_ = s.UnreadRune()
-			t = readNumberOrID(s)
-		}
-
-		tokens = append(tokens, t)
-	}
-
-	tokens = append(tokens, &token{kind: eof})
-	return tokens, nil
-}
-
-func tokenizeString(s string) ([]*token, error) {
-	return tokenize(strings.NewReader(s))
-}