fix single line comment bug (#171)

* fix single line comment bug

* version 0.1.1

Author: Trinidadec

CHANGELOG.md

@@ -6,3 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## Unreleased
+
+## 0.1.1
+
+- fix single line comment bug PR [#171](https://github.com/functionalscript/nanvm/pull/171)

Cargo.toml

@@ -3,15 +3,15 @@ members = ["nanvm-lib", "nanvm"]
 resolver = "2"
 
 [workspace.package]
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 authors = ["Sergey Shandar", "Denys Shandar"]
 license = "GPL-3.0-or-later"
 repository = "https://github.com/functionalscript/nanvm"
 
 [workspace.dependencies]
-nanvm-lib = { path = "nanvm-lib", version = "0.1.0" }
-wasm-bindgen-test = "0.3.42"
+nanvm-lib = { path = "nanvm-lib", version = "0.1.1" }
+wasm-bindgen-test = "0.3.49"
 io-trait = "0.11.0"
 io-test = "0.11.0"
 io-impl = "0.11.0"

nanvm-lib/src/tokenizer/mod.rs

@@ -361,6 +361,7 @@ fn operator_to_token<D: Dealloc>(s: String) -> Option<JsonToken<D>> {
 }
 
 const WHITE_SPACE_CHARS: [char; 4] = [' ', '\n', '\t', '\r'];
+const NEW_LINE_CHARS: [char; 2] = ['\n', '\r'];
 const OPERATOR_CHARS: [char; 10] = ['{', '}', '[', ']', ':', ',', '=', ';', '(', ')'];
 
 fn id_start() -> Vec<RangeInclusive<char>> {
@@ -961,7 +962,7 @@ fn create_singleline_comment_transactions<M: Manager>() -> TransitionMap<(), M>
     type Func<M> = TransitionFunc<M, ()>;
     TransitionMap {
         def: (|_, _, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func<M>,
-        rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _, _| {
+        rm: create_range_map(set(NEW_LINE_CHARS), |_, _, _, _| {
             (default(), TokenizerState::ParseNewLine)
         }),
     }
@@ -1524,6 +1525,19 @@ mod test {
         let result = tokenize(GLOBAL, String::from("0//abc/*"));
         assert_eq!(&result, &[JsonToken::Number(0.0),]);
 
+        let result = tokenize(GLOBAL, String::from("0//abc 1"));
+        assert_eq!(&result, &[JsonToken::Number(0.0)]);
+
+        let result = tokenize(GLOBAL, String::from("0//abc\n1"));
+        assert_eq!(
+            &result,
+            &[
+                JsonToken::Number(0.0),
+                JsonToken::NewLine,
+                JsonToken::Number(1.0)
+            ]
+        );
+
         let result = tokenize(GLOBAL, String::from("0//"));
         assert_eq!(&result, &[JsonToken::Number(0.0),]);