From 126af886b4fa8af8be864a5c7a151f9024d1cd0e Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 2 Nov 2024 19:18:50 +0200 Subject: [PATCH 01/28] json token bigint --- nanvm-lib/src/js/js_bigint.rs | 3 +- nanvm-lib/src/mem/flexible_array/mod.rs | 1 + nanvm-lib/src/tokenizer/mod.rs | 119 +++++++++++------------- 3 files changed, 55 insertions(+), 68 deletions(-) diff --git a/nanvm-lib/src/js/js_bigint.rs b/nanvm-lib/src/js/js_bigint.rs index 9b1e1225..5203fbb9 100644 --- a/nanvm-lib/src/js/js_bigint.rs +++ b/nanvm-lib/src/js/js_bigint.rs @@ -21,6 +21,7 @@ pub enum Sign { Negative = -1, } +#[derive(Debug)] pub struct JsBigintHeader { len: isize, } @@ -449,7 +450,7 @@ fn twos_complement_zip<'a>( } impl JsBigint { - fn sign(&self) -> Sign { + pub fn sign(&self) -> Sign { if self.header.len < 0 { Sign::Negative } else { diff --git a/nanvm-lib/src/mem/flexible_array/mod.rs b/nanvm-lib/src/mem/flexible_array/mod.rs index b2fd0d1f..8468d4ce 100644 --- a/nanvm-lib/src/mem/flexible_array/mod.rs +++ b/nanvm-lib/src/mem/flexible_array/mod.rs @@ -12,6 +12,7 @@ use self::header::FlexibleArrayHeader; use super::{field_layout::FieldLayout, object::Object}; #[repr(transparent)] +#[derive(Debug)] pub struct FlexibleArray { pub header: T, _0: PhantomData, diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 10777a83..1e52f18e 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -1,17 +1,13 @@ -use std::{collections::VecDeque, mem::take, ops::RangeInclusive}; +use std::{collections::VecDeque, mem::take, ops::{Deref, RangeInclusive}}; use crate::{ big_numbers::{ - big_float::BigFloat, - big_int::{BigInt, Sign}, - big_uint::BigUint, - }, - common::{cast::Cast, default::default}, - range_map::{from_one, from_range, merge, merge_list, RangeMap, State}, + self, big_float::BigFloat, big_int::{BigInt, Sign}, big_uint::BigUint + }, common::{cast::Cast, default::default}, js::js_bigint::{self, add, from_u64, JsBigintMutRef, JsBigintRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State} }; -#[derive(Debug, PartialEq)] -pub enum JsonToken { +#[derive(Debug)] +pub enum JsonToken { String(String), Number(f64), ObjectBegin, @@ -23,7 +19,7 @@ pub enum JsonToken { Equals, Dot, ErrorToken(ErrorType), - BigInt(BigInt), + BigInt(JsBigintMutRef), Id(String), NewLine, Semicolon, @@ -42,7 +38,7 @@ pub enum ErrorType { } #[derive(Default)] -pub enum TokenizerState { +pub enum TokenizerState { #[default] Initial, ParseId(String), @@ -51,13 +47,13 @@ pub enum TokenizerState { ParseUnicodeChar(ParseUnicodeCharState), ParseMinus, ParseZero(Sign), - ParseInt(IntegerState), - ParseFracBegin(IntegerState), - ParseFrac(FloatState), - ParseExpBegin(ExpState), - ParseExpSign(ExpState), - ParseExp(ExpState), - ParseBigInt(IntegerState), + ParseInt(JsBigintMutRef), + ParseFracBegin(JsBigintMutRef), + ParseFrac(FloatState), + ParseExpBegin(ExpState), + ParseExpSign(ExpState), + ParseExp(ExpState), + ParseBigInt(JsBigintMutRef), ParseNewLine, ParseCommentStart, ParseSinglelineComment, @@ -66,8 +62,8 @@ pub enum TokenizerState { ParseOperator(String), } -impl TokenizerState { - fn push(self, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState) { +impl TokenizerState { + fn push(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { match self { TokenizerState::Initial => get_next_state((), c, &maps.initial, maps), TokenizerState::ParseId(s) => get_next_state(s, c, &maps.id, maps), @@ -99,13 +95,13 @@ impl TokenizerState { } } - pub fn push_mut(&mut self, c: char, tm: &TransitionMaps) -> Vec { + pub fn push_mut(&mut self, c: char, tm: &TransitionMaps) -> Vec> { let tokens; (tokens, *self) = take(self).push(c, tm); tokens } - fn end(self) -> Vec { + fn end(self) -> Vec> { match self { TokenizerState::Initial | TokenizerState::ParseNewLine @@ -146,7 +142,7 @@ pub struct ParseUnicodeCharState { } impl ParseUnicodeCharState { - fn push(mut self, i: u32) -> (Vec, TokenizerState) { + fn push(mut self, i: u32) -> (Vec>, TokenizerState) { let new_unicode = self.unicode | (i << ((3 - self.index) * 4)); match self.index { 3 => { @@ -169,11 +165,6 @@ impl ParseUnicodeCharState { } } -pub struct IntegerState { - s: Sign, - b: BigUint, -} - pub fn bigfloat_to_f64(bf_10: BigFloat<10>) -> f64 { let bf_2 = bf_10.to_bin(54); bf_2.to_f64() @@ -186,72 +177,67 @@ impl BigUint { } } -impl IntegerState { - fn from_difit(sign: Sign, c: char) -> IntegerState { - IntegerState { - s: sign, - b: BigUint::from_u64(digit_to_number(c)), - } +impl JsBigintMutRef { + fn from_digit(m: M, sign: js_bigint::Sign, c: char) -> JsBigintMutRef { + from_u64(m, sign, digit_to_number(c)) } - fn add_digit(mut self, c: char) -> IntegerState { - self.b = self.b.add_digit(c); - self + fn add_digit(self, m: M, c: char) -> JsBigintMutRef { + add(m, self.deref(), Self::from_digit(m, js_bigint::Sign::Positive, c).deref()) } - fn into_float_state(self) -> FloatState { + fn into_float_state(self) -> FloatState { FloatState { - s: self.s, - b: self.b, + b: self, fe: 0, } } - fn into_exp_state(self) -> ExpState { + fn into_exp_state(self) -> ExpState { ExpState { - s: self.s, - b: self.b, + b: self, fe: 0, es: Sign::Positive, e: 0, } } - fn into_token(self) -> JsonToken { + fn to_old_bigint(self) -> BigInt { + let deref = self.deref(); + let sign = match deref.sign() { + js_bigint::Sign::Positive => big_numbers::big_int::Sign::Positive, + js_bigint::Sign::Negative => big_numbers::big_int::Sign::Negative, + }; + BigInt { sign, value: BigUint { value: deref.items().to_vec() }} + } + + fn into_token(self) -> JsonToken { JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: BigInt { - sign: self.s, - value: self.b, - }, + significand: self.to_old_bigint(), exp: 0, non_zero_reminder: false, })) } - fn into_big_int_token(self) -> JsonToken { - JsonToken::BigInt(BigInt { - sign: self.s, - value: self.b, - }) + fn into_big_int_token(self) -> JsonToken { + JsonToken::BigInt(self) } } -pub struct FloatState { - s: Sign, - b: BigUint, +pub struct FloatState { + b: JsBigintMutRef, fe: i64, } -impl FloatState { - fn add_digit(mut self, c: char) -> FloatState { - self.b = self.b.add_digit(c); +impl FloatState { + fn add_digit(mut self, m: M, c: char) -> FloatState { + self.b = self.b.add_digit(m, c); self.fe -= 1; self } - fn into_exp_state(self) -> ExpState { + fn into_exp_state(self) -> ExpState { ExpState { - s: self.s, b: self.b, fe: self.fe, es: Sign::Positive, @@ -259,7 +245,7 @@ impl FloatState { } } - fn into_token(self) -> JsonToken { + fn into_token(self) -> JsonToken { JsonToken::Number(bigfloat_to_f64(BigFloat { significand: BigInt { sign: self.s, @@ -271,9 +257,8 @@ impl FloatState { } } -pub struct ExpState { - s: Sign, - b: BigUint, +pub struct ExpState { + b: JsBigintMutRef, fe: i64, es: Sign, e: i64, @@ -285,7 +270,7 @@ impl ExpState { self } - fn into_token(self) -> JsonToken { + fn into_token(self) -> JsonToken { let exp = self.fe + match self.es { Sign::Positive => self.e, @@ -304,7 +289,7 @@ impl ExpState { const CP_0: u32 = 0x30; -fn operator_to_token(s: String) -> Option { +fn operator_to_token(s: String) -> Option> { match s.as_str() { "{" => Some(JsonToken::ObjectBegin), "}" => Some(JsonToken::ObjectEnd), From 2388432ca84075fc40ff1c434521b1c85da8c30f Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 2 Nov 2024 20:37:30 +0200 Subject: [PATCH 02/28] fix errors --- nanvm-lib/src/parser/analyzer.rs | 2 +- nanvm-lib/src/parser/any_state.rs | 28 ++++++++++++++-------------- nanvm-lib/src/parser/const_state.rs | 2 +- nanvm-lib/src/parser/json_state.rs | 2 +- nanvm-lib/src/parser/parser.rs | 20 +++++--------------- nanvm-lib/src/parser/root_state.rs | 2 +- nanvm-lib/src/parser/shared.rs | 2 +- nanvm-lib/src/tokenizer/mod.rs | 6 +++--- 8 files changed, 27 insertions(+), 37 deletions(-) diff --git a/nanvm-lib/src/parser/analyzer.rs b/nanvm-lib/src/parser/analyzer.rs index 5af5a26c..2779f62b 100644 --- a/nanvm-lib/src/parser/analyzer.rs +++ b/nanvm-lib/src/parser/analyzer.rs @@ -66,5 +66,5 @@ impl AnalyzerState { } } - fn process_token(&mut self, _token: JsonToken) {} + fn process_token(&mut self, _token: JsonToken) {} } diff --git a/nanvm-lib/src/parser/any_state.rs b/nanvm-lib/src/parser/any_state.rs index f523953e..3c8ab64d 100644 --- a/nanvm-lib/src/parser/any_state.rs +++ b/nanvm-lib/src/parser/any_state.rs @@ -9,7 +9,7 @@ use super::{ use crate::{ common::{cast::Cast, default::default}, js::{any::Any, js_array::new_array, js_object::new_object}, - mem::manager::Manager, + mem::manager::{Dealloc, Manager}, tokenizer::JsonToken, }; use std::collections::BTreeMap; @@ -71,7 +71,7 @@ impl AnyState { pub fn parse( self, manager: M, - token: JsonToken, + token: JsonToken, module_cache: &mut ModuleCache, context_path: String, ) -> ( @@ -100,7 +100,7 @@ impl AnyState { pub fn parse_for_module( self, manager: M, - token: JsonToken, + token: JsonToken, module_cache: &mut ModuleCache, context_path: String, ) -> ( @@ -130,7 +130,7 @@ impl AnyState { } } - pub fn parse_import_begin(self, token: JsonToken) -> AnyResult { + pub fn parse_import_begin(self, token: JsonToken) -> AnyResult { match token { JsonToken::OpeningParenthesis => AnyResult::Continue(AnyState { status: ParsingStatus::ImportValue, @@ -140,7 +140,7 @@ impl AnyState { } } - pub fn parse_import_end(self, token: JsonToken) -> AnyResult { + pub fn parse_import_end(self, token: JsonToken) -> AnyResult { match token { JsonToken::ClosingParenthesis => self.end_import(), _ => AnyResult::Error(ParseError::WrongRequireStatement), @@ -149,7 +149,7 @@ impl AnyState { fn parse_import_value( self, - token: JsonToken, + token: JsonToken, module_cache: &mut ModuleCache, context_path: String, ) -> ( @@ -215,7 +215,7 @@ impl AnyState { } } - pub fn parse_value(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_value(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::ArrayBegin => self.begin_array(), JsonToken::ObjectBegin => self.begin_object(), @@ -284,7 +284,7 @@ impl AnyState { } } - pub fn parse_array_comma(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_array_comma(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::ArrayBegin => self.begin_array(), JsonToken::ObjectBegin => self.begin_object(), @@ -302,7 +302,7 @@ impl AnyState { } } - pub fn parse_array_begin(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_array_begin(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::ArrayBegin => self.begin_array(), JsonToken::ArrayEnd => self.end_array(manager), @@ -317,7 +317,7 @@ impl AnyState { } } - pub fn parse_array_value(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_array_value(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::ArrayEnd => self.end_array(manager), JsonToken::Comma => AnyResult::Continue(AnyState { @@ -355,7 +355,7 @@ impl AnyState { } } - pub fn parse_object_begin(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_object_begin(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::String(s) => self.push_key(s), JsonToken::Id(s) if self.data_type.is_djs() => self.push_key(s), @@ -364,7 +364,7 @@ impl AnyState { } } - pub fn parse_object_next(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_object_next(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::ObjectEnd => self.end_object(manager), JsonToken::Comma => AnyResult::Continue(AnyState { @@ -375,7 +375,7 @@ impl AnyState { } } - pub fn parse_object_comma(self, manager: M, token: JsonToken) -> AnyResult { + pub fn parse_object_comma(self, manager: M, token: JsonToken) -> AnyResult { match token { JsonToken::String(s) => self.push_key(s), JsonToken::ObjectEnd => self.end_object(manager), @@ -383,7 +383,7 @@ impl AnyState { } } - pub fn parse_object_key(self, token: JsonToken) -> AnyResult { + pub fn parse_object_key(self, token: JsonToken) -> AnyResult { match token { JsonToken::Colon => AnyResult::Continue(AnyState { status: ParsingStatus::ObjectColon, diff --git a/nanvm-lib/src/parser/const_state.rs b/nanvm-lib/src/parser/const_state.rs index b57a1b5d..bdeac4b7 100644 --- a/nanvm-lib/src/parser/const_state.rs +++ b/nanvm-lib/src/parser/const_state.rs @@ -15,7 +15,7 @@ impl ConstState { pub fn parse( self, manager: M, - token: JsonToken, + token: JsonToken, module_cache: &mut ModuleCache, context_path: String, ) -> JsonState { diff --git a/nanvm-lib/src/parser/json_state.rs b/nanvm-lib/src/parser/json_state.rs index 04f4c4de..a0134802 100644 --- a/nanvm-lib/src/parser/json_state.rs +++ b/nanvm-lib/src/parser/json_state.rs @@ -18,7 +18,7 @@ impl JsonState { pub fn push( self, manager: M, - token: JsonToken, + token: JsonToken, module_cache: &mut ModuleCache, context_path: String, ) -> ( diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index 8bbcb7f2..f330ea65 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -369,8 +369,7 @@ mod test { }, common::default::default, js::{ - js_array::JsArrayRef, js_bigint::JsBigintRef, js_object::JsObjectRef, - js_string::JsStringRef, type_::Type, + self, js_array::JsArrayRef, js_bigint::{from_u64, new_bigint, JsBigintRef}, js_object::JsObjectRef, js_string::JsStringRef, type_::Type }, mem::{global::GLOBAL, local::Local, manager::Manager}, tokenizer::{tokenize, ErrorType, JsonToken}, @@ -394,7 +393,7 @@ mod test { fn parse_with_virtual_io( manager: M, - iter: impl Iterator, + iter: impl Iterator>, ) -> Result, ParseError> { parse_with_tokens( &mut create_test_context(manager, &virtual_io(), &mut default()), @@ -937,12 +936,7 @@ mod test { let items = result.items(); assert_eq!(items, [0x61, 0x62, 0x63]); - let tokens = [JsonToken::BigInt(BigInt { - sign: Sign::Positive, - value: BigUint { - value: [1].to_vec(), - }, - })]; + let tokens = [JsonToken::BigInt(from_u64(manager, js::js_bigint::Sign::Positive, 1))]; let result = parse_with_virtual_io(manager, tokens.into_iter()); assert!(result.is_ok()); let result = result.unwrap().any.try_move::>(); @@ -951,13 +945,9 @@ mod test { assert_eq!(result.header_len(), 1); let items = result.items(); assert_eq!(items, [0x1]); + - let tokens = [JsonToken::BigInt(BigInt { - sign: Sign::Negative, - value: BigUint { - value: [2, 3].to_vec(), - }, - })]; + let tokens = [JsonToken::BigInt(new_bigint(manager, js::js_bigint::Sign::Negative, [2, 3]))]; let result = parse_with_virtual_io(manager, tokens.into_iter()); assert!(result.is_ok()); let result = result.unwrap().any.try_move::>(); diff --git a/nanvm-lib/src/parser/root_state.rs b/nanvm-lib/src/parser/root_state.rs index 639cdf8c..03526cf0 100644 --- a/nanvm-lib/src/parser/root_state.rs +++ b/nanvm-lib/src/parser/root_state.rs @@ -31,7 +31,7 @@ impl RootState { pub fn parse( self, manager: M, - token: JsonToken, + token: JsonToken, module_cache: &mut ModuleCache, context_path: String, ) -> ( diff --git a/nanvm-lib/src/parser/shared.rs b/nanvm-lib/src/parser/shared.rs index 6546d2be..d49f787b 100644 --- a/nanvm-lib/src/parser/shared.rs +++ b/nanvm-lib/src/parser/shared.rs @@ -152,7 +152,7 @@ fn try_id_to_any( } } -impl JsonToken { +impl JsonToken { pub fn try_to_any( self, manager: M, diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 1e52f18e..2bccd1fc 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -264,13 +264,13 @@ pub struct ExpState { e: i64, } -impl ExpState { - const fn add_digit(mut self, c: char) -> ExpState { +impl ExpState { + const fn add_digit(mut self, c: char) -> ExpState { self.e = self.e * 10 + digit_to_number(c) as i64; self } - fn into_token(self) -> JsonToken { + fn into_token(self) -> JsonToken { let exp = self.fe + match self.es { Sign::Positive => self.e, From 121adb7a96f3e734af469011956a21fc0a2c7786 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Tue, 5 Nov 2024 20:14:56 +0200 Subject: [PATCH 03/28] generic jsontoken --- nanvm-lib/src/parser/parser.rs | 26 +++--- nanvm-lib/src/tokenizer/mod.rs | 139 ++++++++++++++++----------------- 2 files changed, 81 insertions(+), 84 deletions(-) diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index f330ea65..19f9f10a 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -197,7 +197,7 @@ fn root_state_parse( fn const_state_parse( const_state: ConstState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> JsonState { match token { JsonToken::Semicolon => todo!(), @@ -225,7 +225,7 @@ fn const_state_parse( fn any_state_parse_for_module( any_state: AnyState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> JsonState { let result = any_state_parse(any_state, context, token); match result { @@ -241,7 +241,7 @@ fn any_state_parse_for_module( fn any_state_parse_import_value( any_state: AnyState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> AnyResult { match token { JsonToken::String(s) => { @@ -288,7 +288,7 @@ fn any_state_parse_import_value( fn any_state_parse( any_state: AnyState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> AnyResult { match any_state.status { ParsingStatus::Initial | ParsingStatus::ObjectColon => { @@ -310,13 +310,17 @@ fn any_state_parse( fn json_state_push( json_state: JsonState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> JsonState { - if token == JsonToken::NewLine { - return match json_state { - JsonState::ParseRoot(state) => root_state_parse(state, context, token), - _ => json_state, - }; + match token { + JsonToken::NewLine => + { + return match json_state { + JsonState::ParseRoot(state) => root_state_parse(state, context, token), + _ => json_state, + } + }, + _ => {} } match json_state { JsonState::ParseRoot(state) => root_state_parse(state, context, token), @@ -343,7 +347,7 @@ pub fn parse( pub fn parse_with_tokens( context: &mut Context, - iter: impl Iterator, + iter: impl Iterator>, ) -> Result, ParseError> { let mut state = JsonState::ParseRoot(RootState { status: RootStatus::Initial, diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 2bccd1fc..7425cfa9 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -3,7 +3,7 @@ use std::{collections::VecDeque, mem::take, ops::{Deref, RangeInclusive}}; use crate::{ big_numbers::{ self, big_float::BigFloat, big_int::{BigInt, Sign}, big_uint::BigUint - }, common::{cast::Cast, default::default}, js::js_bigint::{self, add, from_u64, JsBigintMutRef, JsBigintRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State} + }, common::{cast::Cast, default::default}, js::js_bigint::{self, add, from_u64, zero, JsBigintMutRef, JsBigintRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State} }; #[derive(Debug)] @@ -361,37 +361,37 @@ fn set(arr: impl IntoIterator) -> Vec> { result } -type Transition = - fn(state: T, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +type Transition = + fn(state: T, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); -struct TransitionMap { - def: Transition, - rm: RangeMap>>, +struct TransitionMap { + def: Transition, + rm: RangeMap>>, } -pub struct TransitionMaps { - initial: TransitionMap<()>, - id: TransitionMap, - string: TransitionMap, - escape_char: TransitionMap, - unicode_char: TransitionMap, - zero: TransitionMap, - int: TransitionMap, - minus: TransitionMap<()>, - frac_begin: TransitionMap, - frac: TransitionMap, - exp_begin: TransitionMap, - exp: TransitionMap, - big_int: TransitionMap, - new_line: TransitionMap<()>, - comment_start: TransitionMap<()>, - singleline_comment: TransitionMap<()>, - multiline_comment: TransitionMap<()>, - multiline_comment_asterix: TransitionMap<()>, - operator: TransitionMap, +pub struct TransitionMaps { + initial: TransitionMap<(), D>, + id: TransitionMap, + string: TransitionMap, + escape_char: TransitionMap, + unicode_char: TransitionMap, + zero: TransitionMap, + int: TransitionMap, D>, + minus: TransitionMap<(), D>, + frac_begin: TransitionMap, D>, + frac: TransitionMap, D>, + exp_begin: TransitionMap, D>, + exp: TransitionMap, D>, + big_int: TransitionMap, D>, + new_line: TransitionMap<(), D>, + comment_start: TransitionMap<(), D>, + singleline_comment: TransitionMap<(), D>, + multiline_comment: TransitionMap<(), D>, + multiline_comment_asterix: TransitionMap<(), D>, + operator: TransitionMap, } -pub fn create_transition_maps() -> TransitionMaps { +pub fn create_transition_maps() -> TransitionMaps { TransitionMaps { initial: create_initial_transitions(), id: create_id_transitions(), @@ -415,12 +415,12 @@ pub fn create_transition_maps() -> TransitionMaps { } } -fn get_next_state( +fn get_next_state( state: T, c: char, tm: &TransitionMap, maps: &TransitionMaps, -) -> (Vec, TokenizerState) +) -> (Vec>, TokenizerState) where T: 'static, { @@ -503,24 +503,24 @@ fn create_string_transactions() -> TransitionMap { } } -fn continue_string_state(mut s: String, c: char) -> (Vec, TokenizerState) { +fn continue_string_state(mut s: String, c: char) -> (Vec>, TokenizerState) { s.push(c); (default(), TokenizerState::ParseString(s)) } -fn transfer_state( - mut vec: Vec, - mut state: TokenizerState, +fn transfer_state( + mut vec: Vec>, + mut state: TokenizerState, c: char, - maps: &TransitionMaps, -) -> (Vec, TokenizerState) { + maps: &TransitionMaps, +) -> (Vec>, TokenizerState) { let next_tokens; (next_tokens, state) = state.push(c, maps); vec.extend(next_tokens); (vec, state) } -fn create_escape_char_transactions() -> TransitionMap { +fn create_escape_char_transactions() -> TransitionMap { TransitionMap { def: |s, c, maps| { transfer_state( @@ -556,12 +556,12 @@ fn create_escape_char_transactions() -> TransitionMap { } } -fn create_unicode_char_transactions() -> TransitionMap { +fn create_unicode_char_transactions() -> TransitionMap { type Func = fn( state: ParseUnicodeCharState, c: char, - maps: &TransitionMaps, - ) -> (Vec, TokenizerState); + maps: &TransitionMaps, + ) -> (Vec>, TokenizerState); TransitionMap { def: |state, c, maps| { transfer_state( @@ -591,7 +591,7 @@ fn create_unicode_char_transactions() -> TransitionMap { } } -fn create_zero_transactions() -> TransitionMap { +fn create_zero_transactions(m: M) -> TransitionMap { type Func = fn(s: Sign, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, @@ -602,10 +602,7 @@ fn create_zero_transactions() -> TransitionMap { (|s, _, _| { ( default(), - TokenizerState::ParseFracBegin(IntegerState { - s, - b: BigUint::ZERO, - }), + TokenizerState::ParseFracBegin(zero(m)), ) }) as Func, ), @@ -613,8 +610,7 @@ fn create_zero_transactions() -> TransitionMap { ( default(), TokenizerState::ParseExpBegin(ExpState { - s, - b: BigUint::ZERO, + b: zero(m), fe: 0, es: Sign::Positive, e: 0, @@ -624,10 +620,7 @@ fn create_zero_transactions() -> TransitionMap { from_one('n', |s, _, _| { ( default(), - TokenizerState::ParseBigInt(IntegerState { - s, - b: BigUint::ZERO, - }), + TokenizerState::ParseBigInt(zero(m)), ) }), create_range_map(terminal_for_number(), |_, c, maps| { @@ -644,9 +637,9 @@ fn create_zero_transactions() -> TransitionMap { } } -fn create_int_transactions() -> TransitionMap { +fn create_int_transactions() -> TransitionMap, D> { type Func = - fn(s: IntegerState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( @@ -671,7 +664,7 @@ fn create_int_transactions() -> TransitionMap { } } -fn create_frac_begin_transactions() -> TransitionMap { +fn create_frac_begin_transactions() -> TransitionMap, D> { type Func = fn(s: IntegerState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { @@ -688,7 +681,7 @@ fn create_frac_begin_transactions() -> TransitionMap { } } -fn create_frac_transactions() -> TransitionMap { +fn create_frac_transactions() -> TransitionMap, D> { type Func = fn(s: FloatState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { @@ -711,7 +704,7 @@ fn create_frac_transactions() -> TransitionMap { } } -fn create_minus_transactions() -> TransitionMap<()> { +fn create_minus_transactions() -> TransitionMap<(), D> { type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, @@ -730,7 +723,7 @@ fn create_minus_transactions() -> TransitionMap<()> { } } -fn create_exp_begin_transactions() -> TransitionMap { +fn create_exp_begin_transactions() -> TransitionMap, D> { type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, @@ -756,7 +749,7 @@ fn create_exp_begin_transactions() -> TransitionMap { } } -fn create_exp_transactions() -> TransitionMap { +fn create_exp_transactions() -> TransitionMap, D> { type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, @@ -772,7 +765,7 @@ fn create_exp_transactions() -> TransitionMap { } } -fn create_big_int_transactions() -> TransitionMap { +fn create_big_int_transactions() -> TransitionMap, D> { type Func = fn(s: IntegerState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { @@ -783,7 +776,7 @@ fn create_big_int_transactions() -> TransitionMap { } } -fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Vec, TokenizerState) { +fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { transfer_state( [JsonToken::ErrorToken(ErrorType::InvalidNumber)].cast(), TokenizerState::Initial, @@ -792,7 +785,7 @@ fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Vec, T ) } -fn create_new_line_transactions() -> TransitionMap<()> { +fn create_new_line_transactions() -> TransitionMap<(), D> { type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, c, maps| { @@ -809,7 +802,7 @@ fn create_new_line_transactions() -> TransitionMap<()> { } } -fn create_comment_start_transactions() -> TransitionMap<()> { +fn create_comment_start_transactions() -> TransitionMap<(), D> { type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, _, _| { @@ -829,7 +822,7 @@ fn create_comment_start_transactions() -> TransitionMap<()> { } } -fn create_singleline_comment_transactions() -> TransitionMap<()> { +fn create_singleline_comment_transactions() -> TransitionMap<(), D> { type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func, @@ -839,7 +832,7 @@ fn create_singleline_comment_transactions() -> TransitionMap<()> { } } -fn create_multiline_comment_transactions() -> TransitionMap<()> { +fn create_multiline_comment_transactions() -> TransitionMap<(), D> { type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, @@ -849,7 +842,7 @@ fn create_multiline_comment_transactions() -> TransitionMap<()> { } } -fn create_multiline_comment_asterix_transactions() -> TransitionMap<()> { +fn create_multiline_comment_asterix_transactions() -> TransitionMap<(), D> { type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, @@ -862,7 +855,7 @@ fn create_multiline_comment_asterix_transactions() -> TransitionMap<()> { } } -fn create_operator_transactions() -> TransitionMap { +fn create_operator_transactions() -> TransitionMap { TransitionMap { def: |s, c, maps| { let token = operator_to_token(s).unwrap(); @@ -886,19 +879,19 @@ fn create_operator_transactions() -> TransitionMap { } } -pub fn tokenize(input: String) -> Vec { +pub fn tokenize(input: String) -> Vec> { TokenizerStateIterator::new(input.chars()).collect() } -pub struct TokenizerStateIterator> { +pub struct TokenizerStateIterator, D: Dealloc> { chars: T, - cache: VecDeque, - state: TokenizerState, - maps: TransitionMaps, + cache: VecDeque>, + state: TokenizerState, + maps: TransitionMaps, end: bool, } -impl> TokenizerStateIterator { +impl, D: Dealloc> TokenizerStateIterator { pub fn new(chars: T) -> Self { Self { chars, @@ -910,8 +903,8 @@ impl> TokenizerStateIterator { } } -impl> Iterator for TokenizerStateIterator { - type Item = JsonToken; +impl, D: Dealloc> Iterator for TokenizerStateIterator { + type Item = JsonToken; fn next(&mut self) -> Option { loop { From e3ebf729f263f074b1b75cfbf8cfca638ae0e00c Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 9 Nov 2024 18:46:05 +0200 Subject: [PATCH 04/28] jsbigintmutref in parser --- nanvm-lib/src/parser/parser.rs | 1778 ++++++++++++++++---------------- nanvm-lib/src/tokenizer/mod.rs | 1164 ++++++++++----------- 2 files changed, 1473 insertions(+), 1469 deletions(-) diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index 19f9f10a..b8bb3931 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -360,894 +360,894 @@ pub fn parse_with_tokens( state.end() } -#[cfg(test)] -mod test { - use io_test::VirtualIo; - use io_trait::Io; - use wasm_bindgen_test::wasm_bindgen_test; - - use crate::{ - big_numbers::{ - big_int::{BigInt, Sign}, - big_uint::BigUint, - }, - common::default::default, - js::{ - self, js_array::JsArrayRef, js_bigint::{from_u64, new_bigint, JsBigintRef}, js_object::JsObjectRef, js_string::JsStringRef, type_::Type - }, - mem::{global::GLOBAL, local::Local, manager::Manager}, - tokenizer::{tokenize, ErrorType, JsonToken}, - }; - - use super::super::{parser::parse, path::concat, shared::DataType}; - - use super::{parse_with_tokens, Context, ModuleCache, ParseError, ParseResult}; - - fn virtual_io() -> VirtualIo { - VirtualIo::new(&[]) - } - - fn create_test_context<'a, M: Manager>( - manager: M, - io: &'a VirtualIo, - module_cache: &'a mut ModuleCache, - ) -> Context<'a, M, VirtualIo> { - Context::new(manager, io, default(), module_cache) - } - - fn parse_with_virtual_io( - manager: M, - iter: impl Iterator>, - ) -> Result, ParseError> { - parse_with_tokens( - &mut create_test_context(manager, &virtual_io(), &mut default()), - iter, - ) - } - - fn test_local() { - let local = Local::default(); - let _ = parse_with_tokens( - &mut create_test_context(&local, &virtual_io(), &mut default()), - [].into_iter(), - ); - } - - fn test_global() { - let _ = { - let global = GLOBAL; - parse_with_tokens( - &mut create_test_context(global, &virtual_io(), &mut default()), - [].into_iter(), - ) - }; - } - - #[test] - #[wasm_bindgen_test] - fn test_json() { - let json_str = include_str!("../../test/test-json.json"); - let tokens = tokenize(json_str.to_owned()); - let local = Local::default(); - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().data_type, DataType::Json); - } - - #[test] - #[wasm_bindgen_test] - fn test_djs() { - let json_str = include_str!("../../test/test-djs.d.cjs"); - let tokens = tokenize(json_str.to_owned()); - let local = Local::default(); - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().data_type, DataType::Cjs); - - let json_str = include_str!("../../test/test-djs.d.mjs"); - let tokens = tokenize(json_str.to_owned()); - let local = Local::default(); - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().data_type, DataType::Mjs); - } - - #[test] - #[wasm_bindgen_test] - fn test_const() { - let local = Local::default(); - test_const_with_manager(&local); - } - - fn test_const_with_manager(manager: M) { - let json_str = include_str!("../../test/test-const.d.cjs"); - let tokens = tokenize(json_str.to_owned()); - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.try_move(), Ok(2.0)); - let item1 = items[1].clone(); - assert_eq!(item1.try_move(), Ok(2.0)); - - let json_str = include_str!("../../test/test-const-error.d.cjs.txt"); - let tokens = tokenize(json_str.to_owned()); - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let json_str = include_str!("../../test/test-const-error-new-line.d.cjs.txt"); - let tokens = tokenize(json_str.to_owned()); - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::NewLineExpected); - } - - #[test] - #[wasm_bindgen_test] - fn test_stack() { - let local = Local::default(); - test_stack_with_manager(&local); - } - - fn test_stack_with_manager(manager: M) { - let json_str = include_str!("../../test/test-stack.d.cjs"); - let tokens = tokenize(json_str.to_owned()); - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - let result_unwrap = item0.try_move::>().unwrap(); - let items = result_unwrap.items(); - let (key0, value0) = items[0].clone(); - let key0_items = key0.items(); - assert_eq!(key0_items, [0x61]); - let result_unwrap = value0.try_move::>().unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.get_type(), Type::Null); - } - - #[test] - #[wasm_bindgen_test] - fn test_import() { - let local = Local::default(); - test_import_with_manager(&local); - } - - fn test_import_with_manager(manager: M) { - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_import_main.d.cjs"); - //let path = "../../test/test-import-main.d.cjs"; - let main_path = "test_import_main.d.cjs"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module = include_str!("../../test/test_import_module.d.cjs"); - let module_path = "test_import_module.d.cjs"; - io.write(module_path, module.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.try_move(), Ok(3.0)); - - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_import_main.d.mjs"); - //let path = "../../test/test-import-main.d.mjs"; - let main_path = "test_import_main.d.mjs"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module = include_str!("../../test/test_import_module.d.mjs"); - let module_path = "test_import_module.d.mjs"; - io.write(module_path, module.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.try_move(), Ok(4.0)); - } - - #[test] - #[wasm_bindgen_test] - fn test_cache() { - let local = Local::default(); - test_cache_with_manager(&local); - } - - fn test_cache_with_manager(manager: M) { - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_cache_main.d.cjs"); - let main_path = "test_cache_main.d.cjs"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module_b = include_str!("../../test/test_cache_b.d.cjs"); - let module_b_path = "test_cache_b.d.cjs"; - io.write(module_b_path, module_b.as_bytes()).unwrap(); - - let module_c = include_str!("../../test/test_cache_c.d.cjs"); - let module_c_path = "test_cache_c.d.cjs"; - io.write(module_c_path, module_c.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.try_move(), Ok(1.0)); - let item1 = items[1].clone(); - assert_eq!(item1.try_move(), Ok(1.0)); - - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_cache_main.d.mjs"); - let main_path = "test_cache_main.d.mjs"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module_b = include_str!("../../test/test_cache_b.d.mjs"); - let module_b_path = "test_cache_b.d.mjs"; - io.write(module_b_path, module_b.as_bytes()).unwrap(); - - let module_c = include_str!("../../test/test_cache_c.d.mjs"); - let module_c_path = "test_cache_c.d.mjs"; - io.write(module_c_path, module_c.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.try_move(), Ok(2.0)); - let item1 = items[1].clone(); - assert_eq!(item1.try_move(), Ok(2.0)); - } - - #[test] - #[wasm_bindgen_test] - fn test_circular_error() { - let local = Local::default(); - test_circular_error_with_manager(&local); - } - - fn test_circular_error_with_manager(manager: M) { - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_circular_1.d.cjs.txt"); - let main_path = "test_circular_1.d.cjs.txt"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module = include_str!("../../test/test_circular_2.d.cjs.txt"); - let module_path = "test_circular_2.d.cjs.txt"; - io.write(module_path, module.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::CircularDependency); - - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_circular_1.d.mjs.txt"); - let main_path = "test_circular_1.d.mjs.txt"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module = include_str!("../../test/test_circular_2.d.mjs.txt"); - let module_path = "test_circular_2.d.mjs.txt"; - io.write(module_path, module.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::CircularDependency); - } - - #[test] - #[wasm_bindgen_test] - fn test_import_error() { - let local = Local::default(); - test_import_error_with_manager(&local); - } - - fn test_import_error_with_manager(manager: M) { - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_import_error.d.cjs.txt"); - //let path = "../../test/test-import-main.d.cjs"; - let main_path = "test_import_error.d.cjs.txt"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module = include_str!("../../test/test_import_module.d.mjs"); - let module_path = "test_import_module.d.mjs"; - io.write(module_path, module.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); - - let io: VirtualIo = VirtualIo::new(&[]); - - let main = include_str!("../../test/test_import_error.d.mjs.txt"); - //let path = "../../test/test-import-main.d.cjs"; - let main_path = "test_import_error.d.mjs.txt"; - io.write(main_path, main.as_bytes()).unwrap(); - - let module = include_str!("../../test/test_import_module.d.cjs"); - let module_path = "test_import_module.d.cjs"; - io.write(module_path, module.as_bytes()).unwrap(); - - let mut mc = default(); - let mut context = Context::new( - manager, - &io, - concat(io.current_dir().unwrap().as_str(), main_path), - &mut mc, - ); - - let result = parse(&mut context); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); - } - - #[test] - #[wasm_bindgen_test] - fn test_trailing_comma() { - let local = Local::default(); - let json_str = include_str!("../../test/test-trailing-comma.d.cjs"); - let tokens = tokenize(json_str.to_owned()); - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - } - - #[test] - #[wasm_bindgen_test] - fn test_check_sizes() { - let local = Local::default(); - { - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("k")), - JsonToken::Colon, - JsonToken::ObjectBegin, - JsonToken::ObjectEnd, - JsonToken::ObjectEnd, - ]; - { - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - let _result_unwrap = result.unwrap(); - } - assert_eq!(local.size(), 0); - } - } - - #[test] - #[wasm_bindgen_test] - fn test_check_sizes2() { - let local = Local::default(); - { - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("k")), - JsonToken::Colon, - JsonToken::ObjectBegin, - JsonToken::ObjectEnd, - JsonToken::ObjectEnd, - ]; - { - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result.unwrap().any; - let _result_unwrap = result_unwrap.try_move::>(); - } - assert_eq!(local.size(), 0); - } - } - - #[test] - #[wasm_bindgen_test] - fn test_data_type() { - let local = Local::default(); - let tokens = [JsonToken::Id(String::from("null"))]; - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().data_type, DataType::Json); - } - - #[test] - #[wasm_bindgen_test] - fn test_export_block() { - let local = Local::default(); - let tokens = [ - JsonToken::Id(String::from("export")), - JsonToken::Id(String::from("default")), - JsonToken::Id(String::from("null")), - ]; - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().data_type, DataType::Mjs); - - let local = Local::default(); - let tokens = [ - JsonToken::Id(String::from("module")), - JsonToken::Dot, - JsonToken::Id(String::from("exports")), - JsonToken::Equals, - JsonToken::Id(String::from("null")), - ]; - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().data_type, DataType::Cjs); - } - - #[test] - #[wasm_bindgen_test] - fn test_id_in_objects() { - let local = Local::default(); - let tokens = [ - JsonToken::Id(String::from("export")), - JsonToken::Id(String::from("default")), - JsonToken::ObjectBegin, - JsonToken::Id(String::from("key")), - JsonToken::Colon, - JsonToken::Number(0.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result.unwrap().any.try_move::>().unwrap(); - let items = result_unwrap.items(); - let (key0, value0) = items[0].clone(); - let key0_items = key0.items(); - assert_eq!(key0_items, [0x6b, 0x65, 0x79]); - assert_eq!(value0.try_move(), Ok(0.0)); - - let local = Local::default(); - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::Id(String::from("key")), - JsonToken::Colon, - JsonToken::Number(0.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(&local, tokens.into_iter()); - assert!(result.is_err()); - } - - #[test] - #[wasm_bindgen_test] - fn test_valid_local() { - test_valid_with_manager(&Local::default()); - } - - #[test] - #[wasm_bindgen_test] - fn test_valid_global() { - test_valid_with_manager(GLOBAL); - } - - fn test_valid_with_manager(manager: M) { - let tokens = [JsonToken::Id(String::from("null"))]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().any.get_type(), Type::Null); - - let tokens = [JsonToken::Id(String::from("true"))]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().any.try_move(), Ok(true)); - - let tokens = [JsonToken::Id(String::from("false"))]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().any.try_move(), Ok(false)); - - let tokens = [JsonToken::Number(0.1)]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - assert_eq!(result.unwrap().any.try_move(), Ok(0.1)); - - let tokens = [JsonToken::String(String::from("abc"))]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result = result.unwrap().any.try_move::>(); - assert!(result.is_ok()); - let result = result.unwrap(); - let items = result.items(); - assert_eq!(items, [0x61, 0x62, 0x63]); - - let tokens = [JsonToken::BigInt(from_u64(manager, js::js_bigint::Sign::Positive, 1))]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result = result.unwrap().any.try_move::>(); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.header_len(), 1); - let items = result.items(); - assert_eq!(items, [0x1]); +// #[cfg(test)] +// mod test { +// use io_test::VirtualIo; +// use io_trait::Io; +// use wasm_bindgen_test::wasm_bindgen_test; + +// use crate::{ +// big_numbers::{ +// big_int::{BigInt, Sign}, +// big_uint::BigUint, +// }, +// common::default::default, +// js::{ +// self, js_array::JsArrayRef, js_bigint::{from_u64, new_bigint, JsBigintRef}, js_object::JsObjectRef, js_string::JsStringRef, type_::Type +// }, +// mem::{global::GLOBAL, local::Local, manager::Manager}, +// tokenizer::{tokenize, ErrorType, JsonToken}, +// }; + +// use super::super::{parser::parse, path::concat, shared::DataType}; + +// use super::{parse_with_tokens, Context, ModuleCache, ParseError, ParseResult}; + +// fn virtual_io() -> VirtualIo { +// VirtualIo::new(&[]) +// } + +// fn create_test_context<'a, M: Manager>( +// manager: M, +// io: &'a VirtualIo, +// module_cache: &'a mut ModuleCache, +// ) -> Context<'a, M, VirtualIo> { +// Context::new(manager, io, default(), module_cache) +// } + +// fn parse_with_virtual_io( +// manager: M, +// iter: impl Iterator>, +// ) -> Result, ParseError> { +// parse_with_tokens( +// &mut create_test_context(manager, &virtual_io(), &mut default()), +// iter, +// ) +// } + +// fn test_local() { +// let local = Local::default(); +// let _ = parse_with_tokens( +// &mut create_test_context(&local, &virtual_io(), &mut default()), +// [].into_iter(), +// ); +// } + +// fn test_global() { +// let _ = { +// let global = GLOBAL; +// parse_with_tokens( +// &mut create_test_context(global, &virtual_io(), &mut default()), +// [].into_iter(), +// ) +// }; +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_json() { +// let json_str = include_str!("../../test/test-json.json"); +// let tokens = tokenize(json_str.to_owned()); +// let local = Local::default(); +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().data_type, DataType::Json); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_djs() { +// let json_str = include_str!("../../test/test-djs.d.cjs"); +// let tokens = tokenize(json_str.to_owned()); +// let local = Local::default(); +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().data_type, DataType::Cjs); + +// let json_str = include_str!("../../test/test-djs.d.mjs"); +// let tokens = tokenize(json_str.to_owned()); +// let local = Local::default(); +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().data_type, DataType::Mjs); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_const() { +// let local = Local::default(); +// test_const_with_manager(&local); +// } + +// fn test_const_with_manager(manager: M) { +// let json_str = include_str!("../../test/test-const.d.cjs"); +// let tokens = tokenize(json_str.to_owned()); +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.try_move(), Ok(2.0)); +// let item1 = items[1].clone(); +// assert_eq!(item1.try_move(), Ok(2.0)); + +// let json_str = include_str!("../../test/test-const-error.d.cjs.txt"); +// let tokens = tokenize(json_str.to_owned()); +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let json_str = include_str!("../../test/test-const-error-new-line.d.cjs.txt"); +// let tokens = tokenize(json_str.to_owned()); +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); +// assert_eq!(result.unwrap_err(), ParseError::NewLineExpected); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_stack() { +// let local = Local::default(); +// test_stack_with_manager(&local); +// } + +// fn test_stack_with_manager(manager: M) { +// let json_str = include_str!("../../test/test-stack.d.cjs"); +// let tokens = tokenize(json_str.to_owned()); +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// let result_unwrap = item0.try_move::>().unwrap(); +// let items = result_unwrap.items(); +// let (key0, value0) = items[0].clone(); +// let key0_items = key0.items(); +// assert_eq!(key0_items, [0x61]); +// let result_unwrap = value0.try_move::>().unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.get_type(), Type::Null); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_import() { +// let local = Local::default(); +// test_import_with_manager(&local); +// } + +// fn test_import_with_manager(manager: M) { +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_import_main.d.cjs"); +// //let path = "../../test/test-import-main.d.cjs"; +// let main_path = "test_import_main.d.cjs"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module = include_str!("../../test/test_import_module.d.cjs"); +// let module_path = "test_import_module.d.cjs"; +// io.write(module_path, module.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.try_move(), Ok(3.0)); + +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_import_main.d.mjs"); +// //let path = "../../test/test-import-main.d.mjs"; +// let main_path = "test_import_main.d.mjs"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module = include_str!("../../test/test_import_module.d.mjs"); +// let module_path = "test_import_module.d.mjs"; +// io.write(module_path, module.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.try_move(), Ok(4.0)); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_cache() { +// let local = Local::default(); +// test_cache_with_manager(&local); +// } + +// fn test_cache_with_manager(manager: M) { +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_cache_main.d.cjs"); +// let main_path = "test_cache_main.d.cjs"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module_b = include_str!("../../test/test_cache_b.d.cjs"); +// let module_b_path = "test_cache_b.d.cjs"; +// io.write(module_b_path, module_b.as_bytes()).unwrap(); + +// let module_c = include_str!("../../test/test_cache_c.d.cjs"); +// let module_c_path = "test_cache_c.d.cjs"; +// io.write(module_c_path, module_c.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.try_move(), Ok(1.0)); +// let item1 = items[1].clone(); +// assert_eq!(item1.try_move(), Ok(1.0)); + +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_cache_main.d.mjs"); +// let main_path = "test_cache_main.d.mjs"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module_b = include_str!("../../test/test_cache_b.d.mjs"); +// let module_b_path = "test_cache_b.d.mjs"; +// io.write(module_b_path, module_b.as_bytes()).unwrap(); + +// let module_c = include_str!("../../test/test_cache_c.d.mjs"); +// let module_c_path = "test_cache_c.d.mjs"; +// io.write(module_c_path, module_c.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.try_move(), Ok(2.0)); +// let item1 = items[1].clone(); +// assert_eq!(item1.try_move(), Ok(2.0)); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_circular_error() { +// let local = Local::default(); +// test_circular_error_with_manager(&local); +// } + +// fn test_circular_error_with_manager(manager: M) { +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_circular_1.d.cjs.txt"); +// let main_path = "test_circular_1.d.cjs.txt"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module = include_str!("../../test/test_circular_2.d.cjs.txt"); +// let module_path = "test_circular_2.d.cjs.txt"; +// io.write(module_path, module.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_err()); +// assert_eq!(result.unwrap_err(), ParseError::CircularDependency); + +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_circular_1.d.mjs.txt"); +// let main_path = "test_circular_1.d.mjs.txt"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module = include_str!("../../test/test_circular_2.d.mjs.txt"); +// let module_path = "test_circular_2.d.mjs.txt"; +// io.write(module_path, module.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_err()); +// assert_eq!(result.unwrap_err(), ParseError::CircularDependency); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_import_error() { +// let local = Local::default(); +// test_import_error_with_manager(&local); +// } + +// fn test_import_error_with_manager(manager: M) { +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_import_error.d.cjs.txt"); +// //let path = "../../test/test-import-main.d.cjs"; +// let main_path = "test_import_error.d.cjs.txt"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module = include_str!("../../test/test_import_module.d.mjs"); +// let module_path = "test_import_module.d.mjs"; +// io.write(module_path, module.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_err()); +// assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); + +// let io: VirtualIo = VirtualIo::new(&[]); + +// let main = include_str!("../../test/test_import_error.d.mjs.txt"); +// //let path = "../../test/test-import-main.d.cjs"; +// let main_path = "test_import_error.d.mjs.txt"; +// io.write(main_path, main.as_bytes()).unwrap(); + +// let module = include_str!("../../test/test_import_module.d.cjs"); +// let module_path = "test_import_module.d.cjs"; +// io.write(module_path, module.as_bytes()).unwrap(); + +// let mut mc = default(); +// let mut context = Context::new( +// manager, +// &io, +// concat(io.current_dir().unwrap().as_str(), main_path), +// &mut mc, +// ); + +// let result = parse(&mut context); +// assert!(result.is_err()); +// assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_trailing_comma() { +// let local = Local::default(); +// let json_str = include_str!("../../test/test-trailing-comma.d.cjs"); +// let tokens = tokenize(json_str.to_owned()); +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_check_sizes() { +// let local = Local::default(); +// { +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("k")), +// JsonToken::Colon, +// JsonToken::ObjectBegin, +// JsonToken::ObjectEnd, +// JsonToken::ObjectEnd, +// ]; +// { +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// let _result_unwrap = result.unwrap(); +// } +// assert_eq!(local.size(), 0); +// } +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_check_sizes2() { +// let local = Local::default(); +// { +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("k")), +// JsonToken::Colon, +// JsonToken::ObjectBegin, +// JsonToken::ObjectEnd, +// JsonToken::ObjectEnd, +// ]; +// { +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result.unwrap().any; +// let _result_unwrap = result_unwrap.try_move::>(); +// } +// assert_eq!(local.size(), 0); +// } +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_data_type() { +// let local = Local::default(); +// let tokens = [JsonToken::Id(String::from("null"))]; +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().data_type, DataType::Json); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_export_block() { +// let local = Local::default(); +// let tokens = [ +// JsonToken::Id(String::from("export")), +// JsonToken::Id(String::from("default")), +// JsonToken::Id(String::from("null")), +// ]; +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().data_type, DataType::Mjs); + +// let local = Local::default(); +// let tokens = [ +// JsonToken::Id(String::from("module")), +// JsonToken::Dot, +// JsonToken::Id(String::from("exports")), +// JsonToken::Equals, +// JsonToken::Id(String::from("null")), +// ]; +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().data_type, DataType::Cjs); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_id_in_objects() { +// let local = Local::default(); +// let tokens = [ +// JsonToken::Id(String::from("export")), +// JsonToken::Id(String::from("default")), +// JsonToken::ObjectBegin, +// JsonToken::Id(String::from("key")), +// JsonToken::Colon, +// JsonToken::Number(0.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result.unwrap().any.try_move::>().unwrap(); +// let items = result_unwrap.items(); +// let (key0, value0) = items[0].clone(); +// let key0_items = key0.items(); +// assert_eq!(key0_items, [0x6b, 0x65, 0x79]); +// assert_eq!(value0.try_move(), Ok(0.0)); + +// let local = Local::default(); +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::Id(String::from("key")), +// JsonToken::Colon, +// JsonToken::Number(0.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(&local, tokens.into_iter()); +// assert!(result.is_err()); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_valid_local() { +// test_valid_with_manager(&Local::default()); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_valid_global() { +// test_valid_with_manager(GLOBAL); +// } + +// fn test_valid_with_manager(manager: M) { +// let tokens = [JsonToken::Id(String::from("null"))]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().any.get_type(), Type::Null); + +// let tokens = [JsonToken::Id(String::from("true"))]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().any.try_move(), Ok(true)); + +// let tokens = [JsonToken::Id(String::from("false"))]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().any.try_move(), Ok(false)); + +// let tokens = [JsonToken::Number(0.1)]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// assert_eq!(result.unwrap().any.try_move(), Ok(0.1)); + +// let tokens = [JsonToken::String(String::from("abc"))]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result = result.unwrap().any.try_move::>(); +// assert!(result.is_ok()); +// let result = result.unwrap(); +// let items = result.items(); +// assert_eq!(items, [0x61, 0x62, 0x63]); + +// let tokens = [JsonToken::BigInt(from_u64(manager, js::js_bigint::Sign::Positive, 1))]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result = result.unwrap().any.try_move::>(); +// assert!(result.is_ok()); +// let result = result.unwrap(); +// assert_eq!(result.header_len(), 1); +// let items = result.items(); +// assert_eq!(items, [0x1]); - let tokens = [JsonToken::BigInt(new_bigint(manager, js::js_bigint::Sign::Negative, [2, 3]))]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result = result.unwrap().any.try_move::>(); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.header_len(), -2); - let items = result.items(); - assert_eq!(items, [0x2, 0x3]); - - let tokens = [JsonToken::ArrayBegin, JsonToken::ArrayEnd]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - assert!(items.is_empty()); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::Number(1.0), - JsonToken::Comma, - JsonToken::Id(String::from("true")), - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let item0 = items[0].clone(); - assert_eq!(item0.try_move(), Ok(1.0)); - let item1 = items[1].clone(); - assert_eq!(item1.try_move(), Ok(true)); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::Number(1.0), - JsonToken::Comma, - JsonToken::Id(String::from("true")), - JsonToken::Comma, - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("k1")), - JsonToken::Colon, - JsonToken::Number(1.0), - JsonToken::Comma, - JsonToken::String(String::from("k0")), - JsonToken::Colon, - JsonToken::Number(0.0), - JsonToken::Comma, - JsonToken::String(String::from("k2")), - JsonToken::Colon, - JsonToken::Number(2.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let (key0, value0) = items[0].clone(); - let key0_items = key0.items(); - assert_eq!(key0_items, [0x6b, 0x30]); - assert_eq!(value0.try_move(), Ok(0.0)); - let (key1, value1) = items[1].clone(); - let key1_items = key1.items(); - assert_eq!(key1_items, [0x6b, 0x31]); - assert_eq!(value1.try_move(), Ok(1.0)); - let (key2, value2) = items[2].clone(); - let key2_items = key2.items(); - assert_eq!(key2_items, [0x6b, 0x32]); - assert_eq!(value2.try_move(), Ok(2.0)); - - let tokens = [JsonToken::ObjectBegin, JsonToken::ObjectEnd]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result - .unwrap() - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - assert!(items.is_empty()); - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("k")), - JsonToken::Colon, - JsonToken::ObjectBegin, - JsonToken::ObjectEnd, - JsonToken::ObjectEnd, - ]; - { - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_ok()); - let result_unwrap = result.unwrap(); - let result_unwrap = result_unwrap - .any - .try_move::>() - .unwrap(); - let items = result_unwrap.items(); - let (_, value0) = items[0].clone(); - let value0_unwrap = value0.try_move::>().unwrap(); - let value0_items = value0_unwrap.items(); - assert!(value0_items.is_empty()); - } - } - - #[test] - #[wasm_bindgen_test] - fn test_invalid_local() { - test_invalid_with_manager(&Local::default()); - } - - #[test] - #[wasm_bindgen_test] - fn test_invalid_global() { - test_invalid_with_manager(GLOBAL); - } - - fn test_invalid_with_manager(manager: M) { - let tokens = []; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [JsonToken::ErrorToken(ErrorType::InvalidNumber)]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [JsonToken::ArrayBegin, JsonToken::Comma, JsonToken::ArrayEnd]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::Number(0.0), - JsonToken::Number(1.0), - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::Number(0.0), - JsonToken::Comma, - JsonToken::Comma, - JsonToken::Number(1.0), - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::ArrayEnd, - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [JsonToken::ArrayBegin, JsonToken::String(String::default())]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::Comma, - JsonToken::Number(1.0), - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [JsonToken::ArrayBegin, JsonToken::Colon, JsonToken::ArrayEnd]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [JsonToken::ArrayEnd]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::Comma, - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::Number(0.0), - JsonToken::Comma, - JsonToken::Number(1.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("key")), - JsonToken::Number(0.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("key")), - JsonToken::Colon, - JsonToken::Colon, - JsonToken::Number(0.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("key0")), - JsonToken::Colon, - JsonToken::Number(0.0), - JsonToken::Comma, - JsonToken::Comma, - JsonToken::String(String::from("key1")), - JsonToken::Colon, - JsonToken::Number(1.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::ObjectEnd, - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::String(String::from("key")), - JsonToken::Colon, - JsonToken::Number(0.0), - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::Comma, - JsonToken::String(String::from("key")), - JsonToken::Colon, - JsonToken::Number(0.0), - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [JsonToken::ObjectEnd]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ArrayBegin, - JsonToken::ObjectBegin, - JsonToken::ArrayEnd, - JsonToken::ObjectEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - - let tokens = [ - JsonToken::ObjectBegin, - JsonToken::ArrayBegin, - JsonToken::ObjectEnd, - JsonToken::ArrayEnd, - ]; - let result = parse_with_virtual_io(manager, tokens.into_iter()); - assert!(result.is_err()); - } -} +// let tokens = [JsonToken::BigInt(new_bigint(manager, js::js_bigint::Sign::Negative, [2, 3]))]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result = result.unwrap().any.try_move::>(); +// assert!(result.is_ok()); +// let result = result.unwrap(); +// assert_eq!(result.header_len(), -2); +// let items = result.items(); +// assert_eq!(items, [0x2, 0x3]); + +// let tokens = [JsonToken::ArrayBegin, JsonToken::ArrayEnd]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// assert!(items.is_empty()); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::Number(1.0), +// JsonToken::Comma, +// JsonToken::Id(String::from("true")), +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let item0 = items[0].clone(); +// assert_eq!(item0.try_move(), Ok(1.0)); +// let item1 = items[1].clone(); +// assert_eq!(item1.try_move(), Ok(true)); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::Number(1.0), +// JsonToken::Comma, +// JsonToken::Id(String::from("true")), +// JsonToken::Comma, +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("k1")), +// JsonToken::Colon, +// JsonToken::Number(1.0), +// JsonToken::Comma, +// JsonToken::String(String::from("k0")), +// JsonToken::Colon, +// JsonToken::Number(0.0), +// JsonToken::Comma, +// JsonToken::String(String::from("k2")), +// JsonToken::Colon, +// JsonToken::Number(2.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let (key0, value0) = items[0].clone(); +// let key0_items = key0.items(); +// assert_eq!(key0_items, [0x6b, 0x30]); +// assert_eq!(value0.try_move(), Ok(0.0)); +// let (key1, value1) = items[1].clone(); +// let key1_items = key1.items(); +// assert_eq!(key1_items, [0x6b, 0x31]); +// assert_eq!(value1.try_move(), Ok(1.0)); +// let (key2, value2) = items[2].clone(); +// let key2_items = key2.items(); +// assert_eq!(key2_items, [0x6b, 0x32]); +// assert_eq!(value2.try_move(), Ok(2.0)); + +// let tokens = [JsonToken::ObjectBegin, JsonToken::ObjectEnd]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result +// .unwrap() +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// assert!(items.is_empty()); +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("k")), +// JsonToken::Colon, +// JsonToken::ObjectBegin, +// JsonToken::ObjectEnd, +// JsonToken::ObjectEnd, +// ]; +// { +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_ok()); +// let result_unwrap = result.unwrap(); +// let result_unwrap = result_unwrap +// .any +// .try_move::>() +// .unwrap(); +// let items = result_unwrap.items(); +// let (_, value0) = items[0].clone(); +// let value0_unwrap = value0.try_move::>().unwrap(); +// let value0_items = value0_unwrap.items(); +// assert!(value0_items.is_empty()); +// } +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_invalid_local() { +// test_invalid_with_manager(&Local::default()); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_invalid_global() { +// test_invalid_with_manager(GLOBAL); +// } + +// fn test_invalid_with_manager(manager: M) { +// let tokens = []; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [JsonToken::ErrorToken(ErrorType::InvalidNumber)]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [JsonToken::ArrayBegin, JsonToken::Comma, JsonToken::ArrayEnd]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::Number(0.0), +// JsonToken::Number(1.0), +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::Number(0.0), +// JsonToken::Comma, +// JsonToken::Comma, +// JsonToken::Number(1.0), +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::ArrayEnd, +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [JsonToken::ArrayBegin, JsonToken::String(String::default())]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::Comma, +// JsonToken::Number(1.0), +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [JsonToken::ArrayBegin, JsonToken::Colon, JsonToken::ArrayEnd]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [JsonToken::ArrayEnd]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::Comma, +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::Number(0.0), +// JsonToken::Comma, +// JsonToken::Number(1.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("key")), +// JsonToken::Number(0.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("key")), +// JsonToken::Colon, +// JsonToken::Colon, +// JsonToken::Number(0.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("key0")), +// JsonToken::Colon, +// JsonToken::Number(0.0), +// JsonToken::Comma, +// JsonToken::Comma, +// JsonToken::String(String::from("key1")), +// JsonToken::Colon, +// JsonToken::Number(1.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::ObjectEnd, +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::String(String::from("key")), +// JsonToken::Colon, +// JsonToken::Number(0.0), +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::Comma, +// JsonToken::String(String::from("key")), +// JsonToken::Colon, +// JsonToken::Number(0.0), +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [JsonToken::ObjectEnd]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ArrayBegin, +// JsonToken::ObjectBegin, +// JsonToken::ArrayEnd, +// JsonToken::ObjectEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); + +// let tokens = [ +// JsonToken::ObjectBegin, +// JsonToken::ArrayBegin, +// JsonToken::ObjectEnd, +// JsonToken::ArrayEnd, +// ]; +// let result = parse_with_virtual_io(manager, tokens.into_iter()); +// assert!(result.is_err()); +// } +// } diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 7425cfa9..af693363 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -334,14 +334,14 @@ const fn digit_to_number(c: char) -> u64 { c as u64 - CP_0 as u64 } -fn start_number(s: Sign, c: char) -> IntegerState { - IntegerState::from_difit(s, c) +fn start_number(manager: M, s: js_bigint::Sign, c: char) -> JsBigintMutRef { + JsBigintMutRef::from_digit(manager, s, c) } -fn create_range_map( +fn create_range_map( list: Vec>, - t: Transition, -) -> RangeMap>> { + t: Transition, +) -> RangeMap>> { let mut result = RangeMap { list: default() }; for range in list { result = merge(from_range(range, t), result); @@ -362,7 +362,7 @@ fn set(arr: impl IntoIterator) -> Vec> { } type Transition = - fn(state: T, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); + fn(state: T, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); struct TransitionMap { def: Transition, @@ -391,18 +391,18 @@ pub struct TransitionMaps { operator: TransitionMap, } -pub fn create_transition_maps() -> TransitionMaps { +pub fn create_transition_maps(manager: M) -> TransitionMaps { TransitionMaps { initial: create_initial_transitions(), id: create_id_transitions(), string: create_string_transactions(), escape_char: create_escape_char_transactions(), unicode_char: create_unicode_char_transactions(), - zero: create_zero_transactions(), - int: create_int_transactions(), + zero: create_zero_transactions(manager), + int: create_int_transactions(manager), minus: create_minus_transactions(), frac_begin: create_frac_begin_transactions(), - frac: create_frac_transactions(), + frac: create_frac_transactions(manager), exp_begin: create_exp_begin_transactions(), exp: create_exp_transactions(), big_int: create_big_int_transactions(), @@ -418,11 +418,11 @@ pub fn create_transition_maps() -> TransitionMaps { fn get_next_state( state: T, c: char, - tm: &TransitionMap, - maps: &TransitionMaps, + tm: &TransitionMap, + maps: &TransitionMaps, ) -> (Vec>, TokenizerState) where - T: 'static, + T: 'static, D: 'static { let entry = tm.rm.get(c); match &entry.value { @@ -431,15 +431,15 @@ where } } -fn create_initial_transitions() -> TransitionMap<()> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +fn create_initial_transitions(manager: M) -> TransitionMap<(), M::Dealloc> { + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { def: (|_, _, _| { ( [JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)].cast(), TokenizerState::Initial, ) - }) as Func, + }) as Func, rm: merge_list( [ create_range_map(operator_chars_with_dot(), |_, c, _| { @@ -448,7 +448,7 @@ fn create_initial_transitions() -> TransitionMap<()> { from_range('1'..='9', |_, c, _| { ( default(), - TokenizerState::ParseInt(start_number(Sign::Positive, c)), + TokenizerState::ParseInt(start_number(manager, Sign::Positive, c)), ) }), from_one('"', |_, _, _| { @@ -474,7 +474,7 @@ fn create_initial_transitions() -> TransitionMap<()> { } } -fn create_id_transitions() -> TransitionMap { +fn create_id_transitions() -> TransitionMap { TransitionMap { def: |s, c, maps| { transfer_state([JsonToken::Id(s)].cast(), TokenizerState::Initial, c, maps) @@ -486,7 +486,7 @@ fn create_id_transitions() -> TransitionMap { } } -fn create_string_transactions() -> TransitionMap { +fn create_string_transactions() -> TransitionMap { TransitionMap { def: |mut s, c, _| { s.push(c); @@ -557,7 +557,7 @@ fn create_escape_char_transactions() -> TransitionMap { } fn create_unicode_char_transactions() -> TransitionMap { - type Func = fn( + type Func = fn( state: ParseUnicodeCharState, c: char, maps: &TransitionMaps, @@ -575,15 +575,15 @@ fn create_unicode_char_transactions() -> TransitionMap, ), from_range( 'a'..='f', - (|state, c, _| state.push(c as u32 - ('a' as u32 - 10))) as Func, + (|state, c, _| state.push(c as u32 - ('a' as u32 - 10))) as Func, ), from_range( 'A'..='F', - (|state, c, _| state.push(c as u32 - ('A' as u32 - 10))) as Func, + (|state, c, _| state.push(c as u32 - ('A' as u32 - 10))) as Func, ), ] .cast(), @@ -591,10 +591,10 @@ fn create_unicode_char_transactions() -> TransitionMap(m: M) -> TransitionMap { - type Func = fn(s: Sign, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +fn create_zero_transactions(manager: M) -> TransitionMap { + type Func = fn(s: Sign, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_one( @@ -602,27 +602,28 @@ fn create_zero_transactions(m: M) -> TransitionMap (|s, _, _| { ( default(), - TokenizerState::ParseFracBegin(zero(m)), + TokenizerState::ParseFracBegin(zero(manager)), ) - }) as Func, + }) as Func, ), create_range_map(set(['e', 'E']), |s, _, _| { ( default(), TokenizerState::ParseExpBegin(ExpState { - b: zero(m), + b: zero(manager), fe: 0, es: Sign::Positive, e: 0, }), ) }), - from_one('n', |s, _, _| { + from_one('n', (|s, _, _| { ( default(), - TokenizerState::ParseBigInt(zero(m)), + TokenizerState::ParseBigInt(zero(manager)), ) - }), + }) as Func + ), create_range_map(terminal_for_number(), |_, c, maps| { transfer_state( [JsonToken::Number(default())].cast(), @@ -637,16 +638,16 @@ fn create_zero_transactions(m: M) -> TransitionMap } } -fn create_int_transactions() -> TransitionMap, D> { - type Func = - fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_int_transactions(manager: M) -> TransitionMap, M::Dealloc> { + type Func = + fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseInt(s.add_digit(c)))) as Func, + (|s, c, _| (default(), TokenizerState::ParseInt(s.add_digit(manager, c)))) as Func, ), from_one('.', |s, _, _| { (default(), TokenizerState::ParseFracBegin(s)) @@ -665,10 +666,10 @@ fn create_int_transactions() -> TransitionMap, D> } fn create_frac_begin_transactions() -> TransitionMap, D> { - type Func = - fn(s: IntegerState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = + fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: from_range( '0'..='9', (|s, c, _| { @@ -681,16 +682,16 @@ fn create_frac_begin_transactions() -> TransitionMap() -> TransitionMap, D> { - type Func = - fn(s: FloatState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +fn create_frac_transactions(manager: M) -> TransitionMap, M::Dealloc> { + type Func = + fn(s: FloatState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseFrac(s.add_digit(c)))) as Func, + (|s, c, _| (default(), TokenizerState::ParseFrac(s.add_digit(manager, c)))) as Func, ), create_range_map(set(['e', 'E']), |s, _, _| { (default(), TokenizerState::ParseExpBegin(s.into_exp_state())) @@ -704,19 +705,19 @@ fn create_frac_transactions() -> TransitionMap, D> { } } -fn create_minus_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +fn create_minus_transactions(manager: M) -> TransitionMap<(), M::Dealloc> { + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge( from_one( '0', - (|_, _, _| (default(), TokenizerState::ParseZero(Sign::Negative))) as Func, + (|_, _, _| (default(), TokenizerState::ParseZero(Sign::Negative))) as Func, ), from_range('1'..='9', |_, c, _| { ( default(), - TokenizerState::ParseInt(start_number(Sign::Negative, c)), + TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Negative, c)), ) }), ), @@ -724,14 +725,14 @@ fn create_minus_transactions() -> TransitionMap<(), D> { } fn create_exp_begin_transactions() -> TransitionMap, D> { - type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, + (|s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, ), from_one('+', |s, _, _| (default(), TokenizerState::ParseExpSign(s))), from_one('-', |mut s, _, _| { @@ -750,13 +751,13 @@ fn create_exp_begin_transactions() -> TransitionMap, D> } fn create_exp_transactions() -> TransitionMap, D> { - type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge( from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, + (|s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, ), create_range_map(terminal_for_number(), |s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) @@ -766,10 +767,10 @@ fn create_exp_transactions() -> TransitionMap, D> { } fn create_big_int_transactions() -> TransitionMap, D> { - type Func = - fn(s: IntegerState, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = + fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: create_range_map(terminal_for_number(), |s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) }), @@ -786,7 +787,7 @@ fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Ve } fn create_new_line_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { def: (|_, c, maps| { transfer_state( @@ -795,7 +796,7 @@ fn create_new_line_transactions() -> TransitionMap<(), D> { c, maps, ) - }) as Func, + }) as Func, rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _| { (default(), TokenizerState::ParseNewLine) }), @@ -803,14 +804,14 @@ fn create_new_line_transactions() -> TransitionMap<(), D> { } fn create_comment_start_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { def: (|_, _, _| { ( [JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)].cast(), TokenizerState::Initial, ) - }) as Func, + }) as Func, rm: merge( from_one('/', |_, _, _| { (default(), TokenizerState::ParseSinglelineComment) @@ -823,9 +824,9 @@ fn create_comment_start_transactions() -> TransitionMap<(), D> { } fn create_singleline_comment_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func, + def: (|_, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func, rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _| { (default(), TokenizerState::ParseNewLine) }), @@ -833,9 +834,9 @@ fn create_singleline_comment_transactions() -> TransitionMap<(), D> } fn create_multiline_comment_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, + def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, rm: from_one('*', |_, _, _| { (default(), TokenizerState::ParseMultilineCommentAsterix) }), @@ -843,9 +844,9 @@ fn create_multiline_comment_transactions() -> TransitionMap<(), D> { } fn create_multiline_comment_asterix_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); + type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, + def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, rm: merge( from_one('/', |_, _, _| (default(), TokenizerState::Initial)), from_one('*', |_, _, _| { @@ -879,32 +880,34 @@ fn create_operator_transactions() -> TransitionMap { } } -pub fn tokenize(input: String) -> Vec> { - TokenizerStateIterator::new(input.chars()).collect() +pub fn tokenize(manager: M, input: String) -> Vec> { + TokenizerStateIterator::new(manager, input.chars()).collect() } -pub struct TokenizerStateIterator, D: Dealloc> { +pub struct TokenizerStateIterator, M: Manager> { + manager: M, chars: T, - cache: VecDeque>, - state: TokenizerState, - maps: TransitionMaps, + cache: VecDeque>, + state: TokenizerState, + maps: TransitionMaps, end: bool, } -impl, D: Dealloc> TokenizerStateIterator { - pub fn new(chars: T) -> Self { +impl, M: Manager> TokenizerStateIterator { + pub fn new(manager: M, chars: T) -> Self { Self { + manager, chars, cache: default(), state: default(), - maps: create_transition_maps(), + maps: create_transition_maps(manager), end: false, } } } -impl, D: Dealloc> Iterator for TokenizerStateIterator { - type Item = JsonToken; +impl, M: Manager> Iterator for TokenizerStateIterator { + type Item = JsonToken; fn next(&mut self) -> Option { loop { @@ -925,503 +928,504 @@ impl, D: Dealloc> Iterator for TokenizerStateIterator::ObjectBegin]); + +// let result = tokenize(String::from("}")); +// assert_eq!(&result, &[JsonToken::ObjectEnd]); + +// let result = tokenize(String::from("[")); +// assert_eq!(&result, &[JsonToken::ArrayBegin]); + +// let result = tokenize(String::from("]")); +// assert_eq!(&result, &[JsonToken::ArrayEnd]); + +// let result = tokenize(String::from(":")); +// assert_eq!(&result, &[JsonToken::Colon]); + +// let result = tokenize(String::from(",")); +// assert_eq!(&result, &[JsonToken::Comma]); + +// let result = tokenize(String::from("=")); +// assert_eq!(&result, &[JsonToken::Equals]); + +// let result = tokenize(String::from(".")); +// assert_eq!(&result, &[JsonToken::Dot]); + +// let result = tokenize(String::from(";")); +// assert_eq!(&result, &[JsonToken::Semicolon]); + +// let result = tokenize(String::from("()")); +// assert_eq!( +// &result, +// &[JsonToken::OpeningParenthesis, JsonToken::ClosingParenthesis] +// ); + +// let result = tokenize(String::from("[{ :, }]")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ArrayBegin, +// JsonToken::ObjectBegin, +// JsonToken::Colon, +// JsonToken::Comma, +// JsonToken::ObjectEnd, +// JsonToken::ArrayEnd +// ] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_id() { +// let result = tokenize(String::from("true")); +// assert_eq!(&result, &[JsonToken::Id(String::from("true"))]); + +// let result = tokenize(String::from("false")); +// assert_eq!(&result, &[JsonToken::Id(String::from("false"))]); + +// let result = tokenize(String::from("null")); +// assert_eq!(&result, &[JsonToken::Id(String::from("null"))]); + +// let result = tokenize(String::from("tru tru")); +// assert_eq!( +// &result, +// &[ +// JsonToken::Id(String::from("tru")), +// JsonToken::Id(String::from("tru")), +// ] +// ); + +// let result = tokenize(String::from("ABCxyz_0123456789$")); +// assert_eq!( +// &result, +// &[JsonToken::Id(String::from("ABCxyz_0123456789$")),] +// ); + +// let result = tokenize(String::from("_")); +// assert_eq!(&result, &[JsonToken::Id(String::from("_")),]); + +// let result = tokenize(String::from("$")); +// assert_eq!(&result, &[JsonToken::Id(String::from("$")),]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_whitespace() { +// let result = tokenize(String::from(" \t\n\r")); +// assert_eq!(&result, &[]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_string() { +// let result = tokenize(String::from("\"\"")); +// assert_eq!(&result, &[JsonToken::String("".to_string())]); + +// let result = tokenize(String::from("\"value\"")); +// assert_eq!(&result, &[JsonToken::String("value".to_string())]); + +// let result = tokenize(String::from("\"value1\" \"value2\"")); +// assert_eq!( +// &result, +// &[ +// JsonToken::String("value1".to_string()), +// JsonToken::String("value2".to_string()) +// ] +// ); + +// let result = tokenize(String::from("\"value")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_escaped_characters() { +// let result = tokenize(String::from("\"\\b\\f\\n\\r\\t\"")); +// assert_eq!( +// &result, +// &[JsonToken::String("\u{8}\u{c}\n\r\t".to_string())] +// ); + +// let result = tokenize(String::from("\"\\x\"")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), +// JsonToken::String("x".to_string()) +// ] +// ); + +// let result = tokenize(String::from("\"\\")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_unicode() { +// let result = tokenize(String::from("\"\\u1234\"")); +// assert_eq!(&result, &[JsonToken::String("ሴ".to_string())]); + +// let result = tokenize(String::from("\"\\uaBcDEeFf\"")); +// assert_eq!(&result, &[JsonToken::String("ꯍEeFf".to_string())]); + +// let result = tokenize(String::from("\"\\uEeFg\"")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidHex), +// JsonToken::String("g".to_string()) +// ] +// ); + +// let result = tokenize(String::from("\"\\uEeF")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_integer() { +// let result = tokenize(String::from("0")); +// assert_eq!(&result, &[JsonToken::Number(0.0)]); + +// let result = tokenize(String::from("-0")); +// assert_eq!(&result, &[JsonToken::Number(0.0)]); + +// let result = tokenize(String::from("0abc")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Id(String::from("abc")) +// ] +// ); + +// let result = tokenize(String::from("0. 2")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Number(2.0) +// ] +// ); + +// let result = tokenize(String::from("1234567890")); +// assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); + +// let result = tokenize(String::from("-1234567890")); +// assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); + +// let result = tokenize(String::from("[0,1]")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ArrayBegin, +// JsonToken::Number(0.0), +// JsonToken::Comma, +// JsonToken::Number(1.0), +// JsonToken::ArrayEnd +// ] +// ); + +// let result = tokenize(String::from("001")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Number(1.0), +// ] +// ); + +// let result = tokenize(String::from("-")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + +// let result = tokenize(String::from("-{}")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::ObjectBegin, +// JsonToken::ObjectEnd +// ] +// ); + +// let result = tokenize(String::from("9007199254740991")); +// assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); + +// let result = tokenize(String::from("9007199254740992")); +// assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); + +// let result = tokenize(String::from("9007199254740993")); +// assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_big_float() { +// let result = tokenize(String::from("340282366920938463463374607431768211456")); +// assert_eq!( +// &result, +// &[JsonToken::Number(bigfloat_to_f64(BigFloat { +// significand: BigInt { +// sign: Sign::Positive, +// value: BigUint { +// value: [0, 0, 1].cast() +// } +// }, +// exp: 0, +// non_zero_reminder: false +// }))] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_float() { +// let result = tokenize(String::from("0.01")); +// assert_eq!(&result, &[JsonToken::Number(0.01)]); + +// let result = tokenize(String::from("[-12.34]")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ArrayBegin, +// JsonToken::Number(-12.34), +// JsonToken::ArrayEnd +// ] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_infinity() { +// let result = tokenize(String::from("1e1000")); +// assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); + +// let result = tokenize(String::from("-1e+1000")); +// assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_exp() { +// let result = tokenize(String::from("1e2")); +// assert_eq!(&result, &[JsonToken::Number(1e2)]); + +// let result = tokenize(String::from("1E+2")); +// assert_eq!(&result, &[JsonToken::Number(1e2)]); + +// let result = tokenize(String::from("0e-2")); +// assert_eq!(&result, &[JsonToken::Number(0.0)]); + +// let result = tokenize(String::from("1e-2")); +// assert_eq!(&result, &[JsonToken::Number(1e-2)]); + +// let result = tokenize(String::from("1.2e+2")); +// assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); + +// let result = tokenize(String::from("12e0000")); +// assert_eq!(&result, &[JsonToken::Number(12.0)]); + +// let result = tokenize(String::from("1e")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + +// let result = tokenize(String::from("1e+")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + +// let result = tokenize(String::from("1e-")); +// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_big_int() { +// let result = tokenize(String::from("0n")); +// assert_eq!(&result, &[JsonToken::BigInt(BigInt::ZERO)]); + +// let result = tokenize(String::from("-0n")); +// assert_eq!( +// &result, +// &[JsonToken::BigInt(BigInt { +// sign: Sign::Negative, +// value: BigUint::ZERO +// })] +// ); + +// let result = tokenize(String::from("1234567890n")); +// assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_u64(1234567890))]); + +// let result = tokenize(String::from("-1234567890n")); +// assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_i64(-1234567890))]); + +// let result = tokenize(String::from("123.456n")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Id(String::from("n")) +// ] +// ); + +// let result = tokenize(String::from("123e456n")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Id(String::from("n")) +// ] +// ); + +// let result = tokenize(String::from("1234567890na")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Id(String::from("a")) +// ] +// ); + +// let result = tokenize(String::from("1234567890nn")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ErrorToken(ErrorType::InvalidNumber), +// JsonToken::Id(String::from("n")) +// ] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_errors() { +// let result = tokenize(String::from("ᄑ")); +// assert_eq!( +// &result, +// &[JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_djs() { +// let result = tokenize(String::from("module.exports = ")); +// assert_eq!( +// &result, +// &[ +// JsonToken::Id(String::from("module")), +// JsonToken::Dot, +// JsonToken::Id(String::from("exports")), +// JsonToken::Equals, +// ] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_singleline_comments() { +// let result = tokenize(String::from("{//abc\n2\n}")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ObjectBegin, +// JsonToken::NewLine, +// JsonToken::Number(2.0), +// JsonToken::NewLine, +// JsonToken::ObjectEnd, +// ] +// ); + +// let result = tokenize(String::from("0//abc/*")); +// assert_eq!(&result, &[JsonToken::Number(0.0),]); + +// let result = tokenize(String::from("0//")); +// assert_eq!(&result, &[JsonToken::Number(0.0),]); + +// let result = tokenize(String::from("0/")); +// assert_eq!( +// &result, +// &[ +// JsonToken::Number(0.0), +// JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), +// ] +// ); + +// let result = tokenize(String::from("0/a")); +// assert_eq!( +// &result, +// &[ +// JsonToken::Number(0.0), +// JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), +// ] +// ); +// } + +// #[test] +// #[wasm_bindgen_test] +// fn test_multiline_comments() { +// let result = tokenize(String::from("{/*abc\ndef*/2}")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ObjectBegin, +// JsonToken::Number(2.0), +// JsonToken::ObjectEnd, +// ] +// ); + +// let result = tokenize(String::from("{/*/* /**/2}")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ObjectBegin, +// JsonToken::Number(2.0), +// JsonToken::ObjectEnd, +// ] +// ); + +// let result = tokenize(String::from("{/*")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ObjectBegin, +// JsonToken::ErrorToken(ErrorType::CommentClosingExpected), +// ] +// ); + +// let result = tokenize(String::from("{/**")); +// assert_eq!( +// &result, +// &[ +// JsonToken::ObjectBegin, +// JsonToken::ErrorToken(ErrorType::CommentClosingExpected), +// ] +// ); +// } +// } From 0405c7499b6b4935e2622dd36a6c34ab3928302b Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 9 Nov 2024 20:52:15 +0200 Subject: [PATCH 05/28] minor --- nanvm-lib/src/tokenizer/mod.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index af693363..dbdbe83f 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -3,7 +3,7 @@ use std::{collections::VecDeque, mem::take, ops::{Deref, RangeInclusive}}; use crate::{ big_numbers::{ self, big_float::BigFloat, big_int::{BigInt, Sign}, big_uint::BigUint - }, common::{cast::Cast, default::default}, js::js_bigint::{self, add, from_u64, zero, JsBigintMutRef, JsBigintRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State} + }, common::{cast::Cast, default::default}, js::js_bigint::{self, add, from_u64, zero, JsBigintMutRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State} }; #[derive(Debug)] @@ -62,8 +62,10 @@ pub enum TokenizerState { ParseOperator(String), } -impl TokenizerState { - fn push(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { +impl TokenizerState +where + D: 'static { + fn push(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { match self { TokenizerState::Initial => get_next_state((), c, &maps.initial, maps), TokenizerState::ParseId(s) => get_next_state(s, c, &maps.id, maps), @@ -95,7 +97,7 @@ impl TokenizerState { } } - pub fn push_mut(&mut self, c: char, tm: &TransitionMaps) -> Vec> { + pub fn push_mut(&mut self, c: char, tm: &TransitionMaps) -> Vec> { let tokens; (tokens, *self) = take(self).push(c, tm); tokens @@ -247,10 +249,7 @@ impl FloatState { fn into_token(self) -> JsonToken { JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: BigInt { - sign: self.s, - value: self.b, - }, + significand: self.b.to_old_bigint(), exp: self.fe, non_zero_reminder: false, })) @@ -393,14 +392,14 @@ pub struct TransitionMaps { pub fn create_transition_maps(manager: M) -> TransitionMaps { TransitionMaps { - initial: create_initial_transitions(), + initial: create_initial_transitions(manager), id: create_id_transitions(), string: create_string_transactions(), escape_char: create_escape_char_transactions(), unicode_char: create_unicode_char_transactions(), zero: create_zero_transactions(manager), int: create_int_transactions(manager), - minus: create_minus_transactions(), + minus: create_minus_transactions(manager), frac_begin: create_frac_begin_transactions(), frac: create_frac_transactions(manager), exp_begin: create_exp_begin_transactions(), From 42e1a87c28919f6408e15dacfa7ca55a8019991a Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 9 Nov 2024 22:04:34 +0200 Subject: [PATCH 06/28] transition map --- nanvm-lib/src/tokenizer/mod.rs | 341 ++++++++++++++++----------------- 1 file changed, 170 insertions(+), 171 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index dbdbe83f..fdd38e9c 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -65,7 +65,7 @@ pub enum TokenizerState { impl TokenizerState where D: 'static { - fn push(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { + fn push>(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { match self { TokenizerState::Initial => get_next_state((), c, &maps.initial, maps), TokenizerState::ParseId(s) => get_next_state(s, c, &maps.id, maps), @@ -180,11 +180,11 @@ impl BigUint { } impl JsBigintMutRef { - fn from_digit(m: M, sign: js_bigint::Sign, c: char) -> JsBigintMutRef { + fn from_digit>(m: M, sign: js_bigint::Sign, c: char) -> JsBigintMutRef { from_u64(m, sign, digit_to_number(c)) } - fn add_digit(self, m: M, c: char) -> JsBigintMutRef { + fn add_digit>(self, m: M, c: char) -> JsBigintMutRef { add(m, self.deref(), Self::from_digit(m, js_bigint::Sign::Positive, c).deref()) } @@ -232,7 +232,7 @@ pub struct FloatState { } impl FloatState { - fn add_digit(mut self, m: M, c: char) -> FloatState { + fn add_digit>(mut self, m: M, c: char) -> FloatState { self.b = self.b.add_digit(m, c); self.fe -= 1; self @@ -276,10 +276,7 @@ impl ExpState { Sign::Negative => -self.e, }; JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: BigInt { - sign: self.s, - value: self.b, - }, + significand: self.b.to_old_bigint(), exp, non_zero_reminder: false, })) @@ -337,10 +334,10 @@ fn start_number(manager: M, s: js_bigint::Sign, c: char) -> JsBigint JsBigintMutRef::from_digit(manager, s, c) } -fn create_range_map( +fn create_range_map( list: Vec>, - t: Transition, -) -> RangeMap>> { + t: Transition, +) -> RangeMap>> { let mut result = RangeMap { list: default() }; for range in list { result = merge(from_range(range, t), result); @@ -360,48 +357,48 @@ fn set(arr: impl IntoIterator) -> Vec> { result } -type Transition = - fn(state: T, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +type Transition = + fn(manager: M, state: T, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); -struct TransitionMap { - def: Transition, - rm: RangeMap>>, +struct TransitionMap { + def: Transition, + rm: RangeMap>>, } -pub struct TransitionMaps { - initial: TransitionMap<(), D>, - id: TransitionMap, - string: TransitionMap, - escape_char: TransitionMap, - unicode_char: TransitionMap, - zero: TransitionMap, - int: TransitionMap, D>, - minus: TransitionMap<(), D>, - frac_begin: TransitionMap, D>, - frac: TransitionMap, D>, - exp_begin: TransitionMap, D>, - exp: TransitionMap, D>, - big_int: TransitionMap, D>, - new_line: TransitionMap<(), D>, - comment_start: TransitionMap<(), D>, - singleline_comment: TransitionMap<(), D>, - multiline_comment: TransitionMap<(), D>, - multiline_comment_asterix: TransitionMap<(), D>, - operator: TransitionMap, +pub struct TransitionMaps { + initial: TransitionMap<(), M>, + id: TransitionMap, + string: TransitionMap, + escape_char: TransitionMap, + unicode_char: TransitionMap, + zero: TransitionMap, + int: TransitionMap, M>, + minus: TransitionMap<(), M>, + frac_begin: TransitionMap, M>, + frac: TransitionMap, M>, + exp_begin: TransitionMap, M>, + exp: TransitionMap, M>, + big_int: TransitionMap, M>, + new_line: TransitionMap<(), M>, + comment_start: TransitionMap<(), M>, + singleline_comment: TransitionMap<(), M>, + multiline_comment: TransitionMap<(), M>, + multiline_comment_asterix: TransitionMap<(), M>, + operator: TransitionMap, } -pub fn create_transition_maps(manager: M) -> TransitionMaps { +pub fn create_transition_maps(manager: M) -> TransitionMaps { TransitionMaps { - initial: create_initial_transitions(manager), + initial: create_initial_transitions(), id: create_id_transitions(), string: create_string_transactions(), escape_char: create_escape_char_transactions(), unicode_char: create_unicode_char_transactions(), - zero: create_zero_transactions(manager), - int: create_int_transactions(manager), - minus: create_minus_transactions(manager), + zero: create_zero_transactions(), + int: create_int_transactions(), + minus: create_minus_transactions(), frac_begin: create_frac_begin_transactions(), - frac: create_frac_transactions(manager), + frac: create_frac_transactions(), exp_begin: create_exp_begin_transactions(), exp: create_exp_transactions(), big_int: create_big_int_transactions(), @@ -430,41 +427,42 @@ where } } -fn create_initial_transitions(manager: M) -> TransitionMap<(), M::Dealloc> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_initial_transitions() -> TransitionMap<(), M> { + type Func = fn(manager: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| { + def: (|_, _, _, _| { ( [JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)].cast(), TokenizerState::Initial, ) - }) as Func, + }) as Func, rm: merge_list( [ - create_range_map(operator_chars_with_dot(), |_, c, _| { + create_range_map(operator_chars_with_dot(), |_, _, c, _| { (default(), TokenizerState::ParseOperator(c.to_string())) }), - from_range('1'..='9', |_, c, _| { + from_range('1'..='9', |manager: M, _, c, _| { ( default(), - TokenizerState::ParseInt(start_number(manager, Sign::Positive, c)), + TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Positive, c)), ) - }), - from_one('"', |_, _, _| { + } + ), + from_one('"', |_, _, _, _| { (default(), TokenizerState::ParseString(String::default())) }), - from_one('0', |_, _, _| { + from_one('0', |_, _, _, _| { (default(), TokenizerState::ParseZero(Sign::Positive)) }), - from_one('-', |_, _, _| (default(), TokenizerState::ParseMinus)), - create_range_map(id_start(), |_, c, _| { + from_one('-', |_, _, _, _| (default(), TokenizerState::ParseMinus)), + create_range_map(id_start(), |_, _, c, _| { (default(), TokenizerState::ParseId(c.to_string())) }), - from_one('\n', |_, _, _| (default(), TokenizerState::ParseNewLine)), - create_range_map(set([' ', '\t', '\r']), |_, _, _| { + from_one('\n', |_, _, _, _| (default(), TokenizerState::ParseNewLine)), + create_range_map(set([' ', '\t', '\r']), |_, _, _, _| { (default(), TokenizerState::Initial) }), - from_one('/', |_, _, _| { + from_one('/', |_, _, _, _| { (default(), TokenizerState::ParseCommentStart) }), ] @@ -473,55 +471,55 @@ fn create_initial_transitions(manager: M) -> TransitionMap<(), M::De } } -fn create_id_transitions() -> TransitionMap { +fn create_id_transitions() -> TransitionMap { TransitionMap { - def: |s, c, maps| { + def: |_, s, c, maps| { transfer_state([JsonToken::Id(s)].cast(), TokenizerState::Initial, c, maps) }, - rm: create_range_map(id_char(), |mut s, c, _| { + rm: create_range_map(id_char(), |_, mut s, c, _| { s.push(c); (default(), TokenizerState::ParseId(s)) }), } } -fn create_string_transactions() -> TransitionMap { +fn create_string_transactions() -> TransitionMap { TransitionMap { - def: |mut s, c, _| { + def: |_, mut s, c, _| { s.push(c); (default(), TokenizerState::ParseString(s)) }, rm: merge( - from_one('"', |s, _, _| { + from_one('"', |_, s, _, _| { ([JsonToken::String(s)].cast(), TokenizerState::Initial) }), - from_one('\\', |s, _, _| { + from_one('\\', |_, s, _, _| { (default(), TokenizerState::ParseEscapeChar(s)) }), ), } } -fn continue_string_state(mut s: String, c: char) -> (Vec>, TokenizerState) { +fn continue_string_state(mut s: String, c: char) -> (Vec>, TokenizerState) { s.push(c); (default(), TokenizerState::ParseString(s)) } -fn transfer_state( - mut vec: Vec>, - mut state: TokenizerState, +fn transfer_state( + mut vec: Vec>, + mut state: TokenizerState, c: char, - maps: &TransitionMaps, -) -> (Vec>, TokenizerState) { + maps: &TransitionMaps, +) -> (Vec>, TokenizerState) { let next_tokens; (next_tokens, state) = state.push(c, maps); vec.extend(next_tokens); (vec, state) } -fn create_escape_char_transactions() -> TransitionMap { +fn create_escape_char_transactions() -> TransitionMap { TransitionMap { - def: |s, c, maps| { + def: |_, s, c, maps| { transfer_state( [JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)].cast(), TokenizerState::ParseString(s), @@ -531,15 +529,15 @@ fn create_escape_char_transactions() -> TransitionMap { }, rm: merge_list( [ - create_range_map(set(['\"', '\\', '/']), |s, c, _| { - continue_string_state(s, c) + create_range_map(set(['\"', '\\', '/']), |_, s, c, _| { + continue_string_state::(s, c) }), - from_one('b', |s, _, _| continue_string_state(s, '\u{8}')), - from_one('f', |s, _, _| continue_string_state(s, '\u{c}')), - from_one('n', |s, _, _| continue_string_state(s, '\n')), - from_one('r', |s, _, _| continue_string_state(s, '\r')), - from_one('t', |s, _, _| continue_string_state(s, '\t')), - from_one('u', |s, _, _| { + from_one('b', |_, s, _, _| continue_string_state::(s, '\u{8}')), + from_one('f', |_, s, _, _| continue_string_state::(s, '\u{c}')), + from_one('n', |_, s, _, _| continue_string_state::(s, '\n')), + from_one('r', |_, s, _, _| continue_string_state::(s, '\r')), + from_one('t', |_, s, _, _| continue_string_state::(s, '\t')), + from_one('u', |_, s, _, _| { ( default(), TokenizerState::ParseUnicodeChar(ParseUnicodeCharState { @@ -555,14 +553,15 @@ fn create_escape_char_transactions() -> TransitionMap { } } -fn create_unicode_char_transactions() -> TransitionMap { - type Func = fn( +fn create_unicode_char_transactions() -> TransitionMap { + type Func = fn( + m: M, state: ParseUnicodeCharState, c: char, - maps: &TransitionMaps, - ) -> (Vec>, TokenizerState); + maps: &TransitionMaps, + ) -> (Vec>, TokenizerState); TransitionMap { - def: |state, c, maps| { + def: |_, state, c, maps| { transfer_state( [JsonToken::ErrorToken(ErrorType::InvalidHex)].cast(), TokenizerState::ParseString(state.s), @@ -574,15 +573,15 @@ fn create_unicode_char_transactions() -> TransitionMap, + (|_, state, c, _| state.push(c as u32 - '0' as u32)) as Func, ), from_range( 'a'..='f', - (|state, c, _| state.push(c as u32 - ('a' as u32 - 10))) as Func, + (|_, state, c, _| state.push(c as u32 - ('a' as u32 - 10))) as Func, ), from_range( 'A'..='F', - (|state, c, _| state.push(c as u32 - ('A' as u32 - 10))) as Func, + (|_, state, c, _| state.push(c as u32 - ('A' as u32 - 10))) as Func, ), ] .cast(), @@ -590,22 +589,22 @@ fn create_unicode_char_transactions() -> TransitionMap(manager: M) -> TransitionMap { - type Func = fn(s: Sign, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_zero_transactions() -> TransitionMap { + type Func = fn(m: M, s: Sign, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_one( '.', - (|s, _, _| { + (|manager, s, _, _| { ( default(), TokenizerState::ParseFracBegin(zero(manager)), ) - }) as Func, + }) as Func, ), - create_range_map(set(['e', 'E']), |s, _, _| { + create_range_map(set(['e', 'E']), |manager, s, _, _| { ( default(), TokenizerState::ParseExpBegin(ExpState { @@ -616,14 +615,14 @@ fn create_zero_transactions(manager: M) -> TransitionMap + }) as Func ), - create_range_map(terminal_for_number(), |_, c, maps| { + create_range_map(terminal_for_number(), |_, _, c, maps| { transfer_state( [JsonToken::Number(default())].cast(), TokenizerState::Initial, @@ -637,25 +636,25 @@ fn create_zero_transactions(manager: M) -> TransitionMap(manager: M) -> TransitionMap, M::Dealloc> { - type Func = - fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_int_transactions() -> TransitionMap, M> { + type Func = + fn(m: M, s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseInt(s.add_digit(manager, c)))) as Func, + (|manager, s, c, _| (default(), TokenizerState::ParseInt(s.add_digit(manager, c)))) as Func, ), - from_one('.', |s, _, _| { + from_one('.', |_, s, _, _| { (default(), TokenizerState::ParseFracBegin(s)) }), - create_range_map(set(['e', 'E']), |s, _, _| { + create_range_map(set(['e', 'E']), |_, s, _, _| { (default(), TokenizerState::ParseExpBegin(s.into_exp_state())) }), - from_one('n', |s, _, _| (default(), TokenizerState::ParseBigInt(s))), - create_range_map(terminal_for_number(), |s, c, maps| { + from_one('n', |_, s, _, _| (default(), TokenizerState::ParseBigInt(s))), + create_range_map(terminal_for_number(), |_, s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) }), ] @@ -664,38 +663,38 @@ fn create_int_transactions(manager: M) -> TransitionMap() -> TransitionMap, D> { - type Func = - fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +fn create_frac_begin_transactions() -> TransitionMap, M> { + type Func = + fn(m: M, s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: from_range( '0'..='9', - (|s, c, _| { + (|manager, s, c, _| { ( default(), - TokenizerState::ParseFrac(s.into_float_state().add_digit(c)), + TokenizerState::ParseFrac(s.into_float_state().add_digit(manager, c)), ) }) as Func, ), } } -fn create_frac_transactions(manager: M) -> TransitionMap, M::Dealloc> { - type Func = - fn(s: FloatState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_frac_transactions() -> TransitionMap, M> { + type Func = + fn(m: M, s: FloatState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseFrac(s.add_digit(manager, c)))) as Func, + (|manager, s, c, _| (default(), TokenizerState::ParseFrac(s.add_digit(manager, c)))) as Func, ), - create_range_map(set(['e', 'E']), |s, _, _| { + create_range_map(set(['e', 'E']), |_, s, _, _| { (default(), TokenizerState::ParseExpBegin(s.into_exp_state())) }), - create_range_map(terminal_for_number(), |s, c, maps| { + create_range_map(terminal_for_number(), |_, s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) }), ] @@ -704,16 +703,16 @@ fn create_frac_transactions(manager: M) -> TransitionMap(manager: M) -> TransitionMap<(), M::Dealloc> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_minus_transactions() -> TransitionMap<(), M> { + type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge( from_one( '0', - (|_, _, _| (default(), TokenizerState::ParseZero(Sign::Negative))) as Func, + (|_, _, _, _| (default(), TokenizerState::ParseZero(Sign::Negative))) as Func, ), - from_range('1'..='9', |_, c, _| { + from_range('1'..='9', |manager, _, c, _| { ( default(), TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Negative, c)), @@ -723,24 +722,24 @@ fn create_minus_transactions(manager: M) -> TransitionMap<(), M::Dea } } -fn create_exp_begin_transactions() -> TransitionMap, D> { - type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_exp_begin_transactions() -> TransitionMap, M> { + type Func = fn(m: M, s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, + (|_, s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, ), - from_one('+', |s, _, _| (default(), TokenizerState::ParseExpSign(s))), - from_one('-', |mut s, _, _| { + from_one('+', |_, s, _, _| (default(), TokenizerState::ParseExpSign(s))), + from_one('-', |_, mut s, _, _| { (default(), { s.es = Sign::Negative; TokenizerState::ParseExpSign(s) }) }), - create_range_map(terminal_for_number(), |s, c, maps| { + create_range_map(terminal_for_number(), |_, s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) }), ] @@ -749,34 +748,34 @@ fn create_exp_begin_transactions() -> TransitionMap, D> } } -fn create_exp_transactions() -> TransitionMap, D> { - type Func = fn(s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_exp_transactions() -> TransitionMap, M> { + type Func = fn(m: M, s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge( from_range( '0'..='9', - (|s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, + (|_, s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, ), - create_range_map(terminal_for_number(), |s, c, maps| { + create_range_map(terminal_for_number(), |_, s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) }), ), } } -fn create_big_int_transactions() -> TransitionMap, D> { - type Func = - fn(s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_big_int_transactions() -> TransitionMap, M> { + type Func = + fn(m: M, s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| tokenize_invalid_number(c, maps)) as Func, - rm: create_range_map(terminal_for_number(), |s, c, maps| { + def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + rm: create_range_map(terminal_for_number(), |_, s, c, maps| { transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) }), } } -fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { +fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { transfer_state( [JsonToken::ErrorToken(ErrorType::InvalidNumber)].cast(), TokenizerState::Initial, @@ -785,86 +784,86 @@ fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Ve ) } -fn create_new_line_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_new_line_transactions() -> TransitionMap<(), M> { + type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, c, maps| { + def: (|_, _, c, maps| { transfer_state( [JsonToken::NewLine].cast(), TokenizerState::Initial, c, maps, ) - }) as Func, - rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _| { + }) as Func, + rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _, _| { (default(), TokenizerState::ParseNewLine) }), } } -fn create_comment_start_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_comment_start_transactions() -> TransitionMap<(), M> { + type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| { + def: (|_, _, _, _| { ( [JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)].cast(), TokenizerState::Initial, ) - }) as Func, + }) as Func, rm: merge( - from_one('/', |_, _, _| { + from_one('/', |_, _, _, _| { (default(), TokenizerState::ParseSinglelineComment) }), - from_one('*', |_, _, _| { + from_one('*', |_, _, _, _| { (default(), TokenizerState::ParseMultilineComment) }), ), } } -fn create_singleline_comment_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_singleline_comment_transactions() -> TransitionMap<(), M> { + type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func, - rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _| { + def: (|_, _, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func, + rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _, _| { (default(), TokenizerState::ParseNewLine) }), } } -fn create_multiline_comment_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_multiline_comment_transactions() -> TransitionMap<(), M> { + type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, - rm: from_one('*', |_, _, _| { + def: (|_, _, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, + rm: from_one('*', |_, _, _, _| { (default(), TokenizerState::ParseMultilineCommentAsterix) }), } } -fn create_multiline_comment_asterix_transactions() -> TransitionMap<(), D> { - type Func = fn(s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_multiline_comment_asterix_transactions() -> TransitionMap<(), M> { + type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { - def: (|_, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, + def: (|_, _, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, rm: merge( - from_one('/', |_, _, _| (default(), TokenizerState::Initial)), - from_one('*', |_, _, _| { + from_one('/', |_, _, _, _| (default(), TokenizerState::Initial)), + from_one('*', |_, _, _, _| { (default(), TokenizerState::ParseMultilineCommentAsterix) }), ), } } -fn create_operator_transactions() -> TransitionMap { +fn create_operator_transactions() -> TransitionMap { TransitionMap { - def: |s, c, maps| { + def: |_, s, c, maps| { let token = operator_to_token(s).unwrap(); transfer_state([token].cast(), TokenizerState::Initial, c, maps) }, - rm: create_range_map(operator_chars_with_dot(), |s, c, maps| { + rm: create_range_map(operator_chars_with_dot(), |_, s, c, maps| { let mut next_string = s.clone(); next_string.push(c); - match operator_to_token(next_string) { + match operator_to_token::(next_string) { Some(_) => { let mut next_string = s.clone(); next_string.push(c); @@ -888,7 +887,7 @@ pub struct TokenizerStateIterator, M: Manager> { chars: T, cache: VecDeque>, state: TokenizerState, - maps: TransitionMaps, + maps: TransitionMaps, end: bool, } From f5f363fd235d0d87b41e6bbf9eb5cf530a750ea0 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 10 Nov 2024 19:50:50 +0200 Subject: [PATCH 07/28] get_next_state --- nanvm-lib/src/tokenizer/mod.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index fdd38e9c..ae2d914e 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -63,8 +63,7 @@ pub enum TokenizerState { } impl TokenizerState -where - D: 'static { +where { fn push>(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { match self { TokenizerState::Initial => get_next_state((), c, &maps.initial, maps), @@ -97,7 +96,7 @@ where } } - pub fn push_mut(&mut self, c: char, tm: &TransitionMaps) -> Vec> { + pub fn push_mut>(&mut self, c: char, tm: &TransitionMaps) -> Vec> { let tokens; (tokens, *self) = take(self).push(c, tm); tokens @@ -411,19 +410,18 @@ pub fn create_transition_maps(manager: M) -> TransitionMaps { } } -fn get_next_state( +fn get_next_state( + manager: M, state: T, c: char, - tm: &TransitionMap, - maps: &TransitionMaps, -) -> (Vec>, TokenizerState) -where - T: 'static, D: 'static + tm: &TransitionMap, + maps: &TransitionMaps, +) -> (Vec>, TokenizerState) { let entry = tm.rm.get(c); match &entry.value { - Some(f) => f(state, c, maps), - None => (tm.def)(state, c, maps), + Some(f) => f(manager, state, c, maps), + None => (tm.def)(manager, state, c, maps), } } From c9f43b983637c75a0a7f763a981eab8e8fb0151a Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 10 Nov 2024 20:21:56 +0200 Subject: [PATCH 08/28] fix case with negative sign --- nanvm-lib/src/tokenizer/mod.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index ae2d914e..438c48c9 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -46,7 +46,7 @@ pub enum TokenizerState { ParseEscapeChar(String), ParseUnicodeChar(ParseUnicodeCharState), ParseMinus, - ParseZero(Sign), + ParseZero(js_bigint::Sign), ParseInt(JsBigintMutRef), ParseFracBegin(JsBigintMutRef), ParseFrac(FloatState), @@ -198,7 +198,7 @@ impl JsBigintMutRef { ExpState { b: self, fe: 0, - es: Sign::Positive, + es: js_bigint::Sign::Positive, e: 0, } } @@ -241,7 +241,7 @@ impl FloatState { ExpState { b: self.b, fe: self.fe, - es: Sign::Positive, + es: js_bigint::Sign::Positive, e: 0, } } @@ -258,7 +258,7 @@ impl FloatState { pub struct ExpState { b: JsBigintMutRef, fe: i64, - es: Sign, + es: js_bigint::Sign, e: i64, } @@ -271,8 +271,8 @@ impl ExpState { fn into_token(self) -> JsonToken { let exp = self.fe + match self.es { - Sign::Positive => self.e, - Sign::Negative => -self.e, + js_bigint::Sign::Positive => self.e, + js_bigint::Sign::Negative => -self.e, }; JsonToken::Number(bigfloat_to_f64(BigFloat { significand: self.b.to_old_bigint(), @@ -370,7 +370,7 @@ pub struct TransitionMaps { string: TransitionMap, escape_char: TransitionMap, unicode_char: TransitionMap, - zero: TransitionMap, + zero: TransitionMap, int: TransitionMap, M>, minus: TransitionMap<(), M>, frac_begin: TransitionMap, M>, @@ -587,8 +587,8 @@ fn create_unicode_char_transactions() -> TransitionMap() -> TransitionMap { - type Func = fn(m: M, s: Sign, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_zero_transactions() -> TransitionMap { + type Func = fn(m: M, s: js_bigint::Sign, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); TransitionMap { def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, rm: merge_list( @@ -598,7 +598,7 @@ fn create_zero_transactions() -> TransitionMap { (|manager, s, _, _| { ( default(), - TokenizerState::ParseFracBegin(zero(manager)), + TokenizerState::ParseFracBegin(from_u64(manager, s, 0)), ) }) as Func, ), @@ -606,9 +606,9 @@ fn create_zero_transactions() -> TransitionMap { ( default(), TokenizerState::ParseExpBegin(ExpState { - b: zero(manager), + b: from_u64(manager, s, 0), fe: 0, - es: Sign::Positive, + es: js_bigint::Sign::Positive, e: 0, }), ) @@ -616,7 +616,7 @@ fn create_zero_transactions() -> TransitionMap { from_one('n', (|manager, s, _, _| { ( default(), - TokenizerState::ParseBigInt(zero(manager)), + TokenizerState::ParseBigInt(from_u64(manager, s, 0)), ) }) as Func ), @@ -708,7 +708,7 @@ fn create_minus_transactions() -> TransitionMap<(), M> { rm: merge( from_one( '0', - (|_, _, _, _| (default(), TokenizerState::ParseZero(Sign::Negative))) as Func, + (|_, _, _, _| (default(), TokenizerState::ParseZero(js_bigint::Sign::Negative))) as Func, ), from_range('1'..='9', |manager, _, c, _| { ( @@ -733,7 +733,7 @@ fn create_exp_begin_transactions() -> TransitionMap Date: Sun, 10 Nov 2024 21:38:08 +0200 Subject: [PATCH 09/28] fix errors --- nanvm-lib/src/parser/analyzer.rs | 23 +- nanvm-lib/src/parser/any_state.rs | 2 +- nanvm-lib/src/parser/json_state.rs | 2 +- nanvm-lib/src/parser/parser.rs | 39 ++- nanvm-lib/src/parser/shared.rs | 4 +- nanvm-lib/src/tokenizer/mod.rs | 419 +++++++++++++++++++---------- 6 files changed, 309 insertions(+), 180 deletions(-) diff --git a/nanvm-lib/src/parser/analyzer.rs b/nanvm-lib/src/parser/analyzer.rs index 2779f62b..057d853c 100644 --- a/nanvm-lib/src/parser/analyzer.rs +++ b/nanvm-lib/src/parser/analyzer.rs @@ -1,7 +1,7 @@ use super::shared::DataType; use crate::ast::Module; use crate::common::default::default; -use crate::mem::manager::Dealloc; +use crate::mem::manager::{Dealloc, Manager}; use crate::tokenizer::{create_transition_maps, JsonToken, TokenizerState, TransitionMaps}; #[derive(Default)] @@ -21,17 +21,17 @@ pub struct AnalyzerResults { pub diagnostics: Vec, } -pub struct AnalyzerState { +pub struct AnalyzerState { parameters: AnalyzerParameters, - tokenizer_state: TokenizerState, - tokenizer_maps: TransitionMaps, + tokenizer_state: TokenizerState, + tokenizer_maps: TransitionMaps, diagnostics_len: usize, // TODO: add line number, column number tracking fields (needed for diagnostics). - module: Module, + module: Module, diagnostics: Vec, } -impl AnalyzerState { +impl AnalyzerState { /// Creates a new analyzer staring state. The caller should check `diagnostics` for errors /// immediately after creation (since `parameters` value can be inconsistent). pub fn new(parameters: AnalyzerParameters) -> Self { @@ -48,8 +48,11 @@ impl AnalyzerState { /// Updates analyzer state with a next input character; the result is the increment in the count /// of `diagnostics`. It's up to the caller to check what was added at the end of `diagnostics` /// - are there any fatal errors, from the point of view of the current parsing session? - pub fn push_mut(&mut self, c: char) -> usize { - for token in self.tokenizer_state.push_mut(c, &self.tokenizer_maps) { + pub fn push_mut(&mut self, manager: M, c: char) -> usize { + for token in self + .tokenizer_state + .push_mut(manager, c, &self.tokenizer_maps) + { self.process_token(token); } let prior_diagnostics_len = self.diagnostics_len; @@ -58,7 +61,7 @@ impl AnalyzerState { } /// Completes the analysis. - pub fn end(self) -> AnalyzerResults { + pub fn end(self) -> AnalyzerResults { // TODO: in case the current state is not a valid end state, add an error to self.diagnostics. AnalyzerResults { module: self.module, @@ -66,5 +69,5 @@ impl AnalyzerState { } } - fn process_token(&mut self, _token: JsonToken) {} + fn process_token(&mut self, _token: JsonToken) {} } diff --git a/nanvm-lib/src/parser/any_state.rs b/nanvm-lib/src/parser/any_state.rs index 3c8ab64d..cb62d43f 100644 --- a/nanvm-lib/src/parser/any_state.rs +++ b/nanvm-lib/src/parser/any_state.rs @@ -9,7 +9,7 @@ use super::{ use crate::{ common::{cast::Cast, default::default}, js::{any::Any, js_array::new_array, js_object::new_object}, - mem::manager::{Dealloc, Manager}, + mem::manager::Manager, tokenizer::JsonToken, }; use std::collections::BTreeMap; diff --git a/nanvm-lib/src/parser/json_state.rs b/nanvm-lib/src/parser/json_state.rs index a0134802..e9decbbb 100644 --- a/nanvm-lib/src/parser/json_state.rs +++ b/nanvm-lib/src/parser/json_state.rs @@ -25,7 +25,7 @@ impl JsonState { /*json_state:*/ JsonState, /*import_path:*/ Option, ) { - if token == JsonToken::NewLine { + if let JsonToken::NewLine = token { return match self { JsonState::ParseRoot(state) => { state.parse(manager, token, module_cache, context_path) diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index b8bb3931..16b0656c 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -36,10 +36,10 @@ impl<'a, M: Manager, I: Io> Context<'a, M, I> { } } -fn root_state_parse( +fn root_state_parse( mut root_state: RootState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> JsonState { match root_state.status { RootStatus::Initial => match token { @@ -167,7 +167,7 @@ fn root_state_parse( let read_result = context.io.read_to_string(current_path.as_str()); match read_result { Ok(s) => { - let tokens = tokenize(s); + let tokens = tokenize(context.manager, s); let res = parse_with_tokens(context, tokens.into_iter()); match res { Ok(r) => { @@ -194,7 +194,7 @@ fn root_state_parse( } } -fn const_state_parse( +fn const_state_parse( const_state: ConstState, context: &mut Context, token: JsonToken, @@ -222,7 +222,7 @@ fn const_state_parse( } } -fn any_state_parse_for_module( +fn any_state_parse_for_module( any_state: AnyState, context: &mut Context, token: JsonToken, @@ -238,7 +238,7 @@ fn any_state_parse_for_module( } } -fn any_state_parse_import_value( +fn any_state_parse_import_value( any_state: AnyState, context: &mut Context, token: JsonToken, @@ -260,7 +260,7 @@ fn any_state_parse_import_value( let read_result = context.io.read_to_string(current_path.as_str()); match read_result { Ok(s) => { - let tokens = tokenize(s); + let tokens = tokenize(context.manager, s); let res = parse_with_tokens(context, tokens.into_iter()); match res { Ok(r) => { @@ -285,7 +285,7 @@ fn any_state_parse_import_value( } } -fn any_state_parse( +fn any_state_parse( any_state: AnyState, context: &mut Context, token: JsonToken, @@ -307,20 +307,16 @@ fn any_state_parse( } } -fn json_state_push( +fn json_state_push( json_state: JsonState, context: &mut Context, token: JsonToken, ) -> JsonState { - match token { - JsonToken::NewLine => - { - return match json_state { - JsonState::ParseRoot(state) => root_state_parse(state, context, token), - _ => json_state, - } - }, - _ => {} + if let JsonToken::NewLine = token { + return match json_state { + JsonState::ParseRoot(state) => root_state_parse(state, context, token), + _ => json_state, + }; } match json_state { JsonState::ParseRoot(state) => root_state_parse(state, context, token), @@ -331,21 +327,21 @@ fn json_state_push( } } -pub fn parse( +pub fn parse( context: &mut Context, ) -> Result, ParseError> { context.module_cache.progress.insert(context.path.clone()); let read_result = context.io.read_to_string(context.path.as_str()); match read_result { Ok(s) => { - let tokens = tokenize(s); + let tokens = tokenize(context.manager, s); parse_with_tokens(context, tokens.into_iter()) } Err(_) => Err(ParseError::CannotReadFile), } } -pub fn parse_with_tokens( +pub fn parse_with_tokens( context: &mut Context, iter: impl Iterator>, ) -> Result, ParseError> { @@ -949,7 +945,6 @@ pub fn parse_with_tokens( // assert_eq!(result.header_len(), 1); // let items = result.items(); // assert_eq!(items, [0x1]); - // let tokens = [JsonToken::BigInt(new_bigint(manager, js::js_bigint::Sign::Negative, [2, 3]))]; // let result = parse_with_virtual_io(manager, tokens.into_iter()); diff --git a/nanvm-lib/src/parser/shared.rs b/nanvm-lib/src/parser/shared.rs index d49f787b..d42d2d57 100644 --- a/nanvm-lib/src/parser/shared.rs +++ b/nanvm-lib/src/parser/shared.rs @@ -153,7 +153,7 @@ fn try_id_to_any( } impl JsonToken { - pub fn try_to_any( + pub fn try_to_any>( self, manager: M, consts: &BTreeMap>, @@ -162,7 +162,7 @@ impl JsonToken { JsonToken::Number(f) => Some(Any::move_from(f)), JsonToken::String(s) => Some(Any::move_from(to_js_string(manager, s))), JsonToken::Id(s) => try_id_to_any(&s, manager, consts), - JsonToken::BigInt(b) => Some(Any::move_from(to_js_bigint(manager, b))), + JsonToken::BigInt(b) => Some(Any::move_from(b.to_ref())), _ => None, } } diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 438c48c9..3c6af811 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -1,9 +1,15 @@ -use std::{collections::VecDeque, mem::take, ops::{Deref, RangeInclusive}}; +use std::{ + collections::VecDeque, + mem::take, + ops::{Deref, RangeInclusive}, +}; use crate::{ - big_numbers::{ - self, big_float::BigFloat, big_int::{BigInt, Sign}, big_uint::BigUint - }, common::{cast::Cast, default::default}, js::js_bigint::{self, add, from_u64, zero, JsBigintMutRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State} + big_numbers::{self, big_float::BigFloat, big_int::BigInt, big_uint::BigUint}, + common::{cast::Cast, default::default}, + js::js_bigint::{self, add, from_u64, JsBigintMutRef}, + mem::manager::{Dealloc, Manager}, + range_map::{from_one, from_range, merge, merge_list, RangeMap, State}, }; #[derive(Debug)] @@ -62,43 +68,62 @@ pub enum TokenizerState { ParseOperator(String), } -impl TokenizerState -where { - fn push>(self, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { +impl TokenizerState { + fn push + 'static>( + self, + manager: M, + c: char, + maps: &TransitionMaps, + ) -> (Vec>, TokenizerState) { match self { - TokenizerState::Initial => get_next_state((), c, &maps.initial, maps), - TokenizerState::ParseId(s) => get_next_state(s, c, &maps.id, maps), - TokenizerState::ParseString(s) => get_next_state(s, c, &maps.string, maps), - TokenizerState::ParseEscapeChar(s) => get_next_state(s, c, &maps.escape_char, maps), - TokenizerState::ParseUnicodeChar(s) => get_next_state(s, c, &maps.unicode_char, maps), - TokenizerState::ParseZero(s) => get_next_state(s, c, &maps.zero, maps), - TokenizerState::ParseInt(s) => get_next_state(s, c, &maps.int, maps), - TokenizerState::ParseMinus => get_next_state((), c, &maps.minus, maps), - TokenizerState::ParseFracBegin(s) => get_next_state(s, c, &maps.frac_begin, maps), - TokenizerState::ParseFrac(s) => get_next_state(s, c, &maps.frac, maps), - TokenizerState::ParseExpBegin(s) => get_next_state(s, c, &maps.exp_begin, maps), + TokenizerState::Initial => get_next_state(manager, (), c, &maps.initial, maps), + TokenizerState::ParseId(s) => get_next_state(manager, s, c, &maps.id, maps), + TokenizerState::ParseString(s) => get_next_state(manager, s, c, &maps.string, maps), + TokenizerState::ParseEscapeChar(s) => { + get_next_state(manager, s, c, &maps.escape_char, maps) + } + TokenizerState::ParseUnicodeChar(s) => { + get_next_state(manager, s, c, &maps.unicode_char, maps) + } + TokenizerState::ParseZero(s) => get_next_state(manager, s, c, &maps.zero, maps), + TokenizerState::ParseInt(s) => get_next_state(manager, s, c, &maps.int, maps), + TokenizerState::ParseMinus => get_next_state(manager, (), c, &maps.minus, maps), + TokenizerState::ParseFracBegin(s) => { + get_next_state(manager, s, c, &maps.frac_begin, maps) + } + TokenizerState::ParseFrac(s) => get_next_state(manager, s, c, &maps.frac, maps), + TokenizerState::ParseExpBegin(s) => { + get_next_state(manager, s, c, &maps.exp_begin, maps) + } TokenizerState::ParseExpSign(s) | TokenizerState::ParseExp(s) => { - get_next_state(s, c, &maps.exp, maps) + get_next_state(manager, s, c, &maps.exp, maps) + } + TokenizerState::ParseBigInt(s) => get_next_state(manager, s, c, &maps.big_int, maps), + TokenizerState::ParseNewLine => get_next_state(manager, (), c, &maps.new_line, maps), + TokenizerState::ParseCommentStart => { + get_next_state(manager, (), c, &maps.comment_start, maps) } - TokenizerState::ParseBigInt(s) => get_next_state(s, c, &maps.big_int, maps), - TokenizerState::ParseNewLine => get_next_state((), c, &maps.new_line, maps), - TokenizerState::ParseCommentStart => get_next_state((), c, &maps.comment_start, maps), TokenizerState::ParseSinglelineComment => { - get_next_state((), c, &maps.singleline_comment, maps) + get_next_state(manager, (), c, &maps.singleline_comment, maps) } TokenizerState::ParseMultilineComment => { - get_next_state((), c, &maps.multiline_comment, maps) + get_next_state(manager, (), c, &maps.multiline_comment, maps) } TokenizerState::ParseMultilineCommentAsterix => { - get_next_state((), c, &maps.multiline_comment_asterix, maps) + get_next_state(manager, (), c, &maps.multiline_comment_asterix, maps) } - TokenizerState::ParseOperator(s) => get_next_state(s, c, &maps.operator, maps), + TokenizerState::ParseOperator(s) => get_next_state(manager, s, c, &maps.operator, maps), } } - pub fn push_mut>(&mut self, c: char, tm: &TransitionMaps) -> Vec> { + pub fn push_mut + 'static>( + &mut self, + manager: M, + c: char, + tm: &TransitionMaps, + ) -> Vec> { let tokens; - (tokens, *self) = take(self).push(c, tm); + (tokens, *self) = take(self).push(manager, c, tm); tokens } @@ -143,13 +168,16 @@ pub struct ParseUnicodeCharState { } impl ParseUnicodeCharState { - fn push(mut self, i: u32) -> (Vec>, TokenizerState) { + fn push( + mut self, + i: u32, + ) -> (Vec>, TokenizerState) { let new_unicode = self.unicode | (i << ((3 - self.index) * 4)); match self.index { 3 => { let c = char::from_u32(new_unicode); match c { - Some(c) => continue_string_state(self.s, c), + Some(c) => continue_string_state::(self.s, c), None => ( [JsonToken::ErrorToken(ErrorType::InvalidHex)].cast(), TokenizerState::Initial, @@ -179,19 +207,24 @@ impl BigUint { } impl JsBigintMutRef { - fn from_digit>(m: M, sign: js_bigint::Sign, c: char) -> JsBigintMutRef { + fn from_digit>( + m: M, + sign: js_bigint::Sign, + c: char, + ) -> JsBigintMutRef { from_u64(m, sign, digit_to_number(c)) } fn add_digit>(self, m: M, c: char) -> JsBigintMutRef { - add(m, self.deref(), Self::from_digit(m, js_bigint::Sign::Positive, c).deref()) + add( + m, + self.deref(), + Self::from_digit(m, js_bigint::Sign::Positive, c).deref(), + ) } fn into_float_state(self) -> FloatState { - FloatState { - b: self, - fe: 0, - } + FloatState { b: self, fe: 0 } } fn into_exp_state(self) -> ExpState { @@ -203,18 +236,23 @@ impl JsBigintMutRef { } } - fn to_old_bigint(self) -> BigInt { + fn into_old_bigint(self) -> BigInt { let deref = self.deref(); let sign = match deref.sign() { js_bigint::Sign::Positive => big_numbers::big_int::Sign::Positive, js_bigint::Sign::Negative => big_numbers::big_int::Sign::Negative, }; - BigInt { sign, value: BigUint { value: deref.items().to_vec() }} + BigInt { + sign, + value: BigUint { + value: deref.items().to_vec(), + }, + } } - fn into_token(self) -> JsonToken { + fn into_token(self) -> JsonToken { JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: self.to_old_bigint(), + significand: self.into_old_bigint(), exp: 0, non_zero_reminder: false, })) @@ -225,7 +263,7 @@ impl JsBigintMutRef { } } -pub struct FloatState { +pub struct FloatState { b: JsBigintMutRef, fe: i64, } @@ -248,7 +286,7 @@ impl FloatState { fn into_token(self) -> JsonToken { JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: self.b.to_old_bigint(), + significand: self.b.into_old_bigint(), exp: self.fe, non_zero_reminder: false, })) @@ -275,7 +313,7 @@ impl ExpState { js_bigint::Sign::Negative => -self.e, }; JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: self.b.to_old_bigint(), + significand: self.b.into_old_bigint(), exp, non_zero_reminder: false, })) @@ -356,8 +394,7 @@ fn set(arr: impl IntoIterator) -> Vec> { result } -type Transition = - fn(manager: M, state: T, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +type Transition = TransitionFunc; struct TransitionMap { def: Transition, @@ -386,7 +423,7 @@ pub struct TransitionMaps { operator: TransitionMap, } -pub fn create_transition_maps(manager: M) -> TransitionMaps { +pub fn create_transition_maps() -> TransitionMaps { TransitionMaps { initial: create_initial_transitions(), id: create_id_transitions(), @@ -410,14 +447,13 @@ pub fn create_transition_maps(manager: M) -> TransitionMaps { } } -fn get_next_state( +fn get_next_state( manager: M, state: T, c: char, tm: &TransitionMap, maps: &TransitionMaps, -) -> (Vec>, TokenizerState) -{ +) -> (Vec>, TokenizerState) { let entry = tm.rm.get(c); match &entry.value { Some(f) => f(manager, state, c, maps), @@ -426,7 +462,7 @@ fn get_next_state( } fn create_initial_transitions() -> TransitionMap<(), M> { - type Func = fn(manager: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); + type Func = TransitionFunc; TransitionMap { def: (|_, _, _, _| { ( @@ -442,15 +478,21 @@ fn create_initial_transitions() -> TransitionMap<(), M> { from_range('1'..='9', |manager: M, _, c, _| { ( default(), - TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Positive, c)), + TokenizerState::ParseInt(start_number( + manager, + js_bigint::Sign::Positive, + c, + )), ) - } - ), + }), from_one('"', |_, _, _, _| { (default(), TokenizerState::ParseString(String::default())) }), from_one('0', |_, _, _, _| { - (default(), TokenizerState::ParseZero(Sign::Positive)) + ( + default(), + TokenizerState::ParseZero(js_bigint::Sign::Positive), + ) }), from_one('-', |_, _, _, _| (default(), TokenizerState::ParseMinus)), create_range_map(id_start(), |_, _, c, _| { @@ -469,10 +511,16 @@ fn create_initial_transitions() -> TransitionMap<(), M> { } } -fn create_id_transitions() -> TransitionMap { +fn create_id_transitions() -> TransitionMap { TransitionMap { - def: |_, s, c, maps| { - transfer_state([JsonToken::Id(s)].cast(), TokenizerState::Initial, c, maps) + def: |manager, s, c, maps| { + transfer_state( + manager, + [JsonToken::Id(s)].cast(), + TokenizerState::Initial, + c, + maps, + ) }, rm: create_range_map(id_char(), |_, mut s, c, _| { s.push(c); @@ -498,27 +546,32 @@ fn create_string_transactions() -> TransitionMap { } } -fn continue_string_state(mut s: String, c: char) -> (Vec>, TokenizerState) { +fn continue_string_state( + mut s: String, + c: char, +) -> (Vec>, TokenizerState) { s.push(c); (default(), TokenizerState::ParseString(s)) } -fn transfer_state( +fn transfer_state( + manager: M, mut vec: Vec>, mut state: TokenizerState, c: char, maps: &TransitionMaps, ) -> (Vec>, TokenizerState) { let next_tokens; - (next_tokens, state) = state.push(c, maps); + (next_tokens, state) = state.push(manager, c, maps); vec.extend(next_tokens); (vec, state) } -fn create_escape_char_transactions() -> TransitionMap { +fn create_escape_char_transactions() -> TransitionMap { TransitionMap { - def: |_, s, c, maps| { + def: |manager, s, c, maps| { transfer_state( + manager, [JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)].cast(), TokenizerState::ParseString(s), c, @@ -551,16 +604,13 @@ fn create_escape_char_transactions() -> TransitionMap { } } -fn create_unicode_char_transactions() -> TransitionMap { - type Func = fn( - m: M, - state: ParseUnicodeCharState, - c: char, - maps: &TransitionMaps, - ) -> (Vec>, TokenizerState); +fn create_unicode_char_transactions( +) -> TransitionMap { + type Func = TransitionFunc; TransitionMap { - def: |_, state, c, maps| { + def: |manager, state, c, maps| { transfer_state( + manager, [JsonToken::ErrorToken(ErrorType::InvalidHex)].cast(), TokenizerState::ParseString(state.s), c, @@ -571,15 +621,15 @@ fn create_unicode_char_transactions() -> TransitionMap, + (|_, state, c, _| state.push::(c as u32 - '0' as u32)) as Func, ), from_range( 'a'..='f', - (|_, state, c, _| state.push(c as u32 - ('a' as u32 - 10))) as Func, + (|_, state, c, _| state.push::(c as u32 - ('a' as u32 - 10))) as Func, ), from_range( 'A'..='F', - (|_, state, c, _| state.push(c as u32 - ('A' as u32 - 10))) as Func, + (|_, state, c, _| state.push::(c as u32 - ('A' as u32 - 10))) as Func, ), ] .cast(), @@ -587,10 +637,10 @@ fn create_unicode_char_transactions() -> TransitionMap() -> TransitionMap { - type Func = fn(m: M, s: js_bigint::Sign, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_zero_transactions() -> TransitionMap { + type Func = TransitionFunc; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge_list( [ from_one( @@ -613,15 +663,18 @@ fn create_zero_transactions() -> TransitionMap { }), ) }), - from_one('n', (|manager, s, _, _| { - ( - default(), - TokenizerState::ParseBigInt(from_u64(manager, s, 0)), - ) - }) as Func - ), - create_range_map(terminal_for_number(), |_, _, c, maps| { + from_one( + 'n', + (|manager, s, _, _| { + ( + default(), + TokenizerState::ParseBigInt(from_u64(manager, s, 0)), + ) + }) as Func, + ), + create_range_map(terminal_for_number(), |manager, _, c, maps| { transfer_state( + manager, [JsonToken::Number(default())].cast(), TokenizerState::Initial, c, @@ -634,16 +687,17 @@ fn create_zero_transactions() -> TransitionMap { } } -fn create_int_transactions() -> TransitionMap, M> { - type Func = - fn(m: M, s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_int_transactions() -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|manager, s, c, _| (default(), TokenizerState::ParseInt(s.add_digit(manager, c)))) as Func, + (|manager, s, c, _| { + (default(), TokenizerState::ParseInt(s.add_digit(manager, c))) + }) as Func, ), from_one('.', |_, s, _, _| { (default(), TokenizerState::ParseFracBegin(s)) @@ -651,9 +705,17 @@ fn create_int_transactions() -> TransitionMap() -> TransitionMap() -> TransitionMap, M> { - type Func = - fn(m: M, s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec, TokenizerState); +fn create_frac_begin_transactions( +) -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: from_range( '0'..='9', (|manager, s, c, _| { @@ -673,27 +735,37 @@ fn create_frac_begin_transactions() -> TransitionMap, ), } } -fn create_frac_transactions() -> TransitionMap, M> { - type Func = - fn(m: M, s: FloatState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_frac_transactions() -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', - (|manager, s, c, _| (default(), TokenizerState::ParseFrac(s.add_digit(manager, c)))) as Func, + (|manager, s, c, _| { + ( + default(), + TokenizerState::ParseFrac(s.add_digit(manager, c)), + ) + }) as Func, ), create_range_map(set(['e', 'E']), |_, s, _, _| { (default(), TokenizerState::ParseExpBegin(s.into_exp_state())) }), - create_range_map(terminal_for_number(), |_, s, c, maps| { - transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) + create_range_map(terminal_for_number(), |manager, s, c, maps| { + transfer_state( + manager, + [s.into_token()].cast(), + TokenizerState::Initial, + c, + maps, + ) }), ] .cast(), @@ -701,44 +773,57 @@ fn create_frac_transactions() -> TransitionMap() -> TransitionMap<(), M> { - type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_minus_transactions() -> TransitionMap<(), M> { + type Func = TransitionFunc; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge( from_one( '0', - (|_, _, _, _| (default(), TokenizerState::ParseZero(js_bigint::Sign::Negative))) as Func, + (|_, _, _, _| { + ( + default(), + TokenizerState::ParseZero(js_bigint::Sign::Negative), + ) + }) as Func, ), from_range('1'..='9', |manager, _, c, _| { ( default(), - TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Negative, c)), + TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Negative, c)), ) }), ), } } -fn create_exp_begin_transactions() -> TransitionMap, M> { - type Func = fn(m: M, s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_exp_begin_transactions() -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge_list( [ from_range( '0'..='9', (|_, s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, ), - from_one('+', |_, s, _, _| (default(), TokenizerState::ParseExpSign(s))), + from_one('+', |_, s, _, _| { + (default(), TokenizerState::ParseExpSign(s)) + }), from_one('-', |_, mut s, _, _| { (default(), { s.es = js_bigint::Sign::Negative; TokenizerState::ParseExpSign(s) }) }), - create_range_map(terminal_for_number(), |_, s, c, maps| { - transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) + create_range_map(terminal_for_number(), |manager, s, c, maps| { + transfer_state( + manager, + [s.into_token()].cast(), + TokenizerState::Initial, + c, + maps, + ) }), ] .cast(), @@ -746,35 +831,52 @@ fn create_exp_begin_transactions() -> TransitionMap() -> TransitionMap, M> { - type Func = fn(m: M, s: ExpState, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_exp_transactions() -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge( from_range( '0'..='9', (|_, s, c, _| (default(), TokenizerState::ParseExp(s.add_digit(c)))) as Func, ), - create_range_map(terminal_for_number(), |_, s, c, maps| { - transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) + create_range_map(terminal_for_number(), |manager, s, c, maps| { + transfer_state( + manager, + [s.into_token()].cast(), + TokenizerState::Initial, + c, + maps, + ) }), ), } } -fn create_big_int_transactions() -> TransitionMap, M> { - type Func = - fn(m: M, s: JsBigintMutRef, c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_big_int_transactions( +) -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { - def: (|_, _, c, maps| tokenize_invalid_number(c, maps)) as Func, - rm: create_range_map(terminal_for_number(), |_, s, c, maps| { - transfer_state([s.into_token()].cast(), TokenizerState::Initial, c, maps) + def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, + rm: create_range_map(terminal_for_number(), |manager, s, c, maps| { + transfer_state( + manager, + [s.into_token()].cast(), + TokenizerState::Initial, + c, + maps, + ) }), } } -fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState) { +fn tokenize_invalid_number( + manager: M, + c: char, + maps: &TransitionMaps, +) -> (Vec>, TokenizerState) { transfer_state( + manager, [JsonToken::ErrorToken(ErrorType::InvalidNumber)].cast(), TokenizerState::Initial, c, @@ -782,11 +884,12 @@ fn tokenize_invalid_number(c: char, maps: &TransitionMaps) -> (Ve ) } -fn create_new_line_transactions() -> TransitionMap<(), M> { - type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); +fn create_new_line_transactions() -> TransitionMap<(), M> { + type Func = TransitionFunc; TransitionMap { - def: (|_, _, c, maps| { + def: (|manager, _, c, maps| { transfer_state( + manager, [JsonToken::NewLine].cast(), TokenizerState::Initial, c, @@ -800,7 +903,7 @@ fn create_new_line_transactions() -> TransitionMap<(), M> { } fn create_comment_start_transactions() -> TransitionMap<(), M> { - type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); + type Func = TransitionFunc; TransitionMap { def: (|_, _, _, _| { ( @@ -819,8 +922,18 @@ fn create_comment_start_transactions() -> TransitionMap<(), M> { } } +type TransitionFunc = fn( + m: M, + s: S, + c: char, + maps: &TransitionMaps, +) -> ( + Vec::Dealloc>>, + TokenizerState<::Dealloc>, +); + fn create_singleline_comment_transactions() -> TransitionMap<(), M> { - type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); + type Func = TransitionFunc; TransitionMap { def: (|_, _, _, _| (default(), TokenizerState::ParseSinglelineComment)) as Func, rm: create_range_map(set(WHITE_SPACE_CHARS), |_, _, _, _| { @@ -830,7 +943,15 @@ fn create_singleline_comment_transactions() -> TransitionMap<(), M> } fn create_multiline_comment_transactions() -> TransitionMap<(), M> { - type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); + type Func = fn( + m: M, + s: (), + c: char, + maps: &TransitionMaps, + ) -> ( + Vec::Dealloc>>, + TokenizerState<::Dealloc>, + ); TransitionMap { def: (|_, _, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, rm: from_one('*', |_, _, _, _| { @@ -840,7 +961,15 @@ fn create_multiline_comment_transactions() -> TransitionMap<(), M> { } fn create_multiline_comment_asterix_transactions() -> TransitionMap<(), M> { - type Func = fn(m: M, s: (), c: char, maps: &TransitionMaps) -> (Vec>, TokenizerState); + type Func = fn( + m: M, + s: (), + c: char, + maps: &TransitionMaps, + ) -> ( + Vec::Dealloc>>, + TokenizerState<::Dealloc>, + ); TransitionMap { def: (|_, _, _, _| (default(), TokenizerState::ParseMultilineComment)) as Func, rm: merge( @@ -852,13 +981,13 @@ fn create_multiline_comment_asterix_transactions() -> TransitionMap< } } -fn create_operator_transactions() -> TransitionMap { +fn create_operator_transactions() -> TransitionMap { TransitionMap { - def: |_, s, c, maps| { + def: |manager, s, c, maps| { let token = operator_to_token(s).unwrap(); - transfer_state([token].cast(), TokenizerState::Initial, c, maps) + transfer_state(manager, [token].cast(), TokenizerState::Initial, c, maps) }, - rm: create_range_map(operator_chars_with_dot(), |_, s, c, maps| { + rm: create_range_map(operator_chars_with_dot(), |manager, s, c, maps| { let mut next_string = s.clone(); next_string.push(c); match operator_to_token::(next_string) { @@ -869,14 +998,14 @@ fn create_operator_transactions() -> TransitionMap { } _ => { let token = operator_to_token(s).unwrap(); - transfer_state([token].cast(), TokenizerState::Initial, c, maps) + transfer_state(manager, [token].cast(), TokenizerState::Initial, c, maps) } } }), } } -pub fn tokenize(manager: M, input: String) -> Vec> { +pub fn tokenize(manager: M, input: String) -> Vec> { TokenizerStateIterator::new(manager, input.chars()).collect() } @@ -889,20 +1018,20 @@ pub struct TokenizerStateIterator, M: Manager> { end: bool, } -impl, M: Manager> TokenizerStateIterator { +impl, M: Manager + 'static> TokenizerStateIterator { pub fn new(manager: M, chars: T) -> Self { Self { manager, chars, cache: default(), state: default(), - maps: create_transition_maps(manager), + maps: create_transition_maps(), end: false, } } } -impl, M: Manager> Iterator for TokenizerStateIterator { +impl, M: Manager + 'static> Iterator for TokenizerStateIterator { type Item = JsonToken; fn next(&mut self) -> Option { @@ -914,7 +1043,9 @@ impl, M: Manager> Iterator for TokenizerStateIterator self.cache.extend(self.state.push_mut(c, &self.maps)), + Some(c) => self + .cache + .extend(self.state.push_mut(self.manager, c, &self.maps)), None => { self.end = true; self.cache.extend(take(&mut self.state).end()) From 59db0c1bc683172fa5775b5652c73285666a1673 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 16 Nov 2024 21:19:03 +0200 Subject: [PATCH 10/28] uncomment some tests --- nanvm-lib/src/tokenizer/mod.rs | 1002 ++++++++++++++++---------------- 1 file changed, 501 insertions(+), 501 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 3c6af811..7a98d409 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -1055,504 +1055,504 @@ impl, M: Manager + 'static> Iterator for TokenizerState } } -// #[cfg(test)] -// mod test { -// use wasm_bindgen_test::wasm_bindgen_test; - -// use crate::{ -// big_numbers::{ -// big_float::BigFloat, -// big_int::{BigInt, Sign}, -// big_uint::BigUint, -// }, common::cast::Cast, mem::local::Local, tokenizer::bigfloat_to_f64 -// }; - -// use super::{tokenize, ErrorType, JsonToken}; - -// #[test] -// #[wasm_bindgen_test] -// fn test_empty() { -// let local = Local::default(); -// let result = tokenize(&local, String::from("")); -// assert_eq!(result.len(), 0); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_ops() { -// let local = Local::default(); - -// let result = tokenize(&local, String::from("{")); -// assert_eq!(&result, &[JsonToken::<&Local>::ObjectBegin]); - -// let result = tokenize(String::from("}")); -// assert_eq!(&result, &[JsonToken::ObjectEnd]); - -// let result = tokenize(String::from("[")); -// assert_eq!(&result, &[JsonToken::ArrayBegin]); - -// let result = tokenize(String::from("]")); -// assert_eq!(&result, &[JsonToken::ArrayEnd]); - -// let result = tokenize(String::from(":")); -// assert_eq!(&result, &[JsonToken::Colon]); - -// let result = tokenize(String::from(",")); -// assert_eq!(&result, &[JsonToken::Comma]); - -// let result = tokenize(String::from("=")); -// assert_eq!(&result, &[JsonToken::Equals]); - -// let result = tokenize(String::from(".")); -// assert_eq!(&result, &[JsonToken::Dot]); - -// let result = tokenize(String::from(";")); -// assert_eq!(&result, &[JsonToken::Semicolon]); - -// let result = tokenize(String::from("()")); -// assert_eq!( -// &result, -// &[JsonToken::OpeningParenthesis, JsonToken::ClosingParenthesis] -// ); - -// let result = tokenize(String::from("[{ :, }]")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ArrayBegin, -// JsonToken::ObjectBegin, -// JsonToken::Colon, -// JsonToken::Comma, -// JsonToken::ObjectEnd, -// JsonToken::ArrayEnd -// ] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_id() { -// let result = tokenize(String::from("true")); -// assert_eq!(&result, &[JsonToken::Id(String::from("true"))]); - -// let result = tokenize(String::from("false")); -// assert_eq!(&result, &[JsonToken::Id(String::from("false"))]); - -// let result = tokenize(String::from("null")); -// assert_eq!(&result, &[JsonToken::Id(String::from("null"))]); - -// let result = tokenize(String::from("tru tru")); -// assert_eq!( -// &result, -// &[ -// JsonToken::Id(String::from("tru")), -// JsonToken::Id(String::from("tru")), -// ] -// ); - -// let result = tokenize(String::from("ABCxyz_0123456789$")); -// assert_eq!( -// &result, -// &[JsonToken::Id(String::from("ABCxyz_0123456789$")),] -// ); - -// let result = tokenize(String::from("_")); -// assert_eq!(&result, &[JsonToken::Id(String::from("_")),]); - -// let result = tokenize(String::from("$")); -// assert_eq!(&result, &[JsonToken::Id(String::from("$")),]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_whitespace() { -// let result = tokenize(String::from(" \t\n\r")); -// assert_eq!(&result, &[]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_string() { -// let result = tokenize(String::from("\"\"")); -// assert_eq!(&result, &[JsonToken::String("".to_string())]); - -// let result = tokenize(String::from("\"value\"")); -// assert_eq!(&result, &[JsonToken::String("value".to_string())]); - -// let result = tokenize(String::from("\"value1\" \"value2\"")); -// assert_eq!( -// &result, -// &[ -// JsonToken::String("value1".to_string()), -// JsonToken::String("value2".to_string()) -// ] -// ); - -// let result = tokenize(String::from("\"value")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_escaped_characters() { -// let result = tokenize(String::from("\"\\b\\f\\n\\r\\t\"")); -// assert_eq!( -// &result, -// &[JsonToken::String("\u{8}\u{c}\n\r\t".to_string())] -// ); - -// let result = tokenize(String::from("\"\\x\"")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), -// JsonToken::String("x".to_string()) -// ] -// ); - -// let result = tokenize(String::from("\"\\")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_unicode() { -// let result = tokenize(String::from("\"\\u1234\"")); -// assert_eq!(&result, &[JsonToken::String("ሴ".to_string())]); - -// let result = tokenize(String::from("\"\\uaBcDEeFf\"")); -// assert_eq!(&result, &[JsonToken::String("ꯍEeFf".to_string())]); - -// let result = tokenize(String::from("\"\\uEeFg\"")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidHex), -// JsonToken::String("g".to_string()) -// ] -// ); - -// let result = tokenize(String::from("\"\\uEeF")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_integer() { -// let result = tokenize(String::from("0")); -// assert_eq!(&result, &[JsonToken::Number(0.0)]); - -// let result = tokenize(String::from("-0")); -// assert_eq!(&result, &[JsonToken::Number(0.0)]); - -// let result = tokenize(String::from("0abc")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Id(String::from("abc")) -// ] -// ); - -// let result = tokenize(String::from("0. 2")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Number(2.0) -// ] -// ); - -// let result = tokenize(String::from("1234567890")); -// assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); - -// let result = tokenize(String::from("-1234567890")); -// assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); - -// let result = tokenize(String::from("[0,1]")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ArrayBegin, -// JsonToken::Number(0.0), -// JsonToken::Comma, -// JsonToken::Number(1.0), -// JsonToken::ArrayEnd -// ] -// ); - -// let result = tokenize(String::from("001")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Number(1.0), -// ] -// ); - -// let result = tokenize(String::from("-")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - -// let result = tokenize(String::from("-{}")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::ObjectBegin, -// JsonToken::ObjectEnd -// ] -// ); - -// let result = tokenize(String::from("9007199254740991")); -// assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); - -// let result = tokenize(String::from("9007199254740992")); -// assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); - -// let result = tokenize(String::from("9007199254740993")); -// assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_big_float() { -// let result = tokenize(String::from("340282366920938463463374607431768211456")); -// assert_eq!( -// &result, -// &[JsonToken::Number(bigfloat_to_f64(BigFloat { -// significand: BigInt { -// sign: Sign::Positive, -// value: BigUint { -// value: [0, 0, 1].cast() -// } -// }, -// exp: 0, -// non_zero_reminder: false -// }))] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_float() { -// let result = tokenize(String::from("0.01")); -// assert_eq!(&result, &[JsonToken::Number(0.01)]); - -// let result = tokenize(String::from("[-12.34]")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ArrayBegin, -// JsonToken::Number(-12.34), -// JsonToken::ArrayEnd -// ] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_infinity() { -// let result = tokenize(String::from("1e1000")); -// assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); - -// let result = tokenize(String::from("-1e+1000")); -// assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_exp() { -// let result = tokenize(String::from("1e2")); -// assert_eq!(&result, &[JsonToken::Number(1e2)]); - -// let result = tokenize(String::from("1E+2")); -// assert_eq!(&result, &[JsonToken::Number(1e2)]); - -// let result = tokenize(String::from("0e-2")); -// assert_eq!(&result, &[JsonToken::Number(0.0)]); - -// let result = tokenize(String::from("1e-2")); -// assert_eq!(&result, &[JsonToken::Number(1e-2)]); - -// let result = tokenize(String::from("1.2e+2")); -// assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); - -// let result = tokenize(String::from("12e0000")); -// assert_eq!(&result, &[JsonToken::Number(12.0)]); - -// let result = tokenize(String::from("1e")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - -// let result = tokenize(String::from("1e+")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - -// let result = tokenize(String::from("1e-")); -// assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_big_int() { -// let result = tokenize(String::from("0n")); -// assert_eq!(&result, &[JsonToken::BigInt(BigInt::ZERO)]); - -// let result = tokenize(String::from("-0n")); -// assert_eq!( -// &result, -// &[JsonToken::BigInt(BigInt { -// sign: Sign::Negative, -// value: BigUint::ZERO -// })] -// ); - -// let result = tokenize(String::from("1234567890n")); -// assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_u64(1234567890))]); - -// let result = tokenize(String::from("-1234567890n")); -// assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_i64(-1234567890))]); - -// let result = tokenize(String::from("123.456n")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Id(String::from("n")) -// ] -// ); - -// let result = tokenize(String::from("123e456n")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Id(String::from("n")) -// ] -// ); - -// let result = tokenize(String::from("1234567890na")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Id(String::from("a")) -// ] -// ); - -// let result = tokenize(String::from("1234567890nn")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ErrorToken(ErrorType::InvalidNumber), -// JsonToken::Id(String::from("n")) -// ] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_errors() { -// let result = tokenize(String::from("ᄑ")); -// assert_eq!( -// &result, -// &[JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_djs() { -// let result = tokenize(String::from("module.exports = ")); -// assert_eq!( -// &result, -// &[ -// JsonToken::Id(String::from("module")), -// JsonToken::Dot, -// JsonToken::Id(String::from("exports")), -// JsonToken::Equals, -// ] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_singleline_comments() { -// let result = tokenize(String::from("{//abc\n2\n}")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ObjectBegin, -// JsonToken::NewLine, -// JsonToken::Number(2.0), -// JsonToken::NewLine, -// JsonToken::ObjectEnd, -// ] -// ); - -// let result = tokenize(String::from("0//abc/*")); -// assert_eq!(&result, &[JsonToken::Number(0.0),]); - -// let result = tokenize(String::from("0//")); -// assert_eq!(&result, &[JsonToken::Number(0.0),]); - -// let result = tokenize(String::from("0/")); -// assert_eq!( -// &result, -// &[ -// JsonToken::Number(0.0), -// JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), -// ] -// ); - -// let result = tokenize(String::from("0/a")); -// assert_eq!( -// &result, -// &[ -// JsonToken::Number(0.0), -// JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), -// ] -// ); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_multiline_comments() { -// let result = tokenize(String::from("{/*abc\ndef*/2}")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ObjectBegin, -// JsonToken::Number(2.0), -// JsonToken::ObjectEnd, -// ] -// ); - -// let result = tokenize(String::from("{/*/* /**/2}")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ObjectBegin, -// JsonToken::Number(2.0), -// JsonToken::ObjectEnd, -// ] -// ); - -// let result = tokenize(String::from("{/*")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ObjectBegin, -// JsonToken::ErrorToken(ErrorType::CommentClosingExpected), -// ] -// ); - -// let result = tokenize(String::from("{/**")); -// assert_eq!( -// &result, -// &[ -// JsonToken::ObjectBegin, -// JsonToken::ErrorToken(ErrorType::CommentClosingExpected), -// ] -// ); -// } -// } +#[cfg(test)] +mod test { + use wasm_bindgen_test::wasm_bindgen_test; + + use crate::{ + big_numbers::{ + big_float::BigFloat, + big_int::{BigInt, Sign}, + big_uint::BigUint, + }, common::cast::Cast, mem::local::Local, tokenizer::bigfloat_to_f64 + }; + + use super::{tokenize, ErrorType, JsonToken}; + + #[test] + #[wasm_bindgen_test] + fn test_empty() { + let local = Local::default(); + let result = tokenize(&local, String::from("")); + assert_eq!(result.len(), 0); + } + + // #[test] + // #[wasm_bindgen_test] + // fn test_ops() { + // let local = Local::default(); + + // let result = tokenize(&local, String::from("{")); + // assert_eq!(&result, &[JsonToken::<&Local>::ObjectBegin]); + + // let result = tokenize(String::from("}")); + // assert_eq!(&result, &[JsonToken::ObjectEnd]); + + // let result = tokenize(String::from("[")); + // assert_eq!(&result, &[JsonToken::ArrayBegin]); + + // let result = tokenize(String::from("]")); + // assert_eq!(&result, &[JsonToken::ArrayEnd]); + + // let result = tokenize(String::from(":")); + // assert_eq!(&result, &[JsonToken::Colon]); + + // let result = tokenize(String::from(",")); + // assert_eq!(&result, &[JsonToken::Comma]); + + // let result = tokenize(String::from("=")); + // assert_eq!(&result, &[JsonToken::Equals]); + + // let result = tokenize(String::from(".")); + // assert_eq!(&result, &[JsonToken::Dot]); + + // let result = tokenize(String::from(";")); + // assert_eq!(&result, &[JsonToken::Semicolon]); + + // let result = tokenize(String::from("()")); + // assert_eq!( + // &result, + // &[JsonToken::OpeningParenthesis, JsonToken::ClosingParenthesis] + // ); + + // let result = tokenize(String::from("[{ :, }]")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ArrayBegin, + // JsonToken::ObjectBegin, + // JsonToken::Colon, + // JsonToken::Comma, + // JsonToken::ObjectEnd, + // JsonToken::ArrayEnd + // ] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_id() { + // let result = tokenize(String::from("true")); + // assert_eq!(&result, &[JsonToken::Id(String::from("true"))]); + + // let result = tokenize(String::from("false")); + // assert_eq!(&result, &[JsonToken::Id(String::from("false"))]); + + // let result = tokenize(String::from("null")); + // assert_eq!(&result, &[JsonToken::Id(String::from("null"))]); + + // let result = tokenize(String::from("tru tru")); + // assert_eq!( + // &result, + // &[ + // JsonToken::Id(String::from("tru")), + // JsonToken::Id(String::from("tru")), + // ] + // ); + + // let result = tokenize(String::from("ABCxyz_0123456789$")); + // assert_eq!( + // &result, + // &[JsonToken::Id(String::from("ABCxyz_0123456789$")),] + // ); + + // let result = tokenize(String::from("_")); + // assert_eq!(&result, &[JsonToken::Id(String::from("_")),]); + + // let result = tokenize(String::from("$")); + // assert_eq!(&result, &[JsonToken::Id(String::from("$")),]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_whitespace() { + // let result = tokenize(String::from(" \t\n\r")); + // assert_eq!(&result, &[]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_string() { + // let result = tokenize(String::from("\"\"")); + // assert_eq!(&result, &[JsonToken::String("".to_string())]); + + // let result = tokenize(String::from("\"value\"")); + // assert_eq!(&result, &[JsonToken::String("value".to_string())]); + + // let result = tokenize(String::from("\"value1\" \"value2\"")); + // assert_eq!( + // &result, + // &[ + // JsonToken::String("value1".to_string()), + // JsonToken::String("value2".to_string()) + // ] + // ); + + // let result = tokenize(String::from("\"value")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_escaped_characters() { + // let result = tokenize(String::from("\"\\b\\f\\n\\r\\t\"")); + // assert_eq!( + // &result, + // &[JsonToken::String("\u{8}\u{c}\n\r\t".to_string())] + // ); + + // let result = tokenize(String::from("\"\\x\"")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + // JsonToken::String("x".to_string()) + // ] + // ); + + // let result = tokenize(String::from("\"\\")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_unicode() { + // let result = tokenize(String::from("\"\\u1234\"")); + // assert_eq!(&result, &[JsonToken::String("ሴ".to_string())]); + + // let result = tokenize(String::from("\"\\uaBcDEeFf\"")); + // assert_eq!(&result, &[JsonToken::String("ꯍEeFf".to_string())]); + + // let result = tokenize(String::from("\"\\uEeFg\"")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidHex), + // JsonToken::String("g".to_string()) + // ] + // ); + + // let result = tokenize(String::from("\"\\uEeF")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_integer() { + // let result = tokenize(String::from("0")); + // assert_eq!(&result, &[JsonToken::Number(0.0)]); + + // let result = tokenize(String::from("-0")); + // assert_eq!(&result, &[JsonToken::Number(0.0)]); + + // let result = tokenize(String::from("0abc")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("abc")) + // ] + // ); + + // let result = tokenize(String::from("0. 2")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Number(2.0) + // ] + // ); + + // let result = tokenize(String::from("1234567890")); + // assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); + + // let result = tokenize(String::from("-1234567890")); + // assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); + + // let result = tokenize(String::from("[0,1]")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ArrayBegin, + // JsonToken::Number(0.0), + // JsonToken::Comma, + // JsonToken::Number(1.0), + // JsonToken::ArrayEnd + // ] + // ); + + // let result = tokenize(String::from("001")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Number(1.0), + // ] + // ); + + // let result = tokenize(String::from("-")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + // let result = tokenize(String::from("-{}")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::ObjectBegin, + // JsonToken::ObjectEnd + // ] + // ); + + // let result = tokenize(String::from("9007199254740991")); + // assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); + + // let result = tokenize(String::from("9007199254740992")); + // assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); + + // let result = tokenize(String::from("9007199254740993")); + // assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_big_float() { + // let result = tokenize(String::from("340282366920938463463374607431768211456")); + // assert_eq!( + // &result, + // &[JsonToken::Number(bigfloat_to_f64(BigFloat { + // significand: BigInt { + // sign: Sign::Positive, + // value: BigUint { + // value: [0, 0, 1].cast() + // } + // }, + // exp: 0, + // non_zero_reminder: false + // }))] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_float() { + // let result = tokenize(String::from("0.01")); + // assert_eq!(&result, &[JsonToken::Number(0.01)]); + + // let result = tokenize(String::from("[-12.34]")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ArrayBegin, + // JsonToken::Number(-12.34), + // JsonToken::ArrayEnd + // ] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_infinity() { + // let result = tokenize(String::from("1e1000")); + // assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); + + // let result = tokenize(String::from("-1e+1000")); + // assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_exp() { + // let result = tokenize(String::from("1e2")); + // assert_eq!(&result, &[JsonToken::Number(1e2)]); + + // let result = tokenize(String::from("1E+2")); + // assert_eq!(&result, &[JsonToken::Number(1e2)]); + + // let result = tokenize(String::from("0e-2")); + // assert_eq!(&result, &[JsonToken::Number(0.0)]); + + // let result = tokenize(String::from("1e-2")); + // assert_eq!(&result, &[JsonToken::Number(1e-2)]); + + // let result = tokenize(String::from("1.2e+2")); + // assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); + + // let result = tokenize(String::from("12e0000")); + // assert_eq!(&result, &[JsonToken::Number(12.0)]); + + // let result = tokenize(String::from("1e")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + // let result = tokenize(String::from("1e+")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + // let result = tokenize(String::from("1e-")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_big_int() { + // let result = tokenize(String::from("0n")); + // assert_eq!(&result, &[JsonToken::BigInt(BigInt::ZERO)]); + + // let result = tokenize(String::from("-0n")); + // assert_eq!( + // &result, + // &[JsonToken::BigInt(BigInt { + // sign: Sign::Negative, + // value: BigUint::ZERO + // })] + // ); + + // let result = tokenize(String::from("1234567890n")); + // assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_u64(1234567890))]); + + // let result = tokenize(String::from("-1234567890n")); + // assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_i64(-1234567890))]); + + // let result = tokenize(String::from("123.456n")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("n")) + // ] + // ); + + // let result = tokenize(String::from("123e456n")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("n")) + // ] + // ); + + // let result = tokenize(String::from("1234567890na")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("a")) + // ] + // ); + + // let result = tokenize(String::from("1234567890nn")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("n")) + // ] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_errors() { + // let result = tokenize(String::from("ᄑ")); + // assert_eq!( + // &result, + // &[JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_djs() { + // let result = tokenize(String::from("module.exports = ")); + // assert_eq!( + // &result, + // &[ + // JsonToken::Id(String::from("module")), + // JsonToken::Dot, + // JsonToken::Id(String::from("exports")), + // JsonToken::Equals, + // ] + // ); + // } + + #[test] + #[wasm_bindgen_test] + fn test_singleline_comments() { + let result = tokenize(String::from("{//abc\n2\n}")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::NewLine, + JsonToken::Number(2.0), + JsonToken::NewLine, + JsonToken::ObjectEnd, + ] + ); + + let result = tokenize(String::from("0//abc/*")); + assert_eq!(&result, &[JsonToken::Number(0.0),]); + + let result = tokenize(String::from("0//")); + assert_eq!(&result, &[JsonToken::Number(0.0),]); + + let result = tokenize(String::from("0/")); + assert_eq!( + &result, + &[ + JsonToken::Number(0.0), + JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + ] + ); + + let result = tokenize(String::from("0/a")); + assert_eq!( + &result, + &[ + JsonToken::Number(0.0), + JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + ] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_multiline_comments() { + let result = tokenize(String::from("{/*abc\ndef*/2}")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::Number(2.0), + JsonToken::ObjectEnd, + ] + ); + + let result = tokenize(String::from("{/*/* /**/2}")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::Number(2.0), + JsonToken::ObjectEnd, + ] + ); + + let result = tokenize(String::from("{/*")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::ErrorToken(ErrorType::CommentClosingExpected), + ] + ); + + let result = tokenize(String::from("{/**")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::ErrorToken(ErrorType::CommentClosingExpected), + ] + ); + } +} From f228113754a2107742d53cfd00f58659d0c4d4ef Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 16 Nov 2024 21:19:48 +0200 Subject: [PATCH 11/28] uncomment some tests --- nanvm-lib/src/tokenizer/mod.rs | 154 ++++++++++++++++----------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 7a98d409..28fe9abe 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -1474,85 +1474,85 @@ mod test { // ); // } - #[test] - #[wasm_bindgen_test] - fn test_singleline_comments() { - let result = tokenize(String::from("{//abc\n2\n}")); - assert_eq!( - &result, - &[ - JsonToken::ObjectBegin, - JsonToken::NewLine, - JsonToken::Number(2.0), - JsonToken::NewLine, - JsonToken::ObjectEnd, - ] - ); + // #[test] + // #[wasm_bindgen_test] + // fn test_singleline_comments() { + // let result = tokenize(String::from("{//abc\n2\n}")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ObjectBegin, + // JsonToken::NewLine, + // JsonToken::Number(2.0), + // JsonToken::NewLine, + // JsonToken::ObjectEnd, + // ] + // ); - let result = tokenize(String::from("0//abc/*")); - assert_eq!(&result, &[JsonToken::Number(0.0),]); + // let result = tokenize(String::from("0//abc/*")); + // assert_eq!(&result, &[JsonToken::Number(0.0),]); - let result = tokenize(String::from("0//")); - assert_eq!(&result, &[JsonToken::Number(0.0),]); + // let result = tokenize(String::from("0//")); + // assert_eq!(&result, &[JsonToken::Number(0.0),]); - let result = tokenize(String::from("0/")); - assert_eq!( - &result, - &[ - JsonToken::Number(0.0), - JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), - ] - ); - - let result = tokenize(String::from("0/a")); - assert_eq!( - &result, - &[ - JsonToken::Number(0.0), - JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), - ] - ); - } + // let result = tokenize(String::from("0/")); + // assert_eq!( + // &result, + // &[ + // JsonToken::Number(0.0), + // JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + // ] + // ); - #[test] - #[wasm_bindgen_test] - fn test_multiline_comments() { - let result = tokenize(String::from("{/*abc\ndef*/2}")); - assert_eq!( - &result, - &[ - JsonToken::ObjectBegin, - JsonToken::Number(2.0), - JsonToken::ObjectEnd, - ] - ); - - let result = tokenize(String::from("{/*/* /**/2}")); - assert_eq!( - &result, - &[ - JsonToken::ObjectBegin, - JsonToken::Number(2.0), - JsonToken::ObjectEnd, - ] - ); - - let result = tokenize(String::from("{/*")); - assert_eq!( - &result, - &[ - JsonToken::ObjectBegin, - JsonToken::ErrorToken(ErrorType::CommentClosingExpected), - ] - ); - - let result = tokenize(String::from("{/**")); - assert_eq!( - &result, - &[ - JsonToken::ObjectBegin, - JsonToken::ErrorToken(ErrorType::CommentClosingExpected), - ] - ); - } + // let result = tokenize(String::from("0/a")); + // assert_eq!( + // &result, + // &[ + // JsonToken::Number(0.0), + // JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + // ] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_multiline_comments() { + // let result = tokenize(String::from("{/*abc\ndef*/2}")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ObjectBegin, + // JsonToken::Number(2.0), + // JsonToken::ObjectEnd, + // ] + // ); + + // let result = tokenize(String::from("{/*/* /**/2}")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ObjectBegin, + // JsonToken::Number(2.0), + // JsonToken::ObjectEnd, + // ] + // ); + + // let result = tokenize(String::from("{/*")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ObjectBegin, + // JsonToken::ErrorToken(ErrorType::CommentClosingExpected), + // ] + // ); + + // let result = tokenize(String::from("{/**")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ObjectBegin, + // JsonToken::ErrorToken(ErrorType::CommentClosingExpected), + // ] + // ); + // } } From 59c4aaa7457f781a72057f8b27526549cc81237b Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 16 Nov 2024 21:22:39 +0200 Subject: [PATCH 12/28] uncomment some tests --- nanvm-lib/src/tokenizer/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 28fe9abe..39a95e09 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -1064,7 +1064,7 @@ mod test { big_float::BigFloat, big_int::{BigInt, Sign}, big_uint::BigUint, - }, common::cast::Cast, mem::local::Local, tokenizer::bigfloat_to_f64 + }, common::cast::Cast, mem::{global::GLOBAL, local::Local}, tokenizer::bigfloat_to_f64 }; use super::{tokenize, ErrorType, JsonToken}; @@ -1072,8 +1072,7 @@ mod test { #[test] #[wasm_bindgen_test] fn test_empty() { - let local = Local::default(); - let result = tokenize(&local, String::from("")); + let result = tokenize(GLOBAL, String::from("")); assert_eq!(result.len(), 0); } From 28360367b6fc62904db924c6b6ae9b2a6b76f920 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 17 Nov 2024 21:39:00 +0200 Subject: [PATCH 13/28] fix add_digit --- nanvm-lib/src/js/js_bigint.rs | 7 + nanvm-lib/src/tokenizer/mod.rs | 533 +++++++++++++++++---------------- 2 files changed, 282 insertions(+), 258 deletions(-) diff --git a/nanvm-lib/src/js/js_bigint.rs b/nanvm-lib/src/js/js_bigint.rs index 5203fbb9..b7169b87 100644 --- a/nanvm-lib/src/js/js_bigint.rs +++ b/nanvm-lib/src/js/js_bigint.rs @@ -357,6 +357,13 @@ pub fn shr(m: M, lhs: &JsBigint, rhs: &JsBigint) -> JsBigintMutRef bool { + if lhs.sign() != rhs.sign() { + return false; + } + return cmp_vec(lhs.items(), rhs.items()) == Ordering::Equal; +} + fn to_twos_complement(value: &JsBigint) -> TwosComplement { TwosComplement { sign: value.sign(), diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 39a95e09..9b202ad6 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -7,7 +7,7 @@ use std::{ use crate::{ big_numbers::{self, big_float::BigFloat, big_int::BigInt, big_uint::BigUint}, common::{cast::Cast, default::default}, - js::js_bigint::{self, add, from_u64, JsBigintMutRef}, + js::js_bigint::{self, add, equals, from_u64, mul, JsBigintMutRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State}, }; @@ -33,6 +33,19 @@ pub enum JsonToken { ClosingParenthesis, } +impl PartialEq for JsonToken { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::String(l0), Self::String(r0)) => l0 == r0, + (Self::Number(l0), Self::Number(r0)) => l0 == r0, + (Self::ErrorToken(l0), Self::ErrorToken(r0)) => l0 == r0, + (Self::BigInt(l0), Self::BigInt(r0)) => equals(l0, r0), + (Self::Id(l0), Self::Id(r0)) => l0 == r0, + _ => core::mem::discriminant(self) == core::mem::discriminant(other), + } + } +} + #[derive(Debug, PartialEq)] pub enum ErrorType { UnexpectedCharacter, @@ -218,8 +231,8 @@ impl JsBigintMutRef { fn add_digit>(self, m: M, c: char) -> JsBigintMutRef { add( m, - self.deref(), - Self::from_digit(m, js_bigint::Sign::Positive, c).deref(), + mul(m, self.deref(), from_u64(m, js_bigint::Sign::Positive, 10).deref()).deref(), + Self::from_digit(m, self.sign(), c).deref(), ) } @@ -1064,7 +1077,13 @@ mod test { big_float::BigFloat, big_int::{BigInt, Sign}, big_uint::BigUint, - }, common::cast::Cast, mem::{global::GLOBAL, local::Local}, tokenizer::bigfloat_to_f64 + }, + common::cast::Cast, + mem::{ + global::{self, Global, GLOBAL}, + local::Local, + }, + tokenizer::bigfloat_to_f64, }; use super::{tokenize, ErrorType, JsonToken}; @@ -1076,247 +1095,245 @@ mod test { assert_eq!(result.len(), 0); } - // #[test] - // #[wasm_bindgen_test] - // fn test_ops() { - // let local = Local::default(); - - // let result = tokenize(&local, String::from("{")); - // assert_eq!(&result, &[JsonToken::<&Local>::ObjectBegin]); - - // let result = tokenize(String::from("}")); - // assert_eq!(&result, &[JsonToken::ObjectEnd]); - - // let result = tokenize(String::from("[")); - // assert_eq!(&result, &[JsonToken::ArrayBegin]); - - // let result = tokenize(String::from("]")); - // assert_eq!(&result, &[JsonToken::ArrayEnd]); - - // let result = tokenize(String::from(":")); - // assert_eq!(&result, &[JsonToken::Colon]); - - // let result = tokenize(String::from(",")); - // assert_eq!(&result, &[JsonToken::Comma]); - - // let result = tokenize(String::from("=")); - // assert_eq!(&result, &[JsonToken::Equals]); - - // let result = tokenize(String::from(".")); - // assert_eq!(&result, &[JsonToken::Dot]); - - // let result = tokenize(String::from(";")); - // assert_eq!(&result, &[JsonToken::Semicolon]); - - // let result = tokenize(String::from("()")); - // assert_eq!( - // &result, - // &[JsonToken::OpeningParenthesis, JsonToken::ClosingParenthesis] - // ); - - // let result = tokenize(String::from("[{ :, }]")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ArrayBegin, - // JsonToken::ObjectBegin, - // JsonToken::Colon, - // JsonToken::Comma, - // JsonToken::ObjectEnd, - // JsonToken::ArrayEnd - // ] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_id() { - // let result = tokenize(String::from("true")); - // assert_eq!(&result, &[JsonToken::Id(String::from("true"))]); - - // let result = tokenize(String::from("false")); - // assert_eq!(&result, &[JsonToken::Id(String::from("false"))]); - - // let result = tokenize(String::from("null")); - // assert_eq!(&result, &[JsonToken::Id(String::from("null"))]); - - // let result = tokenize(String::from("tru tru")); - // assert_eq!( - // &result, - // &[ - // JsonToken::Id(String::from("tru")), - // JsonToken::Id(String::from("tru")), - // ] - // ); - - // let result = tokenize(String::from("ABCxyz_0123456789$")); - // assert_eq!( - // &result, - // &[JsonToken::Id(String::from("ABCxyz_0123456789$")),] - // ); - - // let result = tokenize(String::from("_")); - // assert_eq!(&result, &[JsonToken::Id(String::from("_")),]); - - // let result = tokenize(String::from("$")); - // assert_eq!(&result, &[JsonToken::Id(String::from("$")),]); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_whitespace() { - // let result = tokenize(String::from(" \t\n\r")); - // assert_eq!(&result, &[]); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_string() { - // let result = tokenize(String::from("\"\"")); - // assert_eq!(&result, &[JsonToken::String("".to_string())]); - - // let result = tokenize(String::from("\"value\"")); - // assert_eq!(&result, &[JsonToken::String("value".to_string())]); - - // let result = tokenize(String::from("\"value1\" \"value2\"")); - // assert_eq!( - // &result, - // &[ - // JsonToken::String("value1".to_string()), - // JsonToken::String("value2".to_string()) - // ] - // ); - - // let result = tokenize(String::from("\"value")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_escaped_characters() { - // let result = tokenize(String::from("\"\\b\\f\\n\\r\\t\"")); - // assert_eq!( - // &result, - // &[JsonToken::String("\u{8}\u{c}\n\r\t".to_string())] - // ); - - // let result = tokenize(String::from("\"\\x\"")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), - // JsonToken::String("x".to_string()) - // ] - // ); - - // let result = tokenize(String::from("\"\\")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_unicode() { - // let result = tokenize(String::from("\"\\u1234\"")); - // assert_eq!(&result, &[JsonToken::String("ሴ".to_string())]); - - // let result = tokenize(String::from("\"\\uaBcDEeFf\"")); - // assert_eq!(&result, &[JsonToken::String("ꯍEeFf".to_string())]); + #[test] + #[wasm_bindgen_test] + fn test_ops() { + let result = tokenize(GLOBAL, String::from("{")); + assert_eq!(result[0], JsonToken::::ObjectBegin); + + let result = tokenize(GLOBAL, String::from("}")); + assert_eq!(&result, &[JsonToken::ObjectEnd]); + + let result = tokenize(GLOBAL, String::from("[")); + assert_eq!(&result, &[JsonToken::ArrayBegin]); + + let result = tokenize(GLOBAL, String::from("]")); + assert_eq!(&result, &[JsonToken::ArrayEnd]); + + let result = tokenize(GLOBAL, String::from(":")); + assert_eq!(&result, &[JsonToken::Colon]); + + let result = tokenize(GLOBAL, String::from(",")); + assert_eq!(&result, &[JsonToken::Comma]); + + let result = tokenize(GLOBAL, String::from("=")); + assert_eq!(&result, &[JsonToken::Equals]); + + let result = tokenize(GLOBAL, String::from(".")); + assert_eq!(&result, &[JsonToken::Dot]); + + let result = tokenize(GLOBAL, String::from(";")); + assert_eq!(&result, &[JsonToken::Semicolon]); + + let result = tokenize(GLOBAL, String::from("()")); + assert_eq!( + &result, + &[JsonToken::OpeningParenthesis, JsonToken::ClosingParenthesis] + ); + + let result = tokenize(GLOBAL, String::from("[{ :, }]")); + assert_eq!( + &result, + &[ + JsonToken::ArrayBegin, + JsonToken::ObjectBegin, + JsonToken::Colon, + JsonToken::Comma, + JsonToken::ObjectEnd, + JsonToken::ArrayEnd + ] + ); + } - // let result = tokenize(String::from("\"\\uEeFg\"")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidHex), - // JsonToken::String("g".to_string()) - // ] - // ); + #[test] + #[wasm_bindgen_test] + fn test_id() { + let result = tokenize(GLOBAL, String::from("true")); + assert_eq!(&result, &[JsonToken::Id(String::from("true"))]); + + let result = tokenize(GLOBAL, String::from("false")); + assert_eq!(&result, &[JsonToken::Id(String::from("false"))]); + + let result = tokenize(GLOBAL, String::from("null")); + assert_eq!(&result, &[JsonToken::Id(String::from("null"))]); + + let result = tokenize(GLOBAL, String::from("tru tru")); + assert_eq!( + &result, + &[ + JsonToken::Id(String::from("tru")), + JsonToken::Id(String::from("tru")), + ] + ); - // let result = tokenize(String::from("\"\\uEeF")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); - // } + let result = tokenize(GLOBAL, String::from("ABCxyz_0123456789$")); + assert_eq!( + &result, + &[JsonToken::Id(String::from("ABCxyz_0123456789$")),] + ); - // #[test] - // #[wasm_bindgen_test] - // fn test_integer() { - // let result = tokenize(String::from("0")); - // assert_eq!(&result, &[JsonToken::Number(0.0)]); + let result = tokenize(GLOBAL, String::from("_")); + assert_eq!(&result, &[JsonToken::Id(String::from("_")),]); - // let result = tokenize(String::from("-0")); - // assert_eq!(&result, &[JsonToken::Number(0.0)]); + let result = tokenize(GLOBAL, String::from("$")); + assert_eq!(&result, &[JsonToken::Id(String::from("$")),]); + } - // let result = tokenize(String::from("0abc")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Id(String::from("abc")) - // ] - // ); + #[test] + #[wasm_bindgen_test] + fn test_whitespace() { + let result = tokenize(GLOBAL, String::from(" \t\n\r")); + assert_eq!(&result, &[]); + } - // let result = tokenize(String::from("0. 2")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Number(2.0) - // ] - // ); + #[test] + #[wasm_bindgen_test] + fn test_string() { + let result = tokenize(GLOBAL, String::from("\"\"")); + assert_eq!(&result, &[JsonToken::String("".to_string())]); + + let result = tokenize(GLOBAL, String::from("\"value\"")); + assert_eq!(&result, &[JsonToken::String("value".to_string())]); + + let result = tokenize(GLOBAL, String::from("\"value1\" \"value2\"")); + assert_eq!( + &result, + &[ + JsonToken::String("value1".to_string()), + JsonToken::String("value2".to_string()) + ] + ); - // let result = tokenize(String::from("1234567890")); - // assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); + let result = tokenize(GLOBAL, String::from("\"value")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); + } - // let result = tokenize(String::from("-1234567890")); - // assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); + #[test] + #[wasm_bindgen_test] + fn test_escaped_characters() { + let result = tokenize(GLOBAL, String::from("\"\\b\\f\\n\\r\\t\"")); + assert_eq!( + &result, + &[JsonToken::String("\u{8}\u{c}\n\r\t".to_string())] + ); + + let result = tokenize(GLOBAL, String::from("\"\\x\"")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + JsonToken::String("x".to_string()) + ] + ); - // let result = tokenize(String::from("[0,1]")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ArrayBegin, - // JsonToken::Number(0.0), - // JsonToken::Comma, - // JsonToken::Number(1.0), - // JsonToken::ArrayEnd - // ] - // ); + let result = tokenize(GLOBAL, String::from("\"\\")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); + } - // let result = tokenize(String::from("001")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Number(1.0), - // ] - // ); + #[test] + #[wasm_bindgen_test] + fn test_unicode() { + let result = tokenize(GLOBAL, String::from("\"\\u1234\"")); + assert_eq!(&result, &[JsonToken::String("ሴ".to_string())]); + + let result = tokenize(GLOBAL, String::from("\"\\uaBcDEeFf\"")); + assert_eq!(&result, &[JsonToken::String("ꯍEeFf".to_string())]); + + let result = tokenize(GLOBAL, String::from("\"\\uEeFg\"")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidHex), + JsonToken::String("g".to_string()) + ] + ); - // let result = tokenize(String::from("-")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + let result = tokenize(GLOBAL, String::from("\"\\uEeF")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); + } - // let result = tokenize(String::from("-{}")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::ObjectBegin, - // JsonToken::ObjectEnd - // ] - // ); + #[test] + #[wasm_bindgen_test] + fn test_integer() { + let result = tokenize(GLOBAL, String::from("0")); + assert_eq!(&result, &[JsonToken::Number(0.0)]); + + let result = tokenize(GLOBAL, String::from("-0")); + assert_eq!(&result, &[JsonToken::Number(0.0)]); + + let result = tokenize(GLOBAL, String::from("0abc")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Id(String::from("abc")) + ] + ); + + let result = tokenize(GLOBAL, String::from("0. 2")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Number(2.0) + ] + ); + + let result = tokenize(GLOBAL, String::from("1234567890")); + assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); + + let result = tokenize(GLOBAL, String::from("-1234567890")); + assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); + + let result = tokenize(GLOBAL, String::from("[0,1]")); + assert_eq!( + &result, + &[ + JsonToken::ArrayBegin, + JsonToken::Number(0.0), + JsonToken::Comma, + JsonToken::Number(1.0), + JsonToken::ArrayEnd + ] + ); + + let result = tokenize(GLOBAL, String::from("001")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Number(1.0), + ] + ); + + let result = tokenize(GLOBAL, String::from("-")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + let result = tokenize(GLOBAL, String::from("-{}")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::ObjectBegin, + JsonToken::ObjectEnd + ] + ); - // let result = tokenize(String::from("9007199254740991")); - // assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); + let result = tokenize(GLOBAL, String::from("9007199254740991")); + assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); - // let result = tokenize(String::from("9007199254740992")); - // assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); + let result = tokenize(GLOBAL, String::from("9007199254740992")); + assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); - // let result = tokenize(String::from("9007199254740993")); - // assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); - // } + let result = tokenize(GLOBAL, String::from("9007199254740993")); + assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); + } // #[test] // #[wasm_bindgen_test] // fn test_big_float() { - // let result = tokenize(String::from("340282366920938463463374607431768211456")); + // let result = tokenize(GLOBAL, String::from("340282366920938463463374607431768211456")); // assert_eq!( // &result, // &[JsonToken::Number(bigfloat_to_f64(BigFloat { @@ -1335,10 +1352,10 @@ mod test { // #[test] // #[wasm_bindgen_test] // fn test_float() { - // let result = tokenize(String::from("0.01")); + // let result = tokenize(GLOBAL, String::from("0.01")); // assert_eq!(&result, &[JsonToken::Number(0.01)]); - // let result = tokenize(String::from("[-12.34]")); + // let result = tokenize(GLOBAL, String::from("[-12.34]")); // assert_eq!( // &result, // &[ @@ -1352,51 +1369,51 @@ mod test { // #[test] // #[wasm_bindgen_test] // fn test_infinity() { - // let result = tokenize(String::from("1e1000")); + // let result = tokenize(GLOBAL, String::from("1e1000")); // assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); - // let result = tokenize(String::from("-1e+1000")); + // let result = tokenize(GLOBAL, String::from("-1e+1000")); // assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); // } // #[test] // #[wasm_bindgen_test] // fn test_exp() { - // let result = tokenize(String::from("1e2")); + // let result = tokenize(GLOBAL, String::from("1e2")); // assert_eq!(&result, &[JsonToken::Number(1e2)]); - // let result = tokenize(String::from("1E+2")); + // let result = tokenize(GLOBAL, String::from("1E+2")); // assert_eq!(&result, &[JsonToken::Number(1e2)]); - // let result = tokenize(String::from("0e-2")); + // let result = tokenize(GLOBAL, String::from("0e-2")); // assert_eq!(&result, &[JsonToken::Number(0.0)]); - // let result = tokenize(String::from("1e-2")); + // let result = tokenize(GLOBAL, String::from("1e-2")); // assert_eq!(&result, &[JsonToken::Number(1e-2)]); - // let result = tokenize(String::from("1.2e+2")); + // let result = tokenize(GLOBAL, String::from("1.2e+2")); // assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); - // let result = tokenize(String::from("12e0000")); + // let result = tokenize(GLOBAL, String::from("12e0000")); // assert_eq!(&result, &[JsonToken::Number(12.0)]); - // let result = tokenize(String::from("1e")); + // let result = tokenize(GLOBAL, String::from("1e")); // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - // let result = tokenize(String::from("1e+")); + // let result = tokenize(GLOBAL, String::from("1e+")); // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - // let result = tokenize(String::from("1e-")); + // let result = tokenize(GLOBAL, String::from("1e-")); // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); // } // #[test] // #[wasm_bindgen_test] // fn test_big_int() { - // let result = tokenize(String::from("0n")); + // let result = tokenize(GLOBAL, String::from("0n")); // assert_eq!(&result, &[JsonToken::BigInt(BigInt::ZERO)]); - // let result = tokenize(String::from("-0n")); + // let result = tokenize(GLOBAL, String::from("-0n")); // assert_eq!( // &result, // &[JsonToken::BigInt(BigInt { @@ -1405,13 +1422,13 @@ mod test { // })] // ); - // let result = tokenize(String::from("1234567890n")); + // let result = tokenize(GLOBAL, String::from("1234567890n")); // assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_u64(1234567890))]); - // let result = tokenize(String::from("-1234567890n")); + // let result = tokenize(GLOBAL, String::from("-1234567890n")); // assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_i64(-1234567890))]); - // let result = tokenize(String::from("123.456n")); + // let result = tokenize(GLOBAL, String::from("123.456n")); // assert_eq!( // &result, // &[ @@ -1420,7 +1437,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("123e456n")); + // let result = tokenize(GLOBAL, String::from("123e456n")); // assert_eq!( // &result, // &[ @@ -1429,7 +1446,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("1234567890na")); + // let result = tokenize(GLOBAL, String::from("1234567890na")); // assert_eq!( // &result, // &[ @@ -1438,7 +1455,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("1234567890nn")); + // let result = tokenize(GLOBAL, String::from("1234567890nn")); // assert_eq!( // &result, // &[ @@ -1451,7 +1468,7 @@ mod test { // #[test] // #[wasm_bindgen_test] // fn test_errors() { - // let result = tokenize(String::from("ᄑ")); + // let result = tokenize(GLOBAL, String::from("ᄑ")); // assert_eq!( // &result, // &[JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)] @@ -1461,7 +1478,7 @@ mod test { // #[test] // #[wasm_bindgen_test] // fn test_djs() { - // let result = tokenize(String::from("module.exports = ")); + // let result = tokenize(GLOBAL, String::from("module.exports = ")); // assert_eq!( // &result, // &[ @@ -1476,7 +1493,7 @@ mod test { // #[test] // #[wasm_bindgen_test] // fn test_singleline_comments() { - // let result = tokenize(String::from("{//abc\n2\n}")); + // let result = tokenize(GLOBAL, String::from("{//abc\n2\n}")); // assert_eq!( // &result, // &[ @@ -1488,13 +1505,13 @@ mod test { // ] // ); - // let result = tokenize(String::from("0//abc/*")); + // let result = tokenize(GLOBAL, String::from("0//abc/*")); // assert_eq!(&result, &[JsonToken::Number(0.0),]); - // let result = tokenize(String::from("0//")); + // let result = tokenize(GLOBAL, String::from("0//")); // assert_eq!(&result, &[JsonToken::Number(0.0),]); - // let result = tokenize(String::from("0/")); + // let result = tokenize(GLOBAL, String::from("0/")); // assert_eq!( // &result, // &[ @@ -1503,7 +1520,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("0/a")); + // let result = tokenize(GLOBAL, String::from("0/a")); // assert_eq!( // &result, // &[ @@ -1516,7 +1533,7 @@ mod test { // #[test] // #[wasm_bindgen_test] // fn test_multiline_comments() { - // let result = tokenize(String::from("{/*abc\ndef*/2}")); + // let result = tokenize(GLOBAL, String::from("{/*abc\ndef*/2}")); // assert_eq!( // &result, // &[ @@ -1526,7 +1543,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("{/*/* /**/2}")); + // let result = tokenize(GLOBAL, String::from("{/*/* /**/2}")); // assert_eq!( // &result, // &[ @@ -1536,7 +1553,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("{/*")); + // let result = tokenize(GLOBAL, String::from("{/*")); // assert_eq!( // &result, // &[ @@ -1545,7 +1562,7 @@ mod test { // ] // ); - // let result = tokenize(String::from("{/**")); + // let result = tokenize(GLOBAL, String::from("{/**")); // assert_eq!( // &result, // &[ From 88372dae7ad17a421bf3f7c98b595c2c30c9d052 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 17 Nov 2024 21:43:35 +0200 Subject: [PATCH 14/28] fix tokenizer tests --- nanvm-lib/src/tokenizer/mod.rs | 513 +++++++++++++++++---------------- 1 file changed, 267 insertions(+), 246 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 9b202ad6..0632a4b0 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -231,7 +231,12 @@ impl JsBigintMutRef { fn add_digit>(self, m: M, c: char) -> JsBigintMutRef { add( m, - mul(m, self.deref(), from_u64(m, js_bigint::Sign::Positive, 10).deref()).deref(), + mul( + m, + self.deref(), + from_u64(m, js_bigint::Sign::Positive, 10).deref(), + ) + .deref(), Self::from_digit(m, self.sign(), c).deref(), ) } @@ -1079,10 +1084,8 @@ mod test { big_uint::BigUint, }, common::cast::Cast, - mem::{ - global::{self, Global, GLOBAL}, - local::Local, - }, + js::js_bigint::{self, from_u64, zero}, + mem::global::{Global, GLOBAL}, tokenizer::bigfloat_to_f64, }; @@ -1330,245 +1333,263 @@ mod test { assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); } - // #[test] - // #[wasm_bindgen_test] - // fn test_big_float() { - // let result = tokenize(GLOBAL, String::from("340282366920938463463374607431768211456")); - // assert_eq!( - // &result, - // &[JsonToken::Number(bigfloat_to_f64(BigFloat { - // significand: BigInt { - // sign: Sign::Positive, - // value: BigUint { - // value: [0, 0, 1].cast() - // } - // }, - // exp: 0, - // non_zero_reminder: false - // }))] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_float() { - // let result = tokenize(GLOBAL, String::from("0.01")); - // assert_eq!(&result, &[JsonToken::Number(0.01)]); - - // let result = tokenize(GLOBAL, String::from("[-12.34]")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ArrayBegin, - // JsonToken::Number(-12.34), - // JsonToken::ArrayEnd - // ] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_infinity() { - // let result = tokenize(GLOBAL, String::from("1e1000")); - // assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); - - // let result = tokenize(GLOBAL, String::from("-1e+1000")); - // assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_exp() { - // let result = tokenize(GLOBAL, String::from("1e2")); - // assert_eq!(&result, &[JsonToken::Number(1e2)]); - - // let result = tokenize(GLOBAL, String::from("1E+2")); - // assert_eq!(&result, &[JsonToken::Number(1e2)]); - - // let result = tokenize(GLOBAL, String::from("0e-2")); - // assert_eq!(&result, &[JsonToken::Number(0.0)]); - - // let result = tokenize(GLOBAL, String::from("1e-2")); - // assert_eq!(&result, &[JsonToken::Number(1e-2)]); - - // let result = tokenize(GLOBAL, String::from("1.2e+2")); - // assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); - - // let result = tokenize(GLOBAL, String::from("12e0000")); - // assert_eq!(&result, &[JsonToken::Number(12.0)]); - - // let result = tokenize(GLOBAL, String::from("1e")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - - // let result = tokenize(GLOBAL, String::from("1e+")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - - // let result = tokenize(GLOBAL, String::from("1e-")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_big_int() { - // let result = tokenize(GLOBAL, String::from("0n")); - // assert_eq!(&result, &[JsonToken::BigInt(BigInt::ZERO)]); - - // let result = tokenize(GLOBAL, String::from("-0n")); - // assert_eq!( - // &result, - // &[JsonToken::BigInt(BigInt { - // sign: Sign::Negative, - // value: BigUint::ZERO - // })] - // ); - - // let result = tokenize(GLOBAL, String::from("1234567890n")); - // assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_u64(1234567890))]); - - // let result = tokenize(GLOBAL, String::from("-1234567890n")); - // assert_eq!(&result, &[JsonToken::BigInt(BigInt::from_i64(-1234567890))]); - - // let result = tokenize(GLOBAL, String::from("123.456n")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Id(String::from("n")) - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("123e456n")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Id(String::from("n")) - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("1234567890na")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Id(String::from("a")) - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("1234567890nn")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Id(String::from("n")) - // ] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_errors() { - // let result = tokenize(GLOBAL, String::from("ᄑ")); - // assert_eq!( - // &result, - // &[JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_djs() { - // let result = tokenize(GLOBAL, String::from("module.exports = ")); - // assert_eq!( - // &result, - // &[ - // JsonToken::Id(String::from("module")), - // JsonToken::Dot, - // JsonToken::Id(String::from("exports")), - // JsonToken::Equals, - // ] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_singleline_comments() { - // let result = tokenize(GLOBAL, String::from("{//abc\n2\n}")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ObjectBegin, - // JsonToken::NewLine, - // JsonToken::Number(2.0), - // JsonToken::NewLine, - // JsonToken::ObjectEnd, - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("0//abc/*")); - // assert_eq!(&result, &[JsonToken::Number(0.0),]); - - // let result = tokenize(GLOBAL, String::from("0//")); - // assert_eq!(&result, &[JsonToken::Number(0.0),]); - - // let result = tokenize(GLOBAL, String::from("0/")); - // assert_eq!( - // &result, - // &[ - // JsonToken::Number(0.0), - // JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("0/a")); - // assert_eq!( - // &result, - // &[ - // JsonToken::Number(0.0), - // JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), - // ] - // ); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_multiline_comments() { - // let result = tokenize(GLOBAL, String::from("{/*abc\ndef*/2}")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ObjectBegin, - // JsonToken::Number(2.0), - // JsonToken::ObjectEnd, - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("{/*/* /**/2}")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ObjectBegin, - // JsonToken::Number(2.0), - // JsonToken::ObjectEnd, - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("{/*")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ObjectBegin, - // JsonToken::ErrorToken(ErrorType::CommentClosingExpected), - // ] - // ); - - // let result = tokenize(GLOBAL, String::from("{/**")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ObjectBegin, - // JsonToken::ErrorToken(ErrorType::CommentClosingExpected), - // ] - // ); - // } + #[test] + #[wasm_bindgen_test] + fn test_big_float() { + let result = tokenize( + GLOBAL, + String::from("340282366920938463463374607431768211456"), + ); + assert_eq!( + &result, + &[JsonToken::Number(bigfloat_to_f64(BigFloat { + significand: BigInt { + sign: Sign::Positive, + value: BigUint { + value: [0, 0, 1].cast() + } + }, + exp: 0, + non_zero_reminder: false + }))] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_float() { + let result = tokenize(GLOBAL, String::from("0.01")); + assert_eq!(&result, &[JsonToken::Number(0.01)]); + + let result = tokenize(GLOBAL, String::from("[-12.34]")); + assert_eq!( + &result, + &[ + JsonToken::ArrayBegin, + JsonToken::Number(-12.34), + JsonToken::ArrayEnd + ] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_infinity() { + let result = tokenize(GLOBAL, String::from("1e1000")); + assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); + + let result = tokenize(GLOBAL, String::from("-1e+1000")); + assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); + } + + #[test] + #[wasm_bindgen_test] + fn test_exp() { + let result = tokenize(GLOBAL, String::from("1e2")); + assert_eq!(&result, &[JsonToken::Number(1e2)]); + + let result = tokenize(GLOBAL, String::from("1E+2")); + assert_eq!(&result, &[JsonToken::Number(1e2)]); + + let result = tokenize(GLOBAL, String::from("0e-2")); + assert_eq!(&result, &[JsonToken::Number(0.0)]); + + let result = tokenize(GLOBAL, String::from("1e-2")); + assert_eq!(&result, &[JsonToken::Number(1e-2)]); + + let result = tokenize(GLOBAL, String::from("1.2e+2")); + assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); + + let result = tokenize(GLOBAL, String::from("12e0000")); + assert_eq!(&result, &[JsonToken::Number(12.0)]); + + let result = tokenize(GLOBAL, String::from("1e")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + let result = tokenize(GLOBAL, String::from("1e+")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + let result = tokenize(GLOBAL, String::from("1e-")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + } + + #[test] + #[wasm_bindgen_test] + fn test_big_int() { + let result = tokenize(GLOBAL, String::from("0n")); + assert_eq!(&result, &[JsonToken::BigInt(zero(GLOBAL))]); + + let result = tokenize(GLOBAL, String::from("-0n")); + assert_eq!( + &result, + &[JsonToken::BigInt(from_u64( + GLOBAL, + js_bigint::Sign::Negative, + 0 + ))] + ); + + let result = tokenize(GLOBAL, String::from("1234567890n")); + assert_eq!( + &result, + &[JsonToken::BigInt(from_u64( + GLOBAL, + js_bigint::Sign::Positive, + 1234567890 + ))] + ); + + let result = tokenize(GLOBAL, String::from("-1234567890n")); + assert_eq!( + &result, + &[JsonToken::BigInt(from_u64( + GLOBAL, + js_bigint::Sign::Negative, + 1234567890 + ))] + ); + + let result = tokenize(GLOBAL, String::from("123.456n")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Id(String::from("n")) + ] + ); + + let result = tokenize(GLOBAL, String::from("123e456n")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Id(String::from("n")) + ] + ); + + let result = tokenize(GLOBAL, String::from("1234567890na")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Id(String::from("a")) + ] + ); + + let result = tokenize(GLOBAL, String::from("1234567890nn")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Id(String::from("n")) + ] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_errors() { + let result = tokenize(GLOBAL, String::from("ᄑ")); + assert_eq!( + &result, + &[JsonToken::ErrorToken(ErrorType::UnexpectedCharacter)] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_djs() { + let result = tokenize(GLOBAL, String::from("module.exports = ")); + assert_eq!( + &result, + &[ + JsonToken::Id(String::from("module")), + JsonToken::Dot, + JsonToken::Id(String::from("exports")), + JsonToken::Equals, + ] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_singleline_comments() { + let result = tokenize(GLOBAL, String::from("{//abc\n2\n}")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::NewLine, + JsonToken::Number(2.0), + JsonToken::NewLine, + JsonToken::ObjectEnd, + ] + ); + + let result = tokenize(GLOBAL, String::from("0//abc/*")); + assert_eq!(&result, &[JsonToken::Number(0.0),]); + + let result = tokenize(GLOBAL, String::from("0//")); + assert_eq!(&result, &[JsonToken::Number(0.0),]); + + let result = tokenize(GLOBAL, String::from("0/")); + assert_eq!( + &result, + &[ + JsonToken::Number(0.0), + JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + ] + ); + + let result = tokenize(GLOBAL, String::from("0/a")); + assert_eq!( + &result, + &[ + JsonToken::Number(0.0), + JsonToken::ErrorToken(ErrorType::UnexpectedCharacter), + ] + ); + } + + #[test] + #[wasm_bindgen_test] + fn test_multiline_comments() { + let result = tokenize(GLOBAL, String::from("{/*abc\ndef*/2}")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::Number(2.0), + JsonToken::ObjectEnd, + ] + ); + + let result = tokenize(GLOBAL, String::from("{/*/* /**/2}")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::Number(2.0), + JsonToken::ObjectEnd, + ] + ); + + let result = tokenize(GLOBAL, String::from("{/*")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::ErrorToken(ErrorType::CommentClosingExpected), + ] + ); + + let result = tokenize(GLOBAL, String::from("{/**")); + assert_eq!( + &result, + &[ + JsonToken::ObjectBegin, + JsonToken::ErrorToken(ErrorType::CommentClosingExpected), + ] + ); + } } From 74b52ed62b9e414261ca9e8f6937638541b27e42 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 23 Nov 2024 16:33:01 +0200 Subject: [PATCH 15/28] parser tests --- nanvm-lib/src/parser/parser.rs | 1753 ++++++++++++++++---------------- 1 file changed, 863 insertions(+), 890 deletions(-) diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index 16b0656c..5807cf58 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -356,893 +356,866 @@ pub fn parse_with_tokens( state.end() } -// #[cfg(test)] -// mod test { -// use io_test::VirtualIo; -// use io_trait::Io; -// use wasm_bindgen_test::wasm_bindgen_test; - -// use crate::{ -// big_numbers::{ -// big_int::{BigInt, Sign}, -// big_uint::BigUint, -// }, -// common::default::default, -// js::{ -// self, js_array::JsArrayRef, js_bigint::{from_u64, new_bigint, JsBigintRef}, js_object::JsObjectRef, js_string::JsStringRef, type_::Type -// }, -// mem::{global::GLOBAL, local::Local, manager::Manager}, -// tokenizer::{tokenize, ErrorType, JsonToken}, -// }; - -// use super::super::{parser::parse, path::concat, shared::DataType}; - -// use super::{parse_with_tokens, Context, ModuleCache, ParseError, ParseResult}; - -// fn virtual_io() -> VirtualIo { -// VirtualIo::new(&[]) -// } - -// fn create_test_context<'a, M: Manager>( -// manager: M, -// io: &'a VirtualIo, -// module_cache: &'a mut ModuleCache, -// ) -> Context<'a, M, VirtualIo> { -// Context::new(manager, io, default(), module_cache) -// } - -// fn parse_with_virtual_io( -// manager: M, -// iter: impl Iterator>, -// ) -> Result, ParseError> { -// parse_with_tokens( -// &mut create_test_context(manager, &virtual_io(), &mut default()), -// iter, -// ) -// } - -// fn test_local() { -// let local = Local::default(); -// let _ = parse_with_tokens( -// &mut create_test_context(&local, &virtual_io(), &mut default()), -// [].into_iter(), -// ); -// } - -// fn test_global() { -// let _ = { -// let global = GLOBAL; -// parse_with_tokens( -// &mut create_test_context(global, &virtual_io(), &mut default()), -// [].into_iter(), -// ) -// }; -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_json() { -// let json_str = include_str!("../../test/test-json.json"); -// let tokens = tokenize(json_str.to_owned()); -// let local = Local::default(); -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().data_type, DataType::Json); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_djs() { -// let json_str = include_str!("../../test/test-djs.d.cjs"); -// let tokens = tokenize(json_str.to_owned()); -// let local = Local::default(); -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().data_type, DataType::Cjs); - -// let json_str = include_str!("../../test/test-djs.d.mjs"); -// let tokens = tokenize(json_str.to_owned()); -// let local = Local::default(); -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().data_type, DataType::Mjs); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_const() { -// let local = Local::default(); -// test_const_with_manager(&local); -// } - -// fn test_const_with_manager(manager: M) { -// let json_str = include_str!("../../test/test-const.d.cjs"); -// let tokens = tokenize(json_str.to_owned()); -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.try_move(), Ok(2.0)); -// let item1 = items[1].clone(); -// assert_eq!(item1.try_move(), Ok(2.0)); - -// let json_str = include_str!("../../test/test-const-error.d.cjs.txt"); -// let tokens = tokenize(json_str.to_owned()); -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let json_str = include_str!("../../test/test-const-error-new-line.d.cjs.txt"); -// let tokens = tokenize(json_str.to_owned()); -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); -// assert_eq!(result.unwrap_err(), ParseError::NewLineExpected); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_stack() { -// let local = Local::default(); -// test_stack_with_manager(&local); -// } - -// fn test_stack_with_manager(manager: M) { -// let json_str = include_str!("../../test/test-stack.d.cjs"); -// let tokens = tokenize(json_str.to_owned()); -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// let result_unwrap = item0.try_move::>().unwrap(); -// let items = result_unwrap.items(); -// let (key0, value0) = items[0].clone(); -// let key0_items = key0.items(); -// assert_eq!(key0_items, [0x61]); -// let result_unwrap = value0.try_move::>().unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.get_type(), Type::Null); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_import() { -// let local = Local::default(); -// test_import_with_manager(&local); -// } - -// fn test_import_with_manager(manager: M) { -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_import_main.d.cjs"); -// //let path = "../../test/test-import-main.d.cjs"; -// let main_path = "test_import_main.d.cjs"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module = include_str!("../../test/test_import_module.d.cjs"); -// let module_path = "test_import_module.d.cjs"; -// io.write(module_path, module.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.try_move(), Ok(3.0)); - -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_import_main.d.mjs"); -// //let path = "../../test/test-import-main.d.mjs"; -// let main_path = "test_import_main.d.mjs"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module = include_str!("../../test/test_import_module.d.mjs"); -// let module_path = "test_import_module.d.mjs"; -// io.write(module_path, module.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.try_move(), Ok(4.0)); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_cache() { -// let local = Local::default(); -// test_cache_with_manager(&local); -// } - -// fn test_cache_with_manager(manager: M) { -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_cache_main.d.cjs"); -// let main_path = "test_cache_main.d.cjs"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module_b = include_str!("../../test/test_cache_b.d.cjs"); -// let module_b_path = "test_cache_b.d.cjs"; -// io.write(module_b_path, module_b.as_bytes()).unwrap(); - -// let module_c = include_str!("../../test/test_cache_c.d.cjs"); -// let module_c_path = "test_cache_c.d.cjs"; -// io.write(module_c_path, module_c.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.try_move(), Ok(1.0)); -// let item1 = items[1].clone(); -// assert_eq!(item1.try_move(), Ok(1.0)); - -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_cache_main.d.mjs"); -// let main_path = "test_cache_main.d.mjs"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module_b = include_str!("../../test/test_cache_b.d.mjs"); -// let module_b_path = "test_cache_b.d.mjs"; -// io.write(module_b_path, module_b.as_bytes()).unwrap(); - -// let module_c = include_str!("../../test/test_cache_c.d.mjs"); -// let module_c_path = "test_cache_c.d.mjs"; -// io.write(module_c_path, module_c.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.try_move(), Ok(2.0)); -// let item1 = items[1].clone(); -// assert_eq!(item1.try_move(), Ok(2.0)); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_circular_error() { -// let local = Local::default(); -// test_circular_error_with_manager(&local); -// } - -// fn test_circular_error_with_manager(manager: M) { -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_circular_1.d.cjs.txt"); -// let main_path = "test_circular_1.d.cjs.txt"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module = include_str!("../../test/test_circular_2.d.cjs.txt"); -// let module_path = "test_circular_2.d.cjs.txt"; -// io.write(module_path, module.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_err()); -// assert_eq!(result.unwrap_err(), ParseError::CircularDependency); - -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_circular_1.d.mjs.txt"); -// let main_path = "test_circular_1.d.mjs.txt"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module = include_str!("../../test/test_circular_2.d.mjs.txt"); -// let module_path = "test_circular_2.d.mjs.txt"; -// io.write(module_path, module.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_err()); -// assert_eq!(result.unwrap_err(), ParseError::CircularDependency); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_import_error() { -// let local = Local::default(); -// test_import_error_with_manager(&local); -// } - -// fn test_import_error_with_manager(manager: M) { -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_import_error.d.cjs.txt"); -// //let path = "../../test/test-import-main.d.cjs"; -// let main_path = "test_import_error.d.cjs.txt"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module = include_str!("../../test/test_import_module.d.mjs"); -// let module_path = "test_import_module.d.mjs"; -// io.write(module_path, module.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_err()); -// assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); - -// let io: VirtualIo = VirtualIo::new(&[]); - -// let main = include_str!("../../test/test_import_error.d.mjs.txt"); -// //let path = "../../test/test-import-main.d.cjs"; -// let main_path = "test_import_error.d.mjs.txt"; -// io.write(main_path, main.as_bytes()).unwrap(); - -// let module = include_str!("../../test/test_import_module.d.cjs"); -// let module_path = "test_import_module.d.cjs"; -// io.write(module_path, module.as_bytes()).unwrap(); - -// let mut mc = default(); -// let mut context = Context::new( -// manager, -// &io, -// concat(io.current_dir().unwrap().as_str(), main_path), -// &mut mc, -// ); - -// let result = parse(&mut context); -// assert!(result.is_err()); -// assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_trailing_comma() { -// let local = Local::default(); -// let json_str = include_str!("../../test/test-trailing-comma.d.cjs"); -// let tokens = tokenize(json_str.to_owned()); -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_check_sizes() { -// let local = Local::default(); -// { -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("k")), -// JsonToken::Colon, -// JsonToken::ObjectBegin, -// JsonToken::ObjectEnd, -// JsonToken::ObjectEnd, -// ]; -// { -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// let _result_unwrap = result.unwrap(); -// } -// assert_eq!(local.size(), 0); -// } -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_check_sizes2() { -// let local = Local::default(); -// { -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("k")), -// JsonToken::Colon, -// JsonToken::ObjectBegin, -// JsonToken::ObjectEnd, -// JsonToken::ObjectEnd, -// ]; -// { -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result.unwrap().any; -// let _result_unwrap = result_unwrap.try_move::>(); -// } -// assert_eq!(local.size(), 0); -// } -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_data_type() { -// let local = Local::default(); -// let tokens = [JsonToken::Id(String::from("null"))]; -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().data_type, DataType::Json); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_export_block() { -// let local = Local::default(); -// let tokens = [ -// JsonToken::Id(String::from("export")), -// JsonToken::Id(String::from("default")), -// JsonToken::Id(String::from("null")), -// ]; -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().data_type, DataType::Mjs); - -// let local = Local::default(); -// let tokens = [ -// JsonToken::Id(String::from("module")), -// JsonToken::Dot, -// JsonToken::Id(String::from("exports")), -// JsonToken::Equals, -// JsonToken::Id(String::from("null")), -// ]; -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().data_type, DataType::Cjs); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_id_in_objects() { -// let local = Local::default(); -// let tokens = [ -// JsonToken::Id(String::from("export")), -// JsonToken::Id(String::from("default")), -// JsonToken::ObjectBegin, -// JsonToken::Id(String::from("key")), -// JsonToken::Colon, -// JsonToken::Number(0.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result.unwrap().any.try_move::>().unwrap(); -// let items = result_unwrap.items(); -// let (key0, value0) = items[0].clone(); -// let key0_items = key0.items(); -// assert_eq!(key0_items, [0x6b, 0x65, 0x79]); -// assert_eq!(value0.try_move(), Ok(0.0)); - -// let local = Local::default(); -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::Id(String::from("key")), -// JsonToken::Colon, -// JsonToken::Number(0.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(&local, tokens.into_iter()); -// assert!(result.is_err()); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_valid_local() { -// test_valid_with_manager(&Local::default()); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_valid_global() { -// test_valid_with_manager(GLOBAL); -// } - -// fn test_valid_with_manager(manager: M) { -// let tokens = [JsonToken::Id(String::from("null"))]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().any.get_type(), Type::Null); - -// let tokens = [JsonToken::Id(String::from("true"))]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().any.try_move(), Ok(true)); - -// let tokens = [JsonToken::Id(String::from("false"))]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().any.try_move(), Ok(false)); - -// let tokens = [JsonToken::Number(0.1)]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// assert_eq!(result.unwrap().any.try_move(), Ok(0.1)); - -// let tokens = [JsonToken::String(String::from("abc"))]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result = result.unwrap().any.try_move::>(); -// assert!(result.is_ok()); -// let result = result.unwrap(); -// let items = result.items(); -// assert_eq!(items, [0x61, 0x62, 0x63]); - -// let tokens = [JsonToken::BigInt(from_u64(manager, js::js_bigint::Sign::Positive, 1))]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result = result.unwrap().any.try_move::>(); -// assert!(result.is_ok()); -// let result = result.unwrap(); -// assert_eq!(result.header_len(), 1); -// let items = result.items(); -// assert_eq!(items, [0x1]); - -// let tokens = [JsonToken::BigInt(new_bigint(manager, js::js_bigint::Sign::Negative, [2, 3]))]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result = result.unwrap().any.try_move::>(); -// assert!(result.is_ok()); -// let result = result.unwrap(); -// assert_eq!(result.header_len(), -2); -// let items = result.items(); -// assert_eq!(items, [0x2, 0x3]); - -// let tokens = [JsonToken::ArrayBegin, JsonToken::ArrayEnd]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// assert!(items.is_empty()); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::Number(1.0), -// JsonToken::Comma, -// JsonToken::Id(String::from("true")), -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let item0 = items[0].clone(); -// assert_eq!(item0.try_move(), Ok(1.0)); -// let item1 = items[1].clone(); -// assert_eq!(item1.try_move(), Ok(true)); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::Number(1.0), -// JsonToken::Comma, -// JsonToken::Id(String::from("true")), -// JsonToken::Comma, -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("k1")), -// JsonToken::Colon, -// JsonToken::Number(1.0), -// JsonToken::Comma, -// JsonToken::String(String::from("k0")), -// JsonToken::Colon, -// JsonToken::Number(0.0), -// JsonToken::Comma, -// JsonToken::String(String::from("k2")), -// JsonToken::Colon, -// JsonToken::Number(2.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let (key0, value0) = items[0].clone(); -// let key0_items = key0.items(); -// assert_eq!(key0_items, [0x6b, 0x30]); -// assert_eq!(value0.try_move(), Ok(0.0)); -// let (key1, value1) = items[1].clone(); -// let key1_items = key1.items(); -// assert_eq!(key1_items, [0x6b, 0x31]); -// assert_eq!(value1.try_move(), Ok(1.0)); -// let (key2, value2) = items[2].clone(); -// let key2_items = key2.items(); -// assert_eq!(key2_items, [0x6b, 0x32]); -// assert_eq!(value2.try_move(), Ok(2.0)); - -// let tokens = [JsonToken::ObjectBegin, JsonToken::ObjectEnd]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result -// .unwrap() -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// assert!(items.is_empty()); -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("k")), -// JsonToken::Colon, -// JsonToken::ObjectBegin, -// JsonToken::ObjectEnd, -// JsonToken::ObjectEnd, -// ]; -// { -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_ok()); -// let result_unwrap = result.unwrap(); -// let result_unwrap = result_unwrap -// .any -// .try_move::>() -// .unwrap(); -// let items = result_unwrap.items(); -// let (_, value0) = items[0].clone(); -// let value0_unwrap = value0.try_move::>().unwrap(); -// let value0_items = value0_unwrap.items(); -// assert!(value0_items.is_empty()); -// } -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_invalid_local() { -// test_invalid_with_manager(&Local::default()); -// } - -// #[test] -// #[wasm_bindgen_test] -// fn test_invalid_global() { -// test_invalid_with_manager(GLOBAL); -// } - -// fn test_invalid_with_manager(manager: M) { -// let tokens = []; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [JsonToken::ErrorToken(ErrorType::InvalidNumber)]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [JsonToken::ArrayBegin, JsonToken::Comma, JsonToken::ArrayEnd]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::Number(0.0), -// JsonToken::Number(1.0), -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::Number(0.0), -// JsonToken::Comma, -// JsonToken::Comma, -// JsonToken::Number(1.0), -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::ArrayEnd, -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [JsonToken::ArrayBegin, JsonToken::String(String::default())]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::Comma, -// JsonToken::Number(1.0), -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [JsonToken::ArrayBegin, JsonToken::Colon, JsonToken::ArrayEnd]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [JsonToken::ArrayEnd]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::Comma, -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::Number(0.0), -// JsonToken::Comma, -// JsonToken::Number(1.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("key")), -// JsonToken::Number(0.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("key")), -// JsonToken::Colon, -// JsonToken::Colon, -// JsonToken::Number(0.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("key0")), -// JsonToken::Colon, -// JsonToken::Number(0.0), -// JsonToken::Comma, -// JsonToken::Comma, -// JsonToken::String(String::from("key1")), -// JsonToken::Colon, -// JsonToken::Number(1.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::ObjectEnd, -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::String(String::from("key")), -// JsonToken::Colon, -// JsonToken::Number(0.0), -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::Comma, -// JsonToken::String(String::from("key")), -// JsonToken::Colon, -// JsonToken::Number(0.0), -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [JsonToken::ObjectEnd]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ArrayBegin, -// JsonToken::ObjectBegin, -// JsonToken::ArrayEnd, -// JsonToken::ObjectEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); - -// let tokens = [ -// JsonToken::ObjectBegin, -// JsonToken::ArrayBegin, -// JsonToken::ObjectEnd, -// JsonToken::ArrayEnd, -// ]; -// let result = parse_with_virtual_io(manager, tokens.into_iter()); -// assert!(result.is_err()); -// } -// } +#[cfg(test)] +mod test { + use io_test::VirtualIo; + use io_trait::Io; + use wasm_bindgen_test::wasm_bindgen_test; + + use crate::{ + common::default::default, + js::{ + self, + js_array::JsArrayRef, + js_bigint::{from_u64, new_bigint, JsBigintRef}, + js_object::JsObjectRef, + js_string::JsStringRef, + type_::Type, + }, + mem::{global::GLOBAL, local::Local, manager::Manager}, + tokenizer::{tokenize, ErrorType, JsonToken}, + }; + + use super::super::{parser::parse, path::concat, shared::DataType}; + + use super::{parse_with_tokens, Context, ModuleCache, ParseError, ParseResult}; + + fn virtual_io() -> VirtualIo { + VirtualIo::new(&[]) + } + + fn create_test_context<'a, M: Manager>( + manager: M, + io: &'a VirtualIo, + module_cache: &'a mut ModuleCache, + ) -> Context<'a, M, VirtualIo> { + Context::new(manager, io, default(), module_cache) + } + + fn parse_with_virtual_io( + manager: M, + iter: impl Iterator>, + ) -> Result, ParseError> { + parse_with_tokens( + &mut create_test_context(manager, &virtual_io(), &mut default()), + iter, + ) + } + + fn test_global() { + let _ = { + let global = GLOBAL; + parse_with_tokens( + &mut create_test_context(global, &virtual_io(), &mut default()), + [].into_iter(), + ) + }; + } + + #[test] + #[wasm_bindgen_test] + fn test_json() { + let json_str = include_str!("../../test/test-json.json"); + let tokens = tokenize(GLOBAL, json_str.to_owned()); + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().data_type, DataType::Json); + } + + #[test] + #[wasm_bindgen_test] + fn test_djs() { + let json_str = include_str!("../../test/test-djs.d.cjs"); + let tokens = tokenize(GLOBAL, json_str.to_owned()); + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().data_type, DataType::Cjs); + + let json_str = include_str!("../../test/test-djs.d.mjs"); + let tokens = tokenize(GLOBAL, json_str.to_owned()); + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().data_type, DataType::Mjs); + } + + #[test] + #[wasm_bindgen_test] + fn test_const() { + test_const_with_manager(GLOBAL); + } + + fn test_const_with_manager(manager: M) { + let json_str = include_str!("../../test/test-const.d.cjs"); + let tokens = tokenize(manager, json_str.to_owned()); + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.try_move(), Ok(2.0)); + let item1 = items[1].clone(); + assert_eq!(item1.try_move(), Ok(2.0)); + + let json_str = include_str!("../../test/test-const-error.d.cjs.txt"); + let tokens = tokenize(manager, json_str.to_owned()); + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let json_str = include_str!("../../test/test-const-error-new-line.d.cjs.txt"); + let tokens = tokenize(manager, json_str.to_owned()); + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), ParseError::NewLineExpected); + } + + #[test] + #[wasm_bindgen_test] + fn test_stack() { + test_stack_with_manager(GLOBAL); + } + + fn test_stack_with_manager(manager: M) { + let json_str = include_str!("../../test/test-stack.d.cjs"); + let tokens = tokenize(manager, json_str.to_owned()); + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + let result_unwrap = item0.try_move::>().unwrap(); + let items = result_unwrap.items(); + let (key0, value0) = items[0].clone(); + let key0_items = key0.items(); + assert_eq!(key0_items, [0x61]); + let result_unwrap = value0.try_move::>().unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.get_type(), Type::Null); + } + + #[test] + #[wasm_bindgen_test] + fn test_import() { + test_import_with_manager(GLOBAL); + } + + fn test_import_with_manager(manager: M) { + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_import_main.d.cjs"); + //let path = "../../test/test-import-main.d.cjs"; + let main_path = "test_import_main.d.cjs"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module = include_str!("../../test/test_import_module.d.cjs"); + let module_path = "test_import_module.d.cjs"; + io.write(module_path, module.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.try_move(), Ok(3.0)); + + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_import_main.d.mjs"); + //let path = "../../test/test-import-main.d.mjs"; + let main_path = "test_import_main.d.mjs"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module = include_str!("../../test/test_import_module.d.mjs"); + let module_path = "test_import_module.d.mjs"; + io.write(module_path, module.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.try_move(), Ok(4.0)); + } + + #[test] + #[wasm_bindgen_test] + fn test_cache() { + test_cache_with_manager(GLOBAL); + } + + fn test_cache_with_manager(manager: M) { + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_cache_main.d.cjs"); + let main_path = "test_cache_main.d.cjs"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module_b = include_str!("../../test/test_cache_b.d.cjs"); + let module_b_path = "test_cache_b.d.cjs"; + io.write(module_b_path, module_b.as_bytes()).unwrap(); + + let module_c = include_str!("../../test/test_cache_c.d.cjs"); + let module_c_path = "test_cache_c.d.cjs"; + io.write(module_c_path, module_c.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.try_move(), Ok(1.0)); + let item1 = items[1].clone(); + assert_eq!(item1.try_move(), Ok(1.0)); + + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_cache_main.d.mjs"); + let main_path = "test_cache_main.d.mjs"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module_b = include_str!("../../test/test_cache_b.d.mjs"); + let module_b_path = "test_cache_b.d.mjs"; + io.write(module_b_path, module_b.as_bytes()).unwrap(); + + let module_c = include_str!("../../test/test_cache_c.d.mjs"); + let module_c_path = "test_cache_c.d.mjs"; + io.write(module_c_path, module_c.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.try_move(), Ok(2.0)); + let item1 = items[1].clone(); + assert_eq!(item1.try_move(), Ok(2.0)); + } + + #[test] + #[wasm_bindgen_test] + fn test_circular_error() { + test_circular_error_with_manager(GLOBAL); + } + + fn test_circular_error_with_manager(manager: M) { + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_circular_1.d.cjs.txt"); + let main_path = "test_circular_1.d.cjs.txt"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module = include_str!("../../test/test_circular_2.d.cjs.txt"); + let module_path = "test_circular_2.d.cjs.txt"; + io.write(module_path, module.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), ParseError::CircularDependency); + + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_circular_1.d.mjs.txt"); + let main_path = "test_circular_1.d.mjs.txt"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module = include_str!("../../test/test_circular_2.d.mjs.txt"); + let module_path = "test_circular_2.d.mjs.txt"; + io.write(module_path, module.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), ParseError::CircularDependency); + } + + #[test] + #[wasm_bindgen_test] + fn test_import_error() { + test_import_error_with_manager(GLOBAL); + } + + fn test_import_error_with_manager(manager: M) { + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_import_error.d.cjs.txt"); + //let path = "../../test/test-import-main.d.cjs"; + let main_path = "test_import_error.d.cjs.txt"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module = include_str!("../../test/test_import_module.d.mjs"); + let module_path = "test_import_module.d.mjs"; + io.write(module_path, module.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); + + let io: VirtualIo = VirtualIo::new(&[]); + + let main = include_str!("../../test/test_import_error.d.mjs.txt"); + //let path = "../../test/test-import-main.d.cjs"; + let main_path = "test_import_error.d.mjs.txt"; + io.write(main_path, main.as_bytes()).unwrap(); + + let module = include_str!("../../test/test_import_module.d.cjs"); + let module_path = "test_import_module.d.cjs"; + io.write(module_path, module.as_bytes()).unwrap(); + + let mut mc = default(); + let mut context = Context::new( + manager, + &io, + concat(io.current_dir().unwrap().as_str(), main_path), + &mut mc, + ); + + let result = parse(&mut context); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), ParseError::UnexpectedToken); + } + + #[test] + #[wasm_bindgen_test] + fn test_trailing_comma() { + let json_str = include_str!("../../test/test-trailing-comma.d.cjs"); + let tokens = tokenize(GLOBAL, json_str.to_owned()); + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + } + + #[test] + #[wasm_bindgen_test] + fn test_check_sizes() { + { + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("k")), + JsonToken::Colon, + JsonToken::ObjectBegin, + JsonToken::ObjectEnd, + JsonToken::ObjectEnd, + ]; + { + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + let _result_unwrap = result.unwrap(); + } + //assert_eq!(GLOBAL.size(), 0); + } + } + + #[test] + #[wasm_bindgen_test] + fn test_check_sizes2() { + let local = Local::default(); + { + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("k")), + JsonToken::Colon, + JsonToken::ObjectBegin, + JsonToken::ObjectEnd, + JsonToken::ObjectEnd, + ]; + { + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result.unwrap().any; + let _result_unwrap = result_unwrap.try_move::>(); + } + assert_eq!(local.size(), 0); + } + } + + #[test] + #[wasm_bindgen_test] + fn test_data_type() { + let tokens = [JsonToken::Id(String::from("null"))]; + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().data_type, DataType::Json); + } + + #[test] + #[wasm_bindgen_test] + fn test_export_block() { + let tokens = [ + JsonToken::Id(String::from("export")), + JsonToken::Id(String::from("default")), + JsonToken::Id(String::from("null")), + ]; + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().data_type, DataType::Mjs); + + let tokens = [ + JsonToken::Id(String::from("module")), + JsonToken::Dot, + JsonToken::Id(String::from("exports")), + JsonToken::Equals, + JsonToken::Id(String::from("null")), + ]; + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().data_type, DataType::Cjs); + } + + #[test] + #[wasm_bindgen_test] + fn test_id_in_objects() { + let tokens = [ + JsonToken::Id(String::from("export")), + JsonToken::Id(String::from("default")), + JsonToken::ObjectBegin, + JsonToken::Id(String::from("key")), + JsonToken::Colon, + JsonToken::Number(0.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result.unwrap().any.try_move::>().unwrap(); + let items = result_unwrap.items(); + let (key0, value0) = items[0].clone(); + let key0_items = key0.items(); + assert_eq!(key0_items, [0x6b, 0x65, 0x79]); + assert_eq!(value0.try_move(), Ok(0.0)); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::Id(String::from("key")), + JsonToken::Colon, + JsonToken::Number(0.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(GLOBAL, tokens.into_iter()); + assert!(result.is_err()); + } + + #[test] + #[wasm_bindgen_test] + fn test_valid_global() { + test_valid_with_manager(GLOBAL); + } + + fn test_valid_with_manager(manager: M) { + let tokens = [JsonToken::Id(String::from("null"))]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().any.get_type(), Type::Null); + + let tokens = [JsonToken::Id(String::from("true"))]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().any.try_move(), Ok(true)); + + let tokens = [JsonToken::Id(String::from("false"))]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().any.try_move(), Ok(false)); + + let tokens = [JsonToken::Number(0.1)]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + assert_eq!(result.unwrap().any.try_move(), Ok(0.1)); + + let tokens = [JsonToken::String(String::from("abc"))]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result = result.unwrap().any.try_move::>(); + assert!(result.is_ok()); + let result = result.unwrap(); + let items = result.items(); + assert_eq!(items, [0x61, 0x62, 0x63]); + + let tokens = [JsonToken::BigInt(from_u64( + manager, + js::js_bigint::Sign::Positive, + 1, + ))]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result = result.unwrap().any.try_move::>(); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.header_len(), 1); + let items = result.items(); + assert_eq!(items, [0x1]); + + let tokens = [JsonToken::BigInt(new_bigint( + manager, + js::js_bigint::Sign::Negative, + [2, 3], + ))]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result = result.unwrap().any.try_move::>(); + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.header_len(), -2); + let items = result.items(); + assert_eq!(items, [0x2, 0x3]); + + let tokens = [JsonToken::ArrayBegin, JsonToken::ArrayEnd]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + assert!(items.is_empty()); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::Number(1.0), + JsonToken::Comma, + JsonToken::Id(String::from("true")), + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let item0 = items[0].clone(); + assert_eq!(item0.try_move(), Ok(1.0)); + let item1 = items[1].clone(); + assert_eq!(item1.try_move(), Ok(true)); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::Number(1.0), + JsonToken::Comma, + JsonToken::Id(String::from("true")), + JsonToken::Comma, + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("k1")), + JsonToken::Colon, + JsonToken::Number(1.0), + JsonToken::Comma, + JsonToken::String(String::from("k0")), + JsonToken::Colon, + JsonToken::Number(0.0), + JsonToken::Comma, + JsonToken::String(String::from("k2")), + JsonToken::Colon, + JsonToken::Number(2.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let (key0, value0) = items[0].clone(); + let key0_items = key0.items(); + assert_eq!(key0_items, [0x6b, 0x30]); + assert_eq!(value0.try_move(), Ok(0.0)); + let (key1, value1) = items[1].clone(); + let key1_items = key1.items(); + assert_eq!(key1_items, [0x6b, 0x31]); + assert_eq!(value1.try_move(), Ok(1.0)); + let (key2, value2) = items[2].clone(); + let key2_items = key2.items(); + assert_eq!(key2_items, [0x6b, 0x32]); + assert_eq!(value2.try_move(), Ok(2.0)); + + let tokens = [JsonToken::ObjectBegin, JsonToken::ObjectEnd]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result + .unwrap() + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + assert!(items.is_empty()); + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("k")), + JsonToken::Colon, + JsonToken::ObjectBegin, + JsonToken::ObjectEnd, + JsonToken::ObjectEnd, + ]; + { + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_ok()); + let result_unwrap = result.unwrap(); + let result_unwrap = result_unwrap + .any + .try_move::>() + .unwrap(); + let items = result_unwrap.items(); + let (_, value0) = items[0].clone(); + let value0_unwrap = value0.try_move::>().unwrap(); + let value0_items = value0_unwrap.items(); + assert!(value0_items.is_empty()); + } + } + + #[test] + #[wasm_bindgen_test] + fn test_invalid_global() { + test_invalid_with_manager(GLOBAL); + } + + fn test_invalid_with_manager(manager: M) { + let tokens = []; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [JsonToken::ErrorToken(ErrorType::InvalidNumber)]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [JsonToken::ArrayBegin, JsonToken::Comma, JsonToken::ArrayEnd]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::Number(0.0), + JsonToken::Number(1.0), + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::Number(0.0), + JsonToken::Comma, + JsonToken::Comma, + JsonToken::Number(1.0), + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::ArrayEnd, + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [JsonToken::ArrayBegin, JsonToken::String(String::default())]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::Comma, + JsonToken::Number(1.0), + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [JsonToken::ArrayBegin, JsonToken::Colon, JsonToken::ArrayEnd]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [JsonToken::ArrayEnd]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::Comma, + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::Number(0.0), + JsonToken::Comma, + JsonToken::Number(1.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("key")), + JsonToken::Number(0.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("key")), + JsonToken::Colon, + JsonToken::Colon, + JsonToken::Number(0.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("key0")), + JsonToken::Colon, + JsonToken::Number(0.0), + JsonToken::Comma, + JsonToken::Comma, + JsonToken::String(String::from("key1")), + JsonToken::Colon, + JsonToken::Number(1.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::ObjectEnd, + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::String(String::from("key")), + JsonToken::Colon, + JsonToken::Number(0.0), + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::Comma, + JsonToken::String(String::from("key")), + JsonToken::Colon, + JsonToken::Number(0.0), + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [JsonToken::ObjectEnd]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ArrayBegin, + JsonToken::ObjectBegin, + JsonToken::ArrayEnd, + JsonToken::ObjectEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + + let tokens = [ + JsonToken::ObjectBegin, + JsonToken::ArrayBegin, + JsonToken::ObjectEnd, + JsonToken::ArrayEnd, + ]; + let result = parse_with_virtual_io(manager, tokens.into_iter()); + assert!(result.is_err()); + } +} From 13c937c65cf6e9b05e935e3a96615427045c5339 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 23 Nov 2024 17:15:44 +0200 Subject: [PATCH 16/28] jsbigint cmp --- nanvm-lib/src/js/js_bigint.rs | 43 ++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/nanvm-lib/src/js/js_bigint.rs b/nanvm-lib/src/js/js_bigint.rs index b7169b87..76a18267 100644 --- a/nanvm-lib/src/js/js_bigint.rs +++ b/nanvm-lib/src/js/js_bigint.rs @@ -126,6 +126,16 @@ pub fn is_zero(value: &JsBigint) -> bool { value.items().is_empty() } +impl JsBigint { + pub fn cmp(&self, other: &Self) -> Ordering { + match self.header_len().cmp(&other.header_len()) { + Ordering::Equal => cmp_vec(self.items(), other.items()), + Ordering::Less => Ordering::Less, + Ordering::Greater => Ordering::Greater, + } + } +} + pub fn negative(m: M, value: &JsBigint) -> JsBigintMutRef { if is_zero(value) { return zero(m); @@ -594,7 +604,7 @@ fn shr_on_big(m: M, sign: Sign) -> JsBigintMutRef { #[cfg(test)] mod test { - use std::ops::Deref; + use std::{cmp::Ordering, ops::Deref}; use wasm_bindgen_test::wasm_bindgen_test; @@ -744,6 +754,37 @@ mod test { } } + #[test] + #[wasm_bindgen_test] + fn test_cmp() { + type A = Any; + type BigintRef = JsBigintRef; + + let a_ref = from_u64(Global(), Sign::Positive, 1); + let b_ref = from_u64(Global(), Sign::Positive, 1); + let a = a_ref.deref(); + let b = b_ref.deref(); + assert_eq!(a.cmp(b), Ordering::Equal); + + let a_ref = from_u64(Global(), Sign::Positive, 1); + let b_ref = from_u64(Global(), Sign::Positive, 2); + let a = a_ref.deref(); + let b = b_ref.deref(); + assert_eq!(a.cmp(b), Ordering::Less); + + let a_ref = from_u64(Global(), Sign::Positive, 1); + let b_ref = from_u64(Global(), Sign::Negative, 2); + let a = a_ref.deref(); + let b = b_ref.deref(); + assert_eq!(a.cmp(b), Ordering::Greater); + + let a_ref = new_bigint(Global(), Sign::Positive, [1, 2]); + let b_ref = new_bigint(Global(), Sign::Positive, [2, 1]); + let a = a_ref.deref(); + let b = b_ref.deref(); + assert_eq!(a.cmp(b), Ordering::Greater); + } + #[test] #[wasm_bindgen_test] fn test_shl_zero() { From 2abc76ce3b015be9d4f599286827a7803acd37d5 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 24 Nov 2024 20:41:50 +0200 Subject: [PATCH 17/28] big float --- nanvm-lib/src/big_numbers/big_float.rs | 1398 +++++++++++++----------- nanvm-lib/src/js/js_bigint.rs | 11 +- nanvm-lib/src/tokenizer/mod.rs | 550 +++++----- 3 files changed, 1026 insertions(+), 933 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index 3a924ac8..5d8db0a0 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -1,72 +1,93 @@ -use crate::{big_numbers::big_int::Sign, common::cast::Cast}; - -use super::{big_int::BigInt, big_uint::BigUint}; - -#[derive(Debug, PartialEq, Clone, Eq, Default)] -pub struct BigFloat { - pub significand: BigInt, +use std::{cmp::Ordering, ops::Deref}; + +use crate::{ + common::cast::Cast, + js::js_bigint::{ + div_mod, from_u64, is_zero, mul, pow, pow_u64, shl, shl_on_u64, shr_on_u64, zero, JsBigint, + JsBigintMutRef, Sign, + }, + mem::manager::Manager, +}; + +#[derive(Debug)] +pub struct BigFloat { + pub manager: M, + pub significand: JsBigintMutRef, pub exp: i64, pub non_zero_reminder: bool, } -impl BigFloat { - pub const ZERO: BigFloat = BigFloat { - significand: BigInt::ZERO, +pub fn float_zero(manager: M) -> BigFloat { + BigFloat { + manager, + significand: zero(manager), exp: 0, non_zero_reminder: false, - }; + } +} +impl BigFloat { fn increase_significand(&mut self, precision: u64) { - if self.significand.is_zero() { + if is_zero(self.significand.deref()) { return; } - let min_significand = &BigUint::one() << &BigUint::from_u64(precision); - self.increase_significand_to(&min_significand); + let min_significand = shl( + self.manager, + from_u64(self.manager, Sign::Positive, 1).deref(), + from_u64(self.manager, Sign::Positive, precision).deref(), + ); + self.increase_significand_to(min_significand.deref()); } - fn increase_significand_to(&mut self, min_significand: &BigUint) { - if self.significand.is_zero() { + fn increase_significand_to(&mut self, min_significand: &JsBigint) { + if is_zero(self.significand.deref()) { return; } loop { - if self.significand.value >= *min_significand { - return; + match self.significand.deref().cmp(min_significand) { + Ordering::Greater | Ordering::Equal => return, + _ => {} } - self.significand.value = &self.significand.value << &BigUint::one(); + self.significand = shl_on_u64(self.manager, self.significand.deref(), 1); self.exp -= 1; } } fn decrease_significand(&mut self, precision: u64) { - if self.significand.is_zero() { + if is_zero(self.significand.deref()) { return; } - let max_significand = &BigUint::one() << &BigUint::from_u64(precision); + let max_significand = shl( + self.manager, + from_u64(self.manager, Sign::Positive, 1).deref(), + from_u64(self.manager, Sign::Positive, precision).deref(), + ); loop { - if self.significand.value < max_significand { + if self.significand.deref().cmp(max_significand.deref()) == Ordering::Less { break; } - let last_bit = self.significand.value.get_last_bit(); + let last_bit = self.significand.get_last_bit(); if last_bit == 1 { self.non_zero_reminder = true; } - self.significand.value = &self.significand.value >> &BigUint::one(); + self.significand = shr_on_u64(self.manager, self.significand.deref(), 1); self.exp += 1; } } } -impl BigFloat<10> { - pub fn to_bin(self, precision: u8) -> BigFloat<2> { - if self.significand.is_zero() { - return BigFloat::ZERO; +impl BigFloat<10, M> { + pub fn to_bin(self, precision: u8) -> BigFloat<2, M> { + if is_zero(self.significand.deref()) { + return float_zero(self.manager); } if self.exp == 0 { - let mut result: BigFloat<2> = BigFloat { + let mut result: BigFloat<2, M> = BigFloat { + manager: self.manager, significand: self.significand, exp: self.exp, non_zero_reminder: self.non_zero_reminder, @@ -76,10 +97,15 @@ impl BigFloat<10> { return result; } - let five = BigUint { value: [5].cast() }; + let five = from_u64(self.manager, Sign::Positive, 5); if self.exp > 0 { - let new_sign = &self.significand * &five.pow_u64(self.exp as u64).to_big_int(); - let mut result: BigFloat<2> = BigFloat { + let new_sign = mul( + self.manager, + self.significand.deref(), + pow_u64(self.manager, five.deref(), self.exp as u64).deref(), + ); + let mut result: BigFloat<2, M> = BigFloat { + manager: self.manager, significand: new_sign, exp: self.exp, non_zero_reminder: self.non_zero_reminder, @@ -89,26 +115,38 @@ impl BigFloat<10> { return result; } - let p = five.pow_u64(-self.exp as u64); - let mut bf10 = self.clone(); - let min_significand = &BigUint::one() << &BigUint::from_u64(precision as u64); - bf10.increase_significand_to(&(&p * &min_significand)); + let p = pow_u64(self.manager, five.deref(), -self.exp as u64); + let mut bf10: BigFloat<10, M> = BigFloat { + manager: self.manager, + significand: self.significand, + exp: self.exp, + non_zero_reminder: self.non_zero_reminder, + }; + let min_significand = shl( + self.manager, + from_u64(self.manager, Sign::Positive, 1).deref(), + from_u64(self.manager, Sign::Positive, precision as u64).deref(), + ); + bf10.increase_significand_to( + (mul(self.manager, p.deref(), min_significand.deref())).deref(), + ); - let (q, r) = bf10.significand.div_mod(&p.to_big_int()); - let mut result: BigFloat<2> = BigFloat { + let (q, r) = div_mod(self.manager, bf10.significand.deref(), p.deref()); + let mut result: BigFloat<2, M> = BigFloat { + manager: self.manager, significand: q, exp: bf10.exp, - non_zero_reminder: self.non_zero_reminder || !r.is_zero(), + non_zero_reminder: self.non_zero_reminder || !is_zero(r.deref()), }; result.decrease_significand(precision as u64); result } } -impl BigFloat<2> { +impl BigFloat<2, M> { fn get_frac_round(self) -> u64 { - let mut last_bit = self.significand.value.get_last_bit(); - let mut frac = self.significand.value.value[0] >> 1; + let mut last_bit = self.significand.get_last_bit(); + let mut frac = self.significand.items()[0] >> 1; if last_bit == 1 && !self.non_zero_reminder { last_bit = frac & 1; @@ -132,15 +170,20 @@ impl BigFloat<2> { const INF_BITS: u64 = 2047 << 52; let mut bits: u64 = 0; - if self.significand.sign == Sign::Negative { + if self.significand.header_len() < 0 { bits |= 1 << 63; } - if self.significand.is_zero() { + if is_zero(self.significand.deref()) { return bits; } - let mut value = self.clone(); + let mut value = BigFloat { + manager: self.manager, + significand: self.significand, + exp: self.exp, + non_zero_reminder: self.non_zero_reminder, + }; value.increase_significand(PRECISION + 1); value.decrease_significand(PRECISION + 2); @@ -181,14 +224,19 @@ impl BigFloat<2> { #[cfg(test)] mod test { + use std::ops::Deref; + use wasm_bindgen_test::wasm_bindgen_test; use crate::{ - big_numbers::{ - big_int::{BigInt, Sign}, - big_uint::BigUint, - }, + big_numbers::big_float::float_zero, common::cast::Cast, + js::{ + any::Any, + js_bigint::{self, from_u64, zero, JsBigintRef, Sign}, + type_::Type, + }, + mem::global::{Global, GLOBAL}, }; use super::BigFloat; @@ -196,635 +244,669 @@ mod test { #[test] #[wasm_bindgen_test] fn test_zero() { - let res = BigFloat::ZERO.to_bin(64); - assert_eq!(res, BigFloat::ZERO); + type A = Any; + type BigintRef = JsBigintRef; + + let res = float_zero(GLOBAL).to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert!(o.items().is_empty()); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); let res = BigFloat { - significand: BigInt::ZERO, + manager: GLOBAL, + significand: zero(GLOBAL), exp: 10, - non_zero_reminder: false, + non_zero_reminder: false + }.to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert!(o.items().is_empty()); } - .to_bin(64); - assert_eq!(res, BigFloat::ZERO); + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); let res = BigFloat { - significand: BigInt::ZERO, + manager: GLOBAL, + significand: zero(GLOBAL), exp: -10, - non_zero_reminder: false, + non_zero_reminder: false + }.to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert!(o.items().is_empty()); } - .to_bin(64); - assert_eq!(res, BigFloat::ZERO); + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); } #[test] #[wasm_bindgen_test] fn test_integer() { - let a = BigFloat { - significand: BigInt::from_i64(100), + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), exp: 0, - non_zero_reminder: false, - }; - let res = a.to_bin(7); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(100), - exp: 0, - non_zero_reminder: false - } - ); + non_zero_reminder: false + }.to_bin(7); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[100]); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); - let a = BigFloat { - significand: BigInt::from_i64(1), + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), exp: 1, - non_zero_reminder: false, - }; - let res = a.to_bin(64); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_u64(10 << 60), - exp: -60, - non_zero_reminder: false - } - ); + non_zero_reminder: false + }.to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[10 << 60]); + } + assert_eq!(res.exp, -60); + assert_eq!(res.non_zero_reminder, false); - let a = BigFloat { - significand: BigInt::from_i64(100), + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), exp: 2, - non_zero_reminder: false, - }; - let res = a.to_bin(64); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_u64(10000 << 50), - exp: -50, - non_zero_reminder: false - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(128), - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_bin(9); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(256), - exp: -1, - non_zero_reminder: false - } - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_integer_rounding() { - let a = BigFloat { - significand: BigInt::from_i64(128), - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_bin(4); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(8), - exp: 4, - non_zero_reminder: false - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(129), - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_bin(4); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(8), - exp: 4, - non_zero_reminder: true - } - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_float() { - let a = BigFloat { - significand: BigInt::from_i64(100), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(5); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(20), - exp: -1, - non_zero_reminder: false - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(100), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(64); - assert_eq!( - res, - BigFloat { - significand: BigInt { - sign: Sign::Positive, - value: BigUint { - value: [(1 << 63) + (1 << 61)].cast() - } - }, - exp: -60, - non_zero_reminder: false - } - ); - - let a = BigFloat { - significand: BigInt { - sign: Sign::Positive, - value: BigUint { - value: [0, 1].cast(), - }, - }, - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_bin(53); - assert_eq!( - res, - BigFloat { - significand: BigInt { - sign: Sign::Positive, - value: BigUint { - value: [1 << 52].cast() - } - }, - exp: 12, - non_zero_reminder: false - } - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_rounding() { - let a = BigFloat { - significand: BigInt::from_i64(0b1000_0001), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(5); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b11001), - exp: -1, - non_zero_reminder: true - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(0b1000_0001), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(4); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b1100), - exp: 0, - non_zero_reminder: true - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(0b1000_0001), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(3); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b110), - exp: 1, - non_zero_reminder: true - } - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_rounding_half() { - let a = BigFloat { - significand: BigInt::from_i64(0b101_1010), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(3); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b100), - exp: 1, - non_zero_reminder: true - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(0b101_1011), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(3); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b100), - exp: 1, - non_zero_reminder: true - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(0b110_1101), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(3); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b101), - exp: 1, - non_zero_reminder: true - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(0b110_1110), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(3); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b101), - exp: 1, - non_zero_reminder: true - } - ); - - let a = BigFloat { - significand: BigInt::from_i64(0b1001_0110), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_bin(3); - assert_eq!( - res, - BigFloat { - significand: BigInt::from_i64(0b111), - exp: 1, - non_zero_reminder: true - } - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_zero_to_f64() { - let a = BigFloat { - significand: BigInt::ZERO, - exp: 100, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 0.0); - assert!(res.is_sign_positive()); - - let a = BigFloat { - significand: BigInt { - sign: Sign::Negative, - value: BigUint::ZERO, - }, - exp: 100, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 0.0); - assert!(res.is_sign_negative()); - } - - #[test] - #[wasm_bindgen_test] - fn test_normal_to_f64() { - let a = BigFloat { - significand: BigInt::from_u64(1), - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 1.0); - - let a = BigFloat { - significand: BigInt::from_i64(-3), - exp: -1, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, -1.5); - - let a = BigFloat { - significand: BigInt::from_i64(1), - exp: -1022, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 2.0f64.powf(-1022.0)); - assert!(res.is_normal()); - - let a = BigFloat { - significand: BigInt::from_u64(1 << 59), - exp: -1022 - 59, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 2.0f64.powf(-1022.0)); - assert!(res.is_normal()); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 60) - 1), - exp: -1022 - 60, - non_zero_reminder: true, - }; - let res = a.to_f64(); - assert_eq!(res, 2.0f64.powf(-1022.0)); - assert!(res.is_normal()); - - let a = BigFloat { - significand: BigInt::from_i64(1), - exp: 1023, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 2.0f64.powf(1023.0)); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 52) - 1), - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 4503599627370495f64); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 53) - 1), - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 9007199254740991f64); - } - - #[test] - #[wasm_bindgen_test] - fn test_normal_to_f64_rounding() { - let a = BigFloat { - significand: BigInt::from_u64((1 << 54) - 1), //111111111111111111111111111111111111111111111111111111 - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 18014398509481984f64); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 54) - 2), //111111111111111111111111111111111111111111111111111110 - exp: 0, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 18014398509481982f64); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 - exp: 0, - non_zero_reminder: true, - }; - let res = a.to_f64(); - assert_eq!(res, 18014398509481982f64); + non_zero_reminder: false + }.to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[10000 << 50]); + } + assert_eq!(res.exp, -50); + assert_eq!(res.non_zero_reminder, false); - let a = BigFloat { - significand: BigInt::from_u64((1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 128), exp: 0, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 18014398509481980f64); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 54) - 1), - exp: 969, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert!(res.is_normal()); - - let a = BigFloat { - significand: BigInt::from_u64((1 << 54) - 1), - exp: 970, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert!(res.is_infinite()); - } - - #[test] - #[wasm_bindgen_test] - fn test_infinity_to_f64() { - let a = BigFloat { - significand: BigInt::from_i64(1), - exp: 1024, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert!(res.is_infinite()); - assert!(res.is_sign_positive()); - - let a = BigFloat { - significand: BigInt::from_i64(-1), - exp: 1024, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert!(res.is_infinite()); - assert!(res.is_sign_negative()); - } - - #[test] - #[wasm_bindgen_test] - fn test_subnormal_to_f64() { - let a = BigFloat { - significand: BigInt::from_u64(1), - exp: -1023, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 2.0f64.powf(-1023.0)); - assert!(res.is_subnormal()); - - let a = BigFloat { - significand: BigInt::from_i64(-1), - exp: -1023, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, -(2.0f64.powf(-1023.0))); - assert!(res.is_subnormal()); - - let a = BigFloat { - significand: BigInt::from_u64(1), - exp: -1074, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 1); - assert_eq!(res, 2.0f64.powf(-1074.0)); - assert!(res.is_subnormal()); - - let a = BigFloat { - significand: BigInt::from_u64(1), - exp: -1075, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res, 0.0); - } - - #[test] - #[wasm_bindgen_test] - fn test_subnormal_to_f64_rounding() { - //0.0 => 0 - let a = BigFloat { - significand: BigInt::from_u64(0b100), - exp: -1075, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b10); - assert_eq!(res, 2.0f64.powf(-1073.0)); - assert!(res.is_subnormal()); - - //0.0+ => 0 - let a = BigFloat { - significand: BigInt::from_u64(0b100), - exp: -1075, - non_zero_reminder: true, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b10); - assert_eq!(res, 2.0f64.powf(-1073.0)); - assert!(res.is_subnormal()); - - //0.1 => 0 - let a = BigFloat { - significand: BigInt::from_u64(0b101), - exp: -1075, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b10); - assert_eq!(res, 2.0f64.powf(-1073.0)); - assert!(res.is_subnormal()); - - //0.1+ => 1 - let a = BigFloat { - significand: BigInt::from_u64(0b101), - exp: -1075, - non_zero_reminder: true, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b11); - assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); - assert!(res.is_subnormal()); - - //1.0 => 1 - let a = BigFloat { - significand: BigInt::from_u64(0b110), - exp: -1075, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b11); - assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); - assert!(res.is_subnormal()); - - //1.0+ => 1 - let a = BigFloat { - significand: BigInt::from_u64(0b110), - exp: -1075, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b11); - assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); - assert!(res.is_subnormal()); - - //1.1 => 2 - let a = BigFloat { - significand: BigInt::from_u64(0b111), - exp: -1075, - non_zero_reminder: false, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b100); - assert_eq!(res, 2.0f64.powf(-1072.0)); - assert!(res.is_subnormal()); - - //1.1+ => 2 - let a = BigFloat { - significand: BigInt::from_u64(0b111), - exp: -1075, - non_zero_reminder: true, - }; - let res = a.to_f64(); - assert_eq!(res.to_bits(), 0b100); - assert_eq!(res, 2.0f64.powf(-1072.0)); - assert!(res.is_subnormal()); - } - - #[test] - #[wasm_bindgen_test] - fn test_rust_cast() { - test(18014398509481981); - test(18014398509481982); - test(18014398509481983); - test(18014398509481984); - test(18014398509481985); - - fn test(n: u64) { - let big_float = BigFloat { - significand: BigInt::from_u64(n), - exp: 0, - non_zero_reminder: false, - }; - let f64 = big_float.to_f64(); - assert_eq!(f64, n as f64); + non_zero_reminder: false + }.to_bin(9); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[256]); } + assert_eq!(res.exp, -1); + assert_eq!(res.non_zero_reminder, false); } + + // #[test] + // #[wasm_bindgen_test] + // fn test_integer_rounding() { + // let a = BigFloat { + // significand: BigInt::from_i64(128), + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(4); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(8), + // exp: 4, + // non_zero_reminder: false + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(129), + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(4); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(8), + // exp: 4, + // non_zero_reminder: true + // } + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_float() { + // let a = BigFloat { + // significand: BigInt::from_i64(100), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(5); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(20), + // exp: -1, + // non_zero_reminder: false + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(100), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(64); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt { + // sign: Sign::Positive, + // value: BigUint { + // value: [(1 << 63) + (1 << 61)].cast() + // } + // }, + // exp: -60, + // non_zero_reminder: false + // } + // ); + + // let a = BigFloat { + // significand: BigInt { + // sign: Sign::Positive, + // value: BigUint { + // value: [0, 1].cast(), + // }, + // }, + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(53); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt { + // sign: Sign::Positive, + // value: BigUint { + // value: [1 << 52].cast() + // } + // }, + // exp: 12, + // non_zero_reminder: false + // } + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_rounding() { + // let a = BigFloat { + // significand: BigInt::from_i64(0b1000_0001), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(5); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b11001), + // exp: -1, + // non_zero_reminder: true + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(0b1000_0001), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(4); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b1100), + // exp: 0, + // non_zero_reminder: true + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(0b1000_0001), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(3); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b110), + // exp: 1, + // non_zero_reminder: true + // } + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_rounding_half() { + // let a = BigFloat { + // significand: BigInt::from_i64(0b101_1010), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(3); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b100), + // exp: 1, + // non_zero_reminder: true + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(0b101_1011), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(3); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b100), + // exp: 1, + // non_zero_reminder: true + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(0b110_1101), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(3); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b101), + // exp: 1, + // non_zero_reminder: true + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(0b110_1110), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(3); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b101), + // exp: 1, + // non_zero_reminder: true + // } + // ); + + // let a = BigFloat { + // significand: BigInt::from_i64(0b1001_0110), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_bin(3); + // assert_eq!( + // res, + // BigFloat { + // significand: BigInt::from_i64(0b111), + // exp: 1, + // non_zero_reminder: true + // } + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_zero_to_f64() { + // let a = BigFloat { + // significand: BigInt::ZERO, + // exp: 100, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 0.0); + // assert!(res.is_sign_positive()); + + // let a = BigFloat { + // significand: BigInt { + // sign: Sign::Negative, + // value: BigUint::ZERO, + // }, + // exp: 100, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 0.0); + // assert!(res.is_sign_negative()); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_normal_to_f64() { + // let a = BigFloat { + // significand: BigInt::from_u64(1), + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 1.0); + + // let a = BigFloat { + // significand: BigInt::from_i64(-3), + // exp: -1, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, -1.5); + + // let a = BigFloat { + // significand: BigInt::from_i64(1), + // exp: -1022, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 2.0f64.powf(-1022.0)); + // assert!(res.is_normal()); + + // let a = BigFloat { + // significand: BigInt::from_u64(1 << 59), + // exp: -1022 - 59, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 2.0f64.powf(-1022.0)); + // assert!(res.is_normal()); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 60) - 1), + // exp: -1022 - 60, + // non_zero_reminder: true, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 2.0f64.powf(-1022.0)); + // assert!(res.is_normal()); + + // let a = BigFloat { + // significand: BigInt::from_i64(1), + // exp: 1023, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 2.0f64.powf(1023.0)); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 52) - 1), + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 4503599627370495f64); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 53) - 1), + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 9007199254740991f64); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_normal_to_f64_rounding() { + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 54) - 1), //111111111111111111111111111111111111111111111111111111 + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 18014398509481984f64); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 54) - 2), //111111111111111111111111111111111111111111111111111110 + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 18014398509481982f64); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 + // exp: 0, + // non_zero_reminder: true, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 18014398509481982f64); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 + // exp: 0, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 18014398509481980f64); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 54) - 1), + // exp: 969, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert!(res.is_normal()); + + // let a = BigFloat { + // significand: BigInt::from_u64((1 << 54) - 1), + // exp: 970, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert!(res.is_infinite()); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_infinity_to_f64() { + // let a = BigFloat { + // significand: BigInt::from_i64(1), + // exp: 1024, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert!(res.is_infinite()); + // assert!(res.is_sign_positive()); + + // let a = BigFloat { + // significand: BigInt::from_i64(-1), + // exp: 1024, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert!(res.is_infinite()); + // assert!(res.is_sign_negative()); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_subnormal_to_f64() { + // let a = BigFloat { + // significand: BigInt::from_u64(1), + // exp: -1023, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 2.0f64.powf(-1023.0)); + // assert!(res.is_subnormal()); + + // let a = BigFloat { + // significand: BigInt::from_i64(-1), + // exp: -1023, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, -(2.0f64.powf(-1023.0))); + // assert!(res.is_subnormal()); + + // let a = BigFloat { + // significand: BigInt::from_u64(1), + // exp: -1074, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 1); + // assert_eq!(res, 2.0f64.powf(-1074.0)); + // assert!(res.is_subnormal()); + + // let a = BigFloat { + // significand: BigInt::from_u64(1), + // exp: -1075, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res, 0.0); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_subnormal_to_f64_rounding() { + // 0.0 => 0 + // let a = BigFloat { + // significand: BigInt::from_u64(0b100), + // exp: -1075, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b10); + // assert_eq!(res, 2.0f64.powf(-1073.0)); + // assert!(res.is_subnormal()); + + // 0.0+ => 0 + // let a = BigFloat { + // significand: BigInt::from_u64(0b100), + // exp: -1075, + // non_zero_reminder: true, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b10); + // assert_eq!(res, 2.0f64.powf(-1073.0)); + // assert!(res.is_subnormal()); + + // 0.1 => 0 + // let a = BigFloat { + // significand: BigInt::from_u64(0b101), + // exp: -1075, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b10); + // assert_eq!(res, 2.0f64.powf(-1073.0)); + // assert!(res.is_subnormal()); + + // 0.1+ => 1 + // let a = BigFloat { + // significand: BigInt::from_u64(0b101), + // exp: -1075, + // non_zero_reminder: true, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b11); + // assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); + // assert!(res.is_subnormal()); + + // 1.0 => 1 + // let a = BigFloat { + // significand: BigInt::from_u64(0b110), + // exp: -1075, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b11); + // assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); + // assert!(res.is_subnormal()); + + // 1.0+ => 1 + // let a = BigFloat { + // significand: BigInt::from_u64(0b110), + // exp: -1075, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b11); + // assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); + // assert!(res.is_subnormal()); + + // 1.1 => 2 + // let a = BigFloat { + // significand: BigInt::from_u64(0b111), + // exp: -1075, + // non_zero_reminder: false, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b100); + // assert_eq!(res, 2.0f64.powf(-1072.0)); + // assert!(res.is_subnormal()); + + // 1.1+ => 2 + // let a = BigFloat { + // significand: BigInt::from_u64(0b111), + // exp: -1075, + // non_zero_reminder: true, + // }; + // let res = a.to_f64(); + // assert_eq!(res.to_bits(), 0b100); + // assert_eq!(res, 2.0f64.powf(-1072.0)); + // assert!(res.is_subnormal()); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_rust_cast() { + // test(18014398509481981); + // test(18014398509481982); + // test(18014398509481983); + // test(18014398509481984); + // test(18014398509481985); + + // fn test(n: u64) { + // let big_float = BigFloat { + // significand: BigInt::from_u64(n), + // exp: 0, + // non_zero_reminder: false, + // }; + // let f64 = big_float.to_f64(); + // assert_eq!(f64, n as f64); + // } + // } } diff --git a/nanvm-lib/src/js/js_bigint.rs b/nanvm-lib/src/js/js_bigint.rs index 76a18267..ba84696c 100644 --- a/nanvm-lib/src/js/js_bigint.rs +++ b/nanvm-lib/src/js/js_bigint.rs @@ -134,6 +134,13 @@ impl JsBigint { Ordering::Greater => Ordering::Greater, } } + + pub fn get_last_bit(&self) -> u64 { + if is_zero(self) { + return 0; + } + self.items()[0] & 1 + } } pub fn negative(m: M, value: &JsBigint) -> JsBigintMutRef { @@ -540,7 +547,7 @@ fn cmp_vec(lhs: &[u64], rhs: &[u64]) -> Ordering { Ordering::Equal } -fn shl_on_u64(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef { +pub fn shl_on_u64(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef { let mut vec = lhs.items().to_vec(); let shift_mod = rhs & ((1 << 6) - 1); if shift_mod > 0 { @@ -565,7 +572,7 @@ fn shl_on_u64(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef { +pub fn shr_on_u64(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef { let number_to_remove = (rhs / 64) as usize; if number_to_remove >= lhs.items().len() { return shr_on_big(m, lhs.sign()); diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 0632a4b0..f5e2b52b 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -140,7 +140,7 @@ impl TokenizerState { tokens } - fn end(self) -> Vec> { + fn end>(self, manager: M) -> Vec> { match self { TokenizerState::Initial | TokenizerState::ParseNewLine @@ -159,10 +159,10 @@ impl TokenizerState { [JsonToken::ErrorToken(ErrorType::CommentClosingExpected)].cast() } TokenizerState::ParseZero(_) => [JsonToken::Number(default())].cast(), - TokenizerState::ParseInt(s) => [s.into_token()].cast(), - TokenizerState::ParseFrac(s) => [s.into_token()].cast(), - TokenizerState::ParseExp(s) => [s.into_token()].cast(), - TokenizerState::ParseBigInt(s) => [s.into_big_int_token()].cast(), + TokenizerState::ParseInt(s) => [int_state_into_number_token(manager, s)].cast(), + TokenizerState::ParseFrac(s) => [float_state_into_token(manager, s)].cast(), + TokenizerState::ParseExp(s) => [exp_state_into_token(manager, s)].cast(), + TokenizerState::ParseBigInt(s) => [JsonToken::BigInt(s)].cast(), TokenizerState::ParseMinus | TokenizerState::ParseFracBegin(_) | TokenizerState::ParseExpBegin(_) @@ -207,7 +207,7 @@ impl ParseUnicodeCharState { } } -pub fn bigfloat_to_f64(bf_10: BigFloat<10>) -> f64 { +pub fn bigfloat_to_f64(bf_10: BigFloat<10, M>) -> f64 { let bf_2 = bf_10.to_bin(54); bf_2.to_f64() } @@ -267,18 +267,18 @@ impl JsBigintMutRef { }, } } +} - fn into_token(self) -> JsonToken { - JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: self.into_old_bigint(), - exp: 0, - non_zero_reminder: false, - })) - } - - fn into_big_int_token(self) -> JsonToken { - JsonToken::BigInt(self) - } +fn int_state_into_number_token( + manager: M, + state: JsBigintMutRef, +) -> JsonToken { + JsonToken::Number(bigfloat_to_f64(BigFloat { + manager, + significand: state, + exp: 0, + non_zero_reminder: false, + })) } pub struct FloatState { @@ -301,14 +301,18 @@ impl FloatState { e: 0, } } +} - fn into_token(self) -> JsonToken { - JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: self.b.into_old_bigint(), - exp: self.fe, - non_zero_reminder: false, - })) - } +fn float_state_into_token( + manager: M, + state: FloatState, +) -> JsonToken { + JsonToken::Number(bigfloat_to_f64(BigFloat { + manager, + significand: state.b, + exp: state.fe, + non_zero_reminder: false, + })) } pub struct ExpState { @@ -323,19 +327,23 @@ impl ExpState { self.e = self.e * 10 + digit_to_number(c) as i64; self } +} - fn into_token(self) -> JsonToken { - let exp = self.fe - + match self.es { - js_bigint::Sign::Positive => self.e, - js_bigint::Sign::Negative => -self.e, - }; - JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: self.b.into_old_bigint(), - exp, - non_zero_reminder: false, - })) - } +fn exp_state_into_token( + manager: M, + state: ExpState, +) -> JsonToken { + let exp = state.fe + + match state.es { + js_bigint::Sign::Positive => state.e, + js_bigint::Sign::Negative => -state.e, + }; + JsonToken::Number(bigfloat_to_f64(BigFloat { + manager, + significand: state.b, + exp, + non_zero_reminder: false, + })) } const CP_0: u32 = 0x30; @@ -729,7 +737,7 @@ fn create_int_transactions() -> TransitionMap() -> TransitionMap() -> TransitionMap() -> TransitionMap( rm: create_range_map(terminal_for_number(), |manager, s, c, maps| { transfer_state( manager, - [s.into_token()].cast(), + [JsonToken::BigInt(s)].cast(), TokenizerState::Initial, c, maps, @@ -1066,7 +1074,7 @@ impl, M: Manager + 'static> Iterator for TokenizerState .extend(self.state.push_mut(self.manager, c, &self.maps)), None => { self.end = true; - self.cache.extend(take(&mut self.state).end()) + self.cache.extend(take(&mut self.state).end(self.manager)) } } } @@ -1084,7 +1092,7 @@ mod test { big_uint::BigUint, }, common::cast::Cast, - js::js_bigint::{self, from_u64, zero}, + js::js_bigint::{self, from_u64, new_bigint, zero}, mem::global::{Global, GLOBAL}, tokenizer::bigfloat_to_f64, }; @@ -1255,236 +1263,232 @@ mod test { assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); } - #[test] - #[wasm_bindgen_test] - fn test_integer() { - let result = tokenize(GLOBAL, String::from("0")); - assert_eq!(&result, &[JsonToken::Number(0.0)]); - - let result = tokenize(GLOBAL, String::from("-0")); - assert_eq!(&result, &[JsonToken::Number(0.0)]); - - let result = tokenize(GLOBAL, String::from("0abc")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Id(String::from("abc")) - ] - ); - - let result = tokenize(GLOBAL, String::from("0. 2")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Number(2.0) - ] - ); - - let result = tokenize(GLOBAL, String::from("1234567890")); - assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); - - let result = tokenize(GLOBAL, String::from("-1234567890")); - assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); - - let result = tokenize(GLOBAL, String::from("[0,1]")); - assert_eq!( - &result, - &[ - JsonToken::ArrayBegin, - JsonToken::Number(0.0), - JsonToken::Comma, - JsonToken::Number(1.0), - JsonToken::ArrayEnd - ] - ); - - let result = tokenize(GLOBAL, String::from("001")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Number(1.0), - ] - ); - - let result = tokenize(GLOBAL, String::from("-")); - assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - - let result = tokenize(GLOBAL, String::from("-{}")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::ObjectBegin, - JsonToken::ObjectEnd - ] - ); - - let result = tokenize(GLOBAL, String::from("9007199254740991")); - assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); - - let result = tokenize(GLOBAL, String::from("9007199254740992")); - assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); - - let result = tokenize(GLOBAL, String::from("9007199254740993")); - assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); - } - - #[test] - #[wasm_bindgen_test] - fn test_big_float() { - let result = tokenize( - GLOBAL, - String::from("340282366920938463463374607431768211456"), - ); - assert_eq!( - &result, - &[JsonToken::Number(bigfloat_to_f64(BigFloat { - significand: BigInt { - sign: Sign::Positive, - value: BigUint { - value: [0, 0, 1].cast() - } - }, - exp: 0, - non_zero_reminder: false - }))] - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_float() { - let result = tokenize(GLOBAL, String::from("0.01")); - assert_eq!(&result, &[JsonToken::Number(0.01)]); - - let result = tokenize(GLOBAL, String::from("[-12.34]")); - assert_eq!( - &result, - &[ - JsonToken::ArrayBegin, - JsonToken::Number(-12.34), - JsonToken::ArrayEnd - ] - ); - } - - #[test] - #[wasm_bindgen_test] - fn test_infinity() { - let result = tokenize(GLOBAL, String::from("1e1000")); - assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); - - let result = tokenize(GLOBAL, String::from("-1e+1000")); - assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); - } - - #[test] - #[wasm_bindgen_test] - fn test_exp() { - let result = tokenize(GLOBAL, String::from("1e2")); - assert_eq!(&result, &[JsonToken::Number(1e2)]); - - let result = tokenize(GLOBAL, String::from("1E+2")); - assert_eq!(&result, &[JsonToken::Number(1e2)]); - - let result = tokenize(GLOBAL, String::from("0e-2")); - assert_eq!(&result, &[JsonToken::Number(0.0)]); - - let result = tokenize(GLOBAL, String::from("1e-2")); - assert_eq!(&result, &[JsonToken::Number(1e-2)]); - - let result = tokenize(GLOBAL, String::from("1.2e+2")); - assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); - - let result = tokenize(GLOBAL, String::from("12e0000")); - assert_eq!(&result, &[JsonToken::Number(12.0)]); - - let result = tokenize(GLOBAL, String::from("1e")); - assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - - let result = tokenize(GLOBAL, String::from("1e+")); - assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - - let result = tokenize(GLOBAL, String::from("1e-")); - assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - } - - #[test] - #[wasm_bindgen_test] - fn test_big_int() { - let result = tokenize(GLOBAL, String::from("0n")); - assert_eq!(&result, &[JsonToken::BigInt(zero(GLOBAL))]); - - let result = tokenize(GLOBAL, String::from("-0n")); - assert_eq!( - &result, - &[JsonToken::BigInt(from_u64( - GLOBAL, - js_bigint::Sign::Negative, - 0 - ))] - ); - - let result = tokenize(GLOBAL, String::from("1234567890n")); - assert_eq!( - &result, - &[JsonToken::BigInt(from_u64( - GLOBAL, - js_bigint::Sign::Positive, - 1234567890 - ))] - ); - - let result = tokenize(GLOBAL, String::from("-1234567890n")); - assert_eq!( - &result, - &[JsonToken::BigInt(from_u64( - GLOBAL, - js_bigint::Sign::Negative, - 1234567890 - ))] - ); - - let result = tokenize(GLOBAL, String::from("123.456n")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Id(String::from("n")) - ] - ); - - let result = tokenize(GLOBAL, String::from("123e456n")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Id(String::from("n")) - ] - ); - - let result = tokenize(GLOBAL, String::from("1234567890na")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Id(String::from("a")) - ] - ); - - let result = tokenize(GLOBAL, String::from("1234567890nn")); - assert_eq!( - &result, - &[ - JsonToken::ErrorToken(ErrorType::InvalidNumber), - JsonToken::Id(String::from("n")) - ] - ); - } + // #[test] + // #[wasm_bindgen_test] + // fn test_integer() { + // let result = tokenize(GLOBAL, String::from("0")); + // assert_eq!(&result, &[JsonToken::Number(0.0)]); + + // let result = tokenize(GLOBAL, String::from("-0")); + // assert_eq!(&result, &[JsonToken::Number(0.0)]); + + // let result = tokenize(GLOBAL, String::from("0abc")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("abc")) + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("0. 2")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Number(2.0) + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("1234567890")); + // assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); + + // let result = tokenize(GLOBAL, String::from("-1234567890")); + // assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); + + // let result = tokenize(GLOBAL, String::from("[0,1]")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ArrayBegin, + // JsonToken::Number(0.0), + // JsonToken::Comma, + // JsonToken::Number(1.0), + // JsonToken::ArrayEnd + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("001")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Number(1.0), + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("-")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + // let result = tokenize(GLOBAL, String::from("-{}")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::ObjectBegin, + // JsonToken::ObjectEnd + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("9007199254740991")); + // assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); + + // let result = tokenize(GLOBAL, String::from("9007199254740992")); + // assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); + + // let result = tokenize(GLOBAL, String::from("9007199254740993")); + // assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_big_float() { + // let result = tokenize( + // GLOBAL, + // String::from("340282366920938463463374607431768211456"), + // ); + // assert_eq!( + // &result, + // &[JsonToken::Number(bigfloat_to_f64(BigFloat { + // manager: GLOBAL, + // significand: new_bigint(GLOBAL, js_bigint::Sign::Positive, [0, 0, 1]), + // exp: 0, + // non_zero_reminder: false + // }))] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_float() { + // let result = tokenize(GLOBAL, String::from("0.01")); + // assert_eq!(&result, &[JsonToken::Number(0.01)]); + + // let result = tokenize(GLOBAL, String::from("[-12.34]")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ArrayBegin, + // JsonToken::Number(-12.34), + // JsonToken::ArrayEnd + // ] + // ); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_infinity() { + // let result = tokenize(GLOBAL, String::from("1e1000")); + // assert_eq!(&result, &[JsonToken::Number(f64::INFINITY)]); + + // let result = tokenize(GLOBAL, String::from("-1e+1000")); + // assert_eq!(&result, &[JsonToken::Number(f64::NEG_INFINITY)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_exp() { + // let result = tokenize(GLOBAL, String::from("1e2")); + // assert_eq!(&result, &[JsonToken::Number(1e2)]); + + // let result = tokenize(GLOBAL, String::from("1E+2")); + // assert_eq!(&result, &[JsonToken::Number(1e2)]); + + // let result = tokenize(GLOBAL, String::from("0e-2")); + // assert_eq!(&result, &[JsonToken::Number(0.0)]); + + // let result = tokenize(GLOBAL, String::from("1e-2")); + // assert_eq!(&result, &[JsonToken::Number(1e-2)]); + + // let result = tokenize(GLOBAL, String::from("1.2e+2")); + // assert_eq!(&result, &[JsonToken::Number(1.2e+2)]); + + // let result = tokenize(GLOBAL, String::from("12e0000")); + // assert_eq!(&result, &[JsonToken::Number(12.0)]); + + // let result = tokenize(GLOBAL, String::from("1e")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + // let result = tokenize(GLOBAL, String::from("1e+")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + + // let result = tokenize(GLOBAL, String::from("1e-")); + // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_big_int() { + // let result = tokenize(GLOBAL, String::from("0n")); + // assert_eq!(&result, &[JsonToken::BigInt(zero(GLOBAL))]); + + // let result = tokenize(GLOBAL, String::from("-0n")); + // assert_eq!( + // &result, + // &[JsonToken::BigInt(from_u64( + // GLOBAL, + // js_bigint::Sign::Negative, + // 0 + // ))] + // ); + + // let result = tokenize(GLOBAL, String::from("1234567890n")); + // assert_eq!( + // &result, + // &[JsonToken::BigInt(from_u64( + // GLOBAL, + // js_bigint::Sign::Positive, + // 1234567890 + // ))] + // ); + + // let result = tokenize(GLOBAL, String::from("-1234567890n")); + // assert_eq!( + // &result, + // &[JsonToken::BigInt(from_u64( + // GLOBAL, + // js_bigint::Sign::Negative, + // 1234567890 + // ))] + // ); + + // let result = tokenize(GLOBAL, String::from("123.456n")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("n")) + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("123e456n")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("n")) + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("1234567890na")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("a")) + // ] + // ); + + // let result = tokenize(GLOBAL, String::from("1234567890nn")); + // assert_eq!( + // &result, + // &[ + // JsonToken::ErrorToken(ErrorType::InvalidNumber), + // JsonToken::Id(String::from("n")) + // ] + // ); + // } #[test] #[wasm_bindgen_test] From e6c2799e636c3baa50e9604dc000641b7ffb43ae Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 24 Nov 2024 20:48:15 +0200 Subject: [PATCH 18/28] big float tests --- nanvm-lib/src/big_numbers/big_float.rs | 233 +++++++++++++------------ 1 file changed, 120 insertions(+), 113 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index 5d8db0a0..d0dbca32 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -233,7 +233,7 @@ mod test { common::cast::Cast, js::{ any::Any, - js_bigint::{self, from_u64, zero, JsBigintRef, Sign}, + js_bigint::{self, from_u64, new_bigint, zero, JsBigintRef, Sign}, type_::Type, }, mem::global::{Global, GLOBAL}, @@ -248,7 +248,7 @@ mod test { type BigintRef = JsBigintRef; let res = float_zero(GLOBAL).to_bin(64); - let any = A::move_from(res.significand.to_ref()); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -262,9 +262,10 @@ mod test { manager: GLOBAL, significand: zero(GLOBAL), exp: 10, - non_zero_reminder: false - }.to_bin(64); - let any = A::move_from(res.significand.to_ref()); + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -278,9 +279,10 @@ mod test { manager: GLOBAL, significand: zero(GLOBAL), exp: -10, - non_zero_reminder: false - }.to_bin(64); - let any = A::move_from(res.significand.to_ref()); + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -301,9 +303,10 @@ mod test { manager: GLOBAL, significand: from_u64(GLOBAL, Sign::Positive, 100), exp: 0, - non_zero_reminder: false - }.to_bin(7); - let any = A::move_from(res.significand.to_ref()); + non_zero_reminder: false, + } + .to_bin(7); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -317,9 +320,10 @@ mod test { manager: GLOBAL, significand: from_u64(GLOBAL, Sign::Positive, 1), exp: 1, - non_zero_reminder: false - }.to_bin(64); - let any = A::move_from(res.significand.to_ref()); + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -333,9 +337,10 @@ mod test { manager: GLOBAL, significand: from_u64(GLOBAL, Sign::Positive, 100), exp: 2, - non_zero_reminder: false - }.to_bin(64); - let any = A::move_from(res.significand.to_ref()); + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -349,9 +354,10 @@ mod test { manager: GLOBAL, significand: from_u64(GLOBAL, Sign::Positive, 128), exp: 0, - non_zero_reminder: false - }.to_bin(9); - let any = A::move_from(res.significand.to_ref()); + non_zero_reminder: false, + } + .to_bin(9); + let any = A::move_from(res.significand.to_ref()); assert_eq!(any.get_type(), Type::Bigint); { let o = any.try_move::().unwrap(); @@ -362,103 +368,104 @@ mod test { assert_eq!(res.non_zero_reminder, false); } - // #[test] - // #[wasm_bindgen_test] - // fn test_integer_rounding() { - // let a = BigFloat { - // significand: BigInt::from_i64(128), - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(4); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(8), - // exp: 4, - // non_zero_reminder: false - // } - // ); + #[test] + #[wasm_bindgen_test] + fn test_integer_rounding() { + type A = Any; + type BigintRef = JsBigintRef; - // let a = BigFloat { - // significand: BigInt::from_i64(129), - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(4); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(8), - // exp: 4, - // non_zero_reminder: true - // } - // ); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 128), + exp: 0, + non_zero_reminder: false, + } + .to_bin(4); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[8]); + } + assert_eq!(res.exp, 4); + assert_eq!(res.non_zero_reminder, false); - // #[test] - // #[wasm_bindgen_test] - // fn test_float() { - // let a = BigFloat { - // significand: BigInt::from_i64(100), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(5); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(20), - // exp: -1, - // non_zero_reminder: false - // } - // ); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 129), + exp: 0, + non_zero_reminder: false, + } + .to_bin(4); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[8]); + } + assert_eq!(res.exp, 4); + assert_eq!(res.non_zero_reminder, true); + } - // let a = BigFloat { - // significand: BigInt::from_i64(100), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(64); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt { - // sign: Sign::Positive, - // value: BigUint { - // value: [(1 << 63) + (1 << 61)].cast() - // } - // }, - // exp: -60, - // non_zero_reminder: false - // } - // ); + #[test] + #[wasm_bindgen_test] + fn test_float() { + type A = Any; + type BigintRef = JsBigintRef; - // let a = BigFloat { - // significand: BigInt { - // sign: Sign::Positive, - // value: BigUint { - // value: [0, 1].cast(), - // }, - // }, - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(53); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt { - // sign: Sign::Positive, - // value: BigUint { - // value: [1 << 52].cast() - // } - // }, - // exp: 12, - // non_zero_reminder: false - // } - // ); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), + exp: -1, + non_zero_reminder: false, + } + .to_bin(5); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[20]); + } + assert_eq!(res.exp, -1); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), + exp: -1, + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[(1 << 63) + (1 << 61)]); + } + assert_eq!(res.exp, -60); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: new_bigint(GLOBAL, Sign::Positive, [0, 1]), + exp: 0, + non_zero_reminder: false, + } + .to_bin(53); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[1 << 52]); + } + assert_eq!(res.exp, 12); + assert_eq!(res.non_zero_reminder, false); + } // #[test] // #[wasm_bindgen_test] From cc87231c22bdff182db3e076af326d99071438c9 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 24 Nov 2024 20:50:08 +0200 Subject: [PATCH 19/28] big float tests --- nanvm-lib/src/big_numbers/big_float.rs | 101 ++++++++++++++----------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index d0dbca32..4deffc31 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -467,54 +467,63 @@ mod test { assert_eq!(res.non_zero_reminder, false); } - // #[test] - // #[wasm_bindgen_test] - // fn test_rounding() { - // let a = BigFloat { - // significand: BigInt::from_i64(0b1000_0001), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(5); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b11001), - // exp: -1, - // non_zero_reminder: true - // } - // ); + #[test] + #[wasm_bindgen_test] + fn test_rounding() { + type A = Any; + type BigintRef = JsBigintRef; - // let a = BigFloat { - // significand: BigInt::from_i64(0b1000_0001), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(4); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b1100), - // exp: 0, - // non_zero_reminder: true - // } - // ); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + exp: -1, + non_zero_reminder: false, + } + .to_bin(5); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b11001]); + } + assert_eq!(res.exp, -1); + assert_eq!(res.non_zero_reminder, true); - // let a = BigFloat { - // significand: BigInt::from_i64(0b1000_0001), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(3); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b110), - // exp: 1, - // non_zero_reminder: true - // } - // ); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + exp: -1, + non_zero_reminder: false, + } + .to_bin(4); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b1100]); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b110]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + } // #[test] // #[wasm_bindgen_test] From 60fe2bdc0a8035dc38dabe65b677539355e71a5e Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sun, 24 Nov 2024 20:52:41 +0200 Subject: [PATCH 20/28] big float tests --- nanvm-lib/src/big_numbers/big_float.rs | 161 +++++++++++++------------ 1 file changed, 87 insertions(+), 74 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index 4deffc31..b932f36d 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -525,84 +525,97 @@ mod test { assert_eq!(res.non_zero_reminder, true); } - // #[test] - // #[wasm_bindgen_test] - // fn test_rounding_half() { - // let a = BigFloat { - // significand: BigInt::from_i64(0b101_1010), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(3); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b100), - // exp: 1, - // non_zero_reminder: true - // } - // ); + #[test] + #[wasm_bindgen_test] + fn test_rounding_half() { + type A = Any; + type BigintRef = JsBigintRef; - // let a = BigFloat { - // significand: BigInt::from_i64(0b101_1011), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(3); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b100), - // exp: 1, - // non_zero_reminder: true - // } - // ); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b101_1010), + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b100]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); - // let a = BigFloat { - // significand: BigInt::from_i64(0b110_1101), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(3); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b101), - // exp: 1, - // non_zero_reminder: true - // } - // ); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b101_1011), + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b100]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); - // let a = BigFloat { - // significand: BigInt::from_i64(0b110_1110), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(3); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b101), - // exp: 1, - // non_zero_reminder: true - // } - // ); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b110_1101), + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b101]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); - // let a = BigFloat { - // significand: BigInt::from_i64(0b1001_0110), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_bin(3); - // assert_eq!( - // res, - // BigFloat { - // significand: BigInt::from_i64(0b111), - // exp: 1, - // non_zero_reminder: true - // } - // ); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b110_1110), + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b101]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1001_0110), + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b111]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + } // #[test] // #[wasm_bindgen_test] From 2f485c76e6cca2cac734b45607e26c674a8a5782 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 30 Nov 2024 14:52:10 +0200 Subject: [PATCH 21/28] intstate --- nanvm-lib/src/big_numbers/big_float.rs | 828 +++++++++++++------------ nanvm-lib/src/js/js_bigint.rs | 14 +- nanvm-lib/src/tokenizer/mod.rs | 209 ++++--- 3 files changed, 552 insertions(+), 499 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index b932f36d..fb46f187 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -13,6 +13,7 @@ use crate::{ pub struct BigFloat { pub manager: M, pub significand: JsBigintMutRef, + pub sign: Sign, pub exp: i64, pub non_zero_reminder: bool, } @@ -21,6 +22,7 @@ pub fn float_zero(manager: M) -> BigFloat BigFloat { manager, significand: zero(manager), + sign: Sign::Positive, exp: 0, non_zero_reminder: false, } @@ -46,7 +48,7 @@ impl BigFloat { } loop { - match self.significand.deref().cmp(min_significand) { + match self.significand.deref().compare(min_significand) { Ordering::Greater | Ordering::Equal => return, _ => {} } @@ -66,7 +68,7 @@ impl BigFloat { from_u64(self.manager, Sign::Positive, precision).deref(), ); loop { - if self.significand.deref().cmp(max_significand.deref()) == Ordering::Less { + if self.significand.deref().compare(max_significand.deref()) == Ordering::Less { break; } let last_bit = self.significand.get_last_bit(); @@ -89,6 +91,7 @@ impl BigFloat<10, M> { let mut result: BigFloat<2, M> = BigFloat { manager: self.manager, significand: self.significand, + sign: self.sign, exp: self.exp, non_zero_reminder: self.non_zero_reminder, }; @@ -107,6 +110,7 @@ impl BigFloat<10, M> { let mut result: BigFloat<2, M> = BigFloat { manager: self.manager, significand: new_sign, + sign: self.sign, exp: self.exp, non_zero_reminder: self.non_zero_reminder, }; @@ -119,6 +123,7 @@ impl BigFloat<10, M> { let mut bf10: BigFloat<10, M> = BigFloat { manager: self.manager, significand: self.significand, + sign: self.sign, exp: self.exp, non_zero_reminder: self.non_zero_reminder, }; @@ -135,6 +140,7 @@ impl BigFloat<10, M> { let mut result: BigFloat<2, M> = BigFloat { manager: self.manager, significand: q, + sign: self.sign, exp: bf10.exp, non_zero_reminder: self.non_zero_reminder || !is_zero(r.deref()), }; @@ -170,7 +176,7 @@ impl BigFloat<2, M> { const INF_BITS: u64 = 2047 << 52; let mut bits: u64 = 0; - if self.significand.header_len() < 0 { + if self.sign == Sign::Negative { bits |= 1 << 63; } @@ -181,6 +187,7 @@ impl BigFloat<2, M> { let mut value = BigFloat { manager: self.manager, significand: self.significand, + sign: self.sign, exp: self.exp, non_zero_reminder: self.non_zero_reminder, }; @@ -224,420 +231,439 @@ impl BigFloat<2, M> { #[cfg(test)] mod test { - use std::ops::Deref; - - use wasm_bindgen_test::wasm_bindgen_test; - - use crate::{ - big_numbers::big_float::float_zero, - common::cast::Cast, - js::{ - any::Any, - js_bigint::{self, from_u64, new_bigint, zero, JsBigintRef, Sign}, - type_::Type, - }, - mem::global::{Global, GLOBAL}, - }; - - use super::BigFloat; - - #[test] - #[wasm_bindgen_test] - fn test_zero() { - type A = Any; - type BigintRef = JsBigintRef; - - let res = float_zero(GLOBAL).to_bin(64); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert!(o.items().is_empty()); - } - assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: zero(GLOBAL), - exp: 10, - non_zero_reminder: false, - } - .to_bin(64); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert!(o.items().is_empty()); - } - assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: zero(GLOBAL), - exp: -10, - non_zero_reminder: false, - } - .to_bin(64); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert!(o.items().is_empty()); - } - assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); - } + // use std::ops::Deref; - #[test] - #[wasm_bindgen_test] - fn test_integer() { - type A = Any; - type BigintRef = JsBigintRef; - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 100), - exp: 0, - non_zero_reminder: false, - } - .to_bin(7); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[100]); - } - assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 1), - exp: 1, - non_zero_reminder: false, - } - .to_bin(64); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[10 << 60]); - } - assert_eq!(res.exp, -60); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 100), - exp: 2, - non_zero_reminder: false, - } - .to_bin(64); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[10000 << 50]); - } - assert_eq!(res.exp, -50); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 128), - exp: 0, - non_zero_reminder: false, - } - .to_bin(9); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[256]); - } - assert_eq!(res.exp, -1); - assert_eq!(res.non_zero_reminder, false); - } + // use wasm_bindgen_test::wasm_bindgen_test; - #[test] - #[wasm_bindgen_test] - fn test_integer_rounding() { - type A = Any; - type BigintRef = JsBigintRef; - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 128), - exp: 0, - non_zero_reminder: false, - } - .to_bin(4); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[8]); - } - assert_eq!(res.exp, 4); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 129), - exp: 0, - non_zero_reminder: false, - } - .to_bin(4); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[8]); - } - assert_eq!(res.exp, 4); - assert_eq!(res.non_zero_reminder, true); - } + // use crate::{ + // big_numbers::big_float::float_zero, + // common::cast::Cast, + // js::{ + // any::Any, + // js_bigint::{self, from_u64, new_bigint, zero, JsBigintRef, Sign}, + // type_::Type, + // }, + // mem::global::{Global, GLOBAL}, + // }; - #[test] - #[wasm_bindgen_test] - fn test_float() { - type A = Any; - type BigintRef = JsBigintRef; - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 100), - exp: -1, - non_zero_reminder: false, - } - .to_bin(5); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[20]); - } - assert_eq!(res.exp, -1); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 100), - exp: -1, - non_zero_reminder: false, - } - .to_bin(64); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[(1 << 63) + (1 << 61)]); - } - assert_eq!(res.exp, -60); - assert_eq!(res.non_zero_reminder, false); - - let res = BigFloat { - manager: GLOBAL, - significand: new_bigint(GLOBAL, Sign::Positive, [0, 1]), - exp: 0, - non_zero_reminder: false, - } - .to_bin(53); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[1 << 52]); - } - assert_eq!(res.exp, 12); - assert_eq!(res.non_zero_reminder, false); - } + // use super::BigFloat; - #[test] - #[wasm_bindgen_test] - fn test_rounding() { - type A = Any; - type BigintRef = JsBigintRef; - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), - exp: -1, - non_zero_reminder: false, - } - .to_bin(5); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b11001]); - } - assert_eq!(res.exp, -1); - assert_eq!(res.non_zero_reminder, true); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), - exp: -1, - non_zero_reminder: false, - } - .to_bin(4); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b1100]); - } - assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, true); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), - exp: -1, - non_zero_reminder: false, - } - .to_bin(3); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b110]); - } - assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); - } + // #[test] + // #[wasm_bindgen_test] + // fn test_zero() { + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = float_zero(GLOBAL).to_bin(64); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert!(o.items().is_empty()); + // } + // assert_eq!(res.exp, 0); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: zero(GLOBAL), + // sign: Sign::Positive, + // exp: 10, + // non_zero_reminder: false, + // } + // .to_bin(64); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert!(o.items().is_empty()); + // } + // assert_eq!(res.exp, 0); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: zero(GLOBAL), + // sign: Sign::Positive, + // exp: -10, + // non_zero_reminder: false, + // } + // .to_bin(64); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert!(o.items().is_empty()); + // } + // assert_eq!(res.exp, 0); + // assert_eq!(res.non_zero_reminder, false); + // } - #[test] - #[wasm_bindgen_test] - fn test_rounding_half() { - type A = Any; - type BigintRef = JsBigintRef; - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b101_1010), - exp: -1, - non_zero_reminder: false, - } - .to_bin(3); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b100]); - } - assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b101_1011), - exp: -1, - non_zero_reminder: false, - } - .to_bin(3); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b100]); - } - assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b110_1101), - exp: -1, - non_zero_reminder: false, - } - .to_bin(3); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b101]); - } - assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b110_1110), - exp: -1, - non_zero_reminder: false, - } - .to_bin(3); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b101]); - } - assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); - - let res = BigFloat { - manager: GLOBAL, - significand: from_u64(GLOBAL, Sign::Positive, 0b1001_0110), - exp: -1, - non_zero_reminder: false, - } - .to_bin(3); - let any = A::move_from(res.significand.to_ref()); - assert_eq!(any.get_type(), Type::Bigint); - { - let o = any.try_move::().unwrap(); - assert_eq!(o.sign(), Sign::Positive); - assert_eq!(o.items(), &[0b111]); - } - assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); - } + // #[test] + // #[wasm_bindgen_test] + // fn test_integer() { + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 100), + // sign: Sign::Positive, + // exp: 0, + // non_zero_reminder: false, + // } + // .to_bin(7); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[100]); + // } + // assert_eq!(res.exp, 0); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 1), + // sign: Sign::Positive, + // exp: 1, + // non_zero_reminder: false, + // } + // .to_bin(64); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[10 << 60]); + // } + // assert_eq!(res.exp, -60); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 100), + // sign: Sign::Positive, + // exp: 2, + // non_zero_reminder: false, + // } + // .to_bin(64); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[10000 << 50]); + // } + // assert_eq!(res.exp, -50); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 128), + // sign: Sign::Positive, + // exp: 0, + // non_zero_reminder: false, + // } + // .to_bin(9); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[256]); + // } + // assert_eq!(res.exp, -1); + // assert_eq!(res.non_zero_reminder, false); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_integer_rounding() { + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 128), + // sign: Sign::Positive, + // exp: 0, + // non_zero_reminder: false, + // } + // .to_bin(4); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[8]); + // } + // assert_eq!(res.exp, 4); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 129), + // sign: Sign::Positive, + // exp: 0, + // non_zero_reminder: false, + // } + // .to_bin(4); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[8]); + // } + // assert_eq!(res.exp, 4); + // assert_eq!(res.non_zero_reminder, true); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_float() { + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 100), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(5); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[20]); + // } + // assert_eq!(res.exp, -1); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 100), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(64); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[(1 << 63) + (1 << 61)]); + // } + // assert_eq!(res.exp, -60); + // assert_eq!(res.non_zero_reminder, false); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: new_bigint(GLOBAL, Sign::Positive, [0, 1]), + // sign: Sign::Positive, + // exp: 0, + // non_zero_reminder: false, + // } + // .to_bin(53); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[1 << 52]); + // } + // assert_eq!(res.exp, 12); + // assert_eq!(res.non_zero_reminder, false); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_rounding() { + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(5); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b11001]); + // } + // assert_eq!(res.exp, -1); + // assert_eq!(res.non_zero_reminder, true); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(4); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b1100]); + // } + // assert_eq!(res.exp, 0); + // assert_eq!(res.non_zero_reminder, true); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(3); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b110]); + // } + // assert_eq!(res.exp, 1); + // assert_eq!(res.non_zero_reminder, true); + // } + + // #[test] + // #[wasm_bindgen_test] + // fn test_rounding_half() { + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b101_1010), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(3); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b100]); + // } + // assert_eq!(res.exp, 1); + // assert_eq!(res.non_zero_reminder, true); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b101_1011), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(3); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b100]); + // } + // assert_eq!(res.exp, 1); + // assert_eq!(res.non_zero_reminder, true); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b110_1101), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(3); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b101]); + // } + // assert_eq!(res.exp, 1); + // assert_eq!(res.non_zero_reminder, true); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b110_1110), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(3); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b101]); + // } + // assert_eq!(res.exp, 1); + // assert_eq!(res.non_zero_reminder, true); + + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Positive, 0b1001_0110), + // sign: Sign::Positive, + // exp: -1, + // non_zero_reminder: false, + // } + // .to_bin(3); + // let any = A::move_from(res.significand.to_ref()); + // assert_eq!(any.get_type(), Type::Bigint); + // { + // let o = any.try_move::().unwrap(); + // assert_eq!(o.sign(), Sign::Positive); + // assert_eq!(o.items(), &[0b111]); + // } + // assert_eq!(res.exp, 1); + // assert_eq!(res.non_zero_reminder, true); + // } // #[test] // #[wasm_bindgen_test] // fn test_zero_to_f64() { - // let a = BigFloat { - // significand: BigInt::ZERO, + // type A = Any; + // type BigintRef = JsBigintRef; + + // let res = BigFloat { + // manager: GLOBAL, + // significand: zero(GLOBAL), // exp: 100, // non_zero_reminder: false, - // }; - // let res = a.to_f64(); + // }.to_f64(); // assert_eq!(res, 0.0); // assert!(res.is_sign_positive()); - // let a = BigFloat { - // significand: BigInt { - // sign: Sign::Negative, - // value: BigUint::ZERO, - // }, + // let res = BigFloat { + // manager: GLOBAL, + // significand: from_u64(GLOBAL, Sign::Negative, 0), // exp: 100, // non_zero_reminder: false, - // }; - // let res = a.to_f64(); + // }.to_f64(); // assert_eq!(res, 0.0); // assert!(res.is_sign_negative()); // } diff --git a/nanvm-lib/src/js/js_bigint.rs b/nanvm-lib/src/js/js_bigint.rs index ba84696c..ab1aea98 100644 --- a/nanvm-lib/src/js/js_bigint.rs +++ b/nanvm-lib/src/js/js_bigint.rs @@ -15,7 +15,7 @@ use crate::{ use super::{bitset::BIGINT, ref_cast::RefCast}; -#[derive(Debug, PartialEq, Clone, Eq)] +#[derive(Debug, PartialEq, Clone, Copy, Eq)] pub enum Sign { Positive = 1, Negative = -1, @@ -127,7 +127,7 @@ pub fn is_zero(value: &JsBigint) -> bool { } impl JsBigint { - pub fn cmp(&self, other: &Self) -> Ordering { + pub fn compare(&self, other: &Self) -> Ordering { match self.header_len().cmp(&other.header_len()) { Ordering::Equal => cmp_vec(self.items(), other.items()), Ordering::Less => Ordering::Less, @@ -228,7 +228,7 @@ pub fn div_mod( let mut result: Vec = default(); loop { if cmp_vec(&a, b) == Ordering::Less { - return (new_bigint(m, sign.clone(), result), new_bigint(m, sign, a)); + return (new_bigint(m, sign, result), new_bigint(m, sign, a)); } let a_high_digit = a.len() - 1; let b_high_digit = b.len() - 1; @@ -771,25 +771,25 @@ mod test { let b_ref = from_u64(Global(), Sign::Positive, 1); let a = a_ref.deref(); let b = b_ref.deref(); - assert_eq!(a.cmp(b), Ordering::Equal); + assert_eq!(a.compare(b), Ordering::Equal); let a_ref = from_u64(Global(), Sign::Positive, 1); let b_ref = from_u64(Global(), Sign::Positive, 2); let a = a_ref.deref(); let b = b_ref.deref(); - assert_eq!(a.cmp(b), Ordering::Less); + assert_eq!(a.compare(b), Ordering::Less); let a_ref = from_u64(Global(), Sign::Positive, 1); let b_ref = from_u64(Global(), Sign::Negative, 2); let a = a_ref.deref(); let b = b_ref.deref(); - assert_eq!(a.cmp(b), Ordering::Greater); + assert_eq!(a.compare(b), Ordering::Greater); let a_ref = new_bigint(Global(), Sign::Positive, [1, 2]); let b_ref = new_bigint(Global(), Sign::Positive, [2, 1]); let a = a_ref.deref(); let b = b_ref.deref(); - assert_eq!(a.cmp(b), Ordering::Greater); + assert_eq!(a.compare(b), Ordering::Greater); } #[test] diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index f5e2b52b..2980d394 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -66,8 +66,8 @@ pub enum TokenizerState { ParseUnicodeChar(ParseUnicodeCharState), ParseMinus, ParseZero(js_bigint::Sign), - ParseInt(JsBigintMutRef), - ParseFracBegin(JsBigintMutRef), + ParseInt(IntState), + ParseFracBegin(IntState), ParseFrac(FloatState), ParseExpBegin(ExpState), ParseExpSign(ExpState), @@ -219,13 +219,14 @@ impl BigUint { } } +pub struct IntState { + b: JsBigintMutRef, + s: js_bigint::Sign, +} + impl JsBigintMutRef { - fn from_digit>( - m: M, - sign: js_bigint::Sign, - c: char, - ) -> JsBigintMutRef { - from_u64(m, sign, digit_to_number(c)) + fn from_digit>(m: M, c: char) -> JsBigintMutRef { + from_u64(m, js_bigint::Sign::Positive, digit_to_number(c)) } fn add_digit>(self, m: M, c: char) -> JsBigintMutRef { @@ -237,17 +238,24 @@ impl JsBigintMutRef { from_u64(m, js_bigint::Sign::Positive, 10).deref(), ) .deref(), - Self::from_digit(m, self.sign(), c).deref(), + Self::from_digit(m, c).deref(), ) } +} +impl IntState { fn into_float_state(self) -> FloatState { - FloatState { b: self, fe: 0 } + FloatState { + b: self.b, + s: self.s, + fe: 0, + } } fn into_exp_state(self) -> ExpState { ExpState { - b: self, + b: self.b, + s: self.s, fe: 0, es: js_bigint::Sign::Positive, e: 0, @@ -255,7 +263,7 @@ impl JsBigintMutRef { } fn into_old_bigint(self) -> BigInt { - let deref = self.deref(); + let deref = self.b.deref(); let sign = match deref.sign() { js_bigint::Sign::Positive => big_numbers::big_int::Sign::Positive, js_bigint::Sign::Negative => big_numbers::big_int::Sign::Negative, @@ -271,11 +279,12 @@ impl JsBigintMutRef { fn int_state_into_number_token( manager: M, - state: JsBigintMutRef, + state: IntState, ) -> JsonToken { JsonToken::Number(bigfloat_to_f64(BigFloat { manager, - significand: state, + significand: state.b, + sign: state.s, exp: 0, non_zero_reminder: false, })) @@ -283,6 +292,7 @@ fn int_state_into_number_token( pub struct FloatState { b: JsBigintMutRef, + s: js_bigint::Sign, fe: i64, } @@ -296,6 +306,7 @@ impl FloatState { fn into_exp_state(self) -> ExpState { ExpState { b: self.b, + s: self.s, fe: self.fe, es: js_bigint::Sign::Positive, e: 0, @@ -310,6 +321,7 @@ fn float_state_into_token( JsonToken::Number(bigfloat_to_f64(BigFloat { manager, significand: state.b, + sign: state.s, exp: state.fe, non_zero_reminder: false, })) @@ -317,6 +329,7 @@ fn float_state_into_token( pub struct ExpState { b: JsBigintMutRef, + s: js_bigint::Sign, fe: i64, es: js_bigint::Sign, e: i64, @@ -341,6 +354,7 @@ fn exp_state_into_token( JsonToken::Number(bigfloat_to_f64(BigFloat { manager, significand: state.b, + sign: state.s, exp, non_zero_reminder: false, })) @@ -393,8 +407,11 @@ const fn digit_to_number(c: char) -> u64 { c as u64 - CP_0 as u64 } -fn start_number(manager: M, s: js_bigint::Sign, c: char) -> JsBigintMutRef { - JsBigintMutRef::from_digit(manager, s, c) +fn start_number(manager: M, s: js_bigint::Sign, c: char) -> IntState { + IntState { + b: JsBigintMutRef::from_digit(manager, c), + s, + } } fn create_range_map( @@ -434,9 +451,9 @@ pub struct TransitionMaps { escape_char: TransitionMap, unicode_char: TransitionMap, zero: TransitionMap, - int: TransitionMap, M>, + int: TransitionMap, M>, minus: TransitionMap<(), M>, - frac_begin: TransitionMap, M>, + frac_begin: TransitionMap, M>, frac: TransitionMap, M>, exp_begin: TransitionMap, M>, exp: TransitionMap, M>, @@ -674,7 +691,10 @@ fn create_zero_transactions() -> TransitionMap, ), @@ -683,6 +703,7 @@ fn create_zero_transactions() -> TransitionMap() -> TransitionMap() -> TransitionMap, M> { - type Func = TransitionFunc::Dealloc>>; +fn create_int_transactions() -> TransitionMap, M> { + type Func = TransitionFunc::Dealloc>>; TransitionMap { def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge_list( @@ -722,7 +743,13 @@ fn create_int_transactions() -> TransitionMap, ), from_one('.', |_, s, _, _| { @@ -732,7 +759,7 @@ fn create_int_transactions() -> TransitionMap() -> TransitionMap( -) -> TransitionMap, M> { - type Func = TransitionFunc::Dealloc>>; +fn create_frac_begin_transactions() -> TransitionMap, M> +{ + type Func = TransitionFunc::Dealloc>>; TransitionMap { def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: from_range( @@ -1263,83 +1290,83 @@ mod test { assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::MissingQuotes)]); } - // #[test] - // #[wasm_bindgen_test] - // fn test_integer() { - // let result = tokenize(GLOBAL, String::from("0")); - // assert_eq!(&result, &[JsonToken::Number(0.0)]); + #[test] + #[wasm_bindgen_test] + fn test_integer() { + let result = tokenize(GLOBAL, String::from("0")); + assert_eq!(&result, &[JsonToken::Number(0.0)]); - // let result = tokenize(GLOBAL, String::from("-0")); - // assert_eq!(&result, &[JsonToken::Number(0.0)]); + let result = tokenize(GLOBAL, String::from("-0")); + assert_eq!(&result, &[JsonToken::Number(0.0)]); - // let result = tokenize(GLOBAL, String::from("0abc")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Id(String::from("abc")) - // ] - // ); + let result = tokenize(GLOBAL, String::from("0abc")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Id(String::from("abc")) + ] + ); - // let result = tokenize(GLOBAL, String::from("0. 2")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Number(2.0) - // ] - // ); + let result = tokenize(GLOBAL, String::from("0. 2")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Number(2.0) + ] + ); - // let result = tokenize(GLOBAL, String::from("1234567890")); - // assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); + let result = tokenize(GLOBAL, String::from("1234567890")); + assert_eq!(&result, &[JsonToken::Number(1234567890.0)]); - // let result = tokenize(GLOBAL, String::from("-1234567890")); - // assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); + let result = tokenize(GLOBAL, String::from("-1234567890")); + assert_eq!(&result, &[JsonToken::Number(-1234567890.0)]); - // let result = tokenize(GLOBAL, String::from("[0,1]")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ArrayBegin, - // JsonToken::Number(0.0), - // JsonToken::Comma, - // JsonToken::Number(1.0), - // JsonToken::ArrayEnd - // ] - // ); + let result = tokenize(GLOBAL, String::from("[0,1]")); + assert_eq!( + &result, + &[ + JsonToken::ArrayBegin, + JsonToken::Number(0.0), + JsonToken::Comma, + JsonToken::Number(1.0), + JsonToken::ArrayEnd + ] + ); - // let result = tokenize(GLOBAL, String::from("001")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::Number(1.0), - // ] - // ); + let result = tokenize(GLOBAL, String::from("001")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::Number(1.0), + ] + ); - // let result = tokenize(GLOBAL, String::from("-")); - // assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); + let result = tokenize(GLOBAL, String::from("-")); + assert_eq!(&result, &[JsonToken::ErrorToken(ErrorType::InvalidNumber)]); - // let result = tokenize(GLOBAL, String::from("-{}")); - // assert_eq!( - // &result, - // &[ - // JsonToken::ErrorToken(ErrorType::InvalidNumber), - // JsonToken::ObjectBegin, - // JsonToken::ObjectEnd - // ] - // ); + let result = tokenize(GLOBAL, String::from("-{}")); + assert_eq!( + &result, + &[ + JsonToken::ErrorToken(ErrorType::InvalidNumber), + JsonToken::ObjectBegin, + JsonToken::ObjectEnd + ] + ); - // let result = tokenize(GLOBAL, String::from("9007199254740991")); - // assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); + let result = tokenize(GLOBAL, String::from("9007199254740991")); + assert_eq!(&result, &[JsonToken::Number(9007199254740991.0)]); - // let result = tokenize(GLOBAL, String::from("9007199254740992")); - // assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); + let result = tokenize(GLOBAL, String::from("9007199254740992")); + assert_eq!(&result, &[JsonToken::Number(9007199254740992.0)]); - // let result = tokenize(GLOBAL, String::from("9007199254740993")); - // assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); - // } + let result = tokenize(GLOBAL, String::from("9007199254740993")); + assert_eq!(&result, &[JsonToken::Number(9007199254740993.0)]); + } // #[test] // #[wasm_bindgen_test] From 78ffd71779f9638fe89dc338f876bca98919f0ee Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 30 Nov 2024 14:53:12 +0200 Subject: [PATCH 22/28] big_float tests --- nanvm-lib/src/big_numbers/big_float.rs | 812 ++++++++++++------------- 1 file changed, 406 insertions(+), 406 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index fb46f187..c04c7f57 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -231,417 +231,417 @@ impl BigFloat<2, M> { #[cfg(test)] mod test { - // use std::ops::Deref; - - // use wasm_bindgen_test::wasm_bindgen_test; - - // use crate::{ - // big_numbers::big_float::float_zero, - // common::cast::Cast, - // js::{ - // any::Any, - // js_bigint::{self, from_u64, new_bigint, zero, JsBigintRef, Sign}, - // type_::Type, - // }, - // mem::global::{Global, GLOBAL}, - // }; - - // use super::BigFloat; - - // #[test] - // #[wasm_bindgen_test] - // fn test_zero() { - // type A = Any; - // type BigintRef = JsBigintRef; - - // let res = float_zero(GLOBAL).to_bin(64); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert!(o.items().is_empty()); - // } - // assert_eq!(res.exp, 0); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: zero(GLOBAL), - // sign: Sign::Positive, - // exp: 10, - // non_zero_reminder: false, - // } - // .to_bin(64); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert!(o.items().is_empty()); - // } - // assert_eq!(res.exp, 0); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: zero(GLOBAL), - // sign: Sign::Positive, - // exp: -10, - // non_zero_reminder: false, - // } - // .to_bin(64); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert!(o.items().is_empty()); - // } - // assert_eq!(res.exp, 0); - // assert_eq!(res.non_zero_reminder, false); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_integer() { - // type A = Any; - // type BigintRef = JsBigintRef; - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 100), - // sign: Sign::Positive, - // exp: 0, - // non_zero_reminder: false, - // } - // .to_bin(7); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[100]); - // } - // assert_eq!(res.exp, 0); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 1), - // sign: Sign::Positive, - // exp: 1, - // non_zero_reminder: false, - // } - // .to_bin(64); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[10 << 60]); - // } - // assert_eq!(res.exp, -60); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 100), - // sign: Sign::Positive, - // exp: 2, - // non_zero_reminder: false, - // } - // .to_bin(64); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[10000 << 50]); - // } - // assert_eq!(res.exp, -50); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 128), - // sign: Sign::Positive, - // exp: 0, - // non_zero_reminder: false, - // } - // .to_bin(9); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[256]); - // } - // assert_eq!(res.exp, -1); - // assert_eq!(res.non_zero_reminder, false); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_integer_rounding() { - // type A = Any; - // type BigintRef = JsBigintRef; - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 128), - // sign: Sign::Positive, - // exp: 0, - // non_zero_reminder: false, - // } - // .to_bin(4); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[8]); - // } - // assert_eq!(res.exp, 4); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 129), - // sign: Sign::Positive, - // exp: 0, - // non_zero_reminder: false, - // } - // .to_bin(4); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[8]); - // } - // assert_eq!(res.exp, 4); - // assert_eq!(res.non_zero_reminder, true); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_float() { - // type A = Any; - // type BigintRef = JsBigintRef; - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 100), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(5); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[20]); - // } - // assert_eq!(res.exp, -1); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 100), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(64); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[(1 << 63) + (1 << 61)]); - // } - // assert_eq!(res.exp, -60); - // assert_eq!(res.non_zero_reminder, false); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: new_bigint(GLOBAL, Sign::Positive, [0, 1]), - // sign: Sign::Positive, - // exp: 0, - // non_zero_reminder: false, - // } - // .to_bin(53); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[1 << 52]); - // } - // assert_eq!(res.exp, 12); - // assert_eq!(res.non_zero_reminder, false); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_rounding() { - // type A = Any; - // type BigintRef = JsBigintRef; - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(5); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b11001]); - // } - // assert_eq!(res.exp, -1); - // assert_eq!(res.non_zero_reminder, true); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(4); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b1100]); - // } - // assert_eq!(res.exp, 0); - // assert_eq!(res.non_zero_reminder, true); - - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(3); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b110]); - // } - // assert_eq!(res.exp, 1); - // assert_eq!(res.non_zero_reminder, true); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_rounding_half() { - // type A = Any; - // type BigintRef = JsBigintRef; + use std::ops::Deref; + + use wasm_bindgen_test::wasm_bindgen_test; + + use crate::{ + big_numbers::big_float::float_zero, + common::cast::Cast, + js::{ + any::Any, + js_bigint::{self, from_u64, new_bigint, zero, JsBigintRef, Sign}, + type_::Type, + }, + mem::global::{Global, GLOBAL}, + }; + + use super::BigFloat; + + #[test] + #[wasm_bindgen_test] + fn test_zero() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = float_zero(GLOBAL).to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert!(o.items().is_empty()); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: zero(GLOBAL), + sign: Sign::Positive, + exp: 10, + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert!(o.items().is_empty()); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: zero(GLOBAL), + sign: Sign::Positive, + exp: -10, + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert!(o.items().is_empty()); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); + } - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b101_1010), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(3); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b100]); - // } - // assert_eq!(res.exp, 1); - // assert_eq!(res.non_zero_reminder, true); + #[test] + #[wasm_bindgen_test] + fn test_integer() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_bin(7); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[100]); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: 1, + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[10 << 60]); + } + assert_eq!(res.exp, -60); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), + sign: Sign::Positive, + exp: 2, + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[10000 << 50]); + } + assert_eq!(res.exp, -50); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 128), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_bin(9); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[256]); + } + assert_eq!(res.exp, -1); + assert_eq!(res.non_zero_reminder, false); + } - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b101_1011), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(3); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b100]); - // } - // assert_eq!(res.exp, 1); - // assert_eq!(res.non_zero_reminder, true); + #[test] + #[wasm_bindgen_test] + fn test_integer_rounding() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 128), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_bin(4); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[8]); + } + assert_eq!(res.exp, 4); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 129), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_bin(4); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[8]); + } + assert_eq!(res.exp, 4); + assert_eq!(res.non_zero_reminder, true); + } - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b110_1101), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(3); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b101]); - // } - // assert_eq!(res.exp, 1); - // assert_eq!(res.non_zero_reminder, true); + #[test] + #[wasm_bindgen_test] + fn test_float() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(5); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[20]); + } + assert_eq!(res.exp, -1); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 100), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(64); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[(1 << 63) + (1 << 61)]); + } + assert_eq!(res.exp, -60); + assert_eq!(res.non_zero_reminder, false); + + let res = BigFloat { + manager: GLOBAL, + significand: new_bigint(GLOBAL, Sign::Positive, [0, 1]), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_bin(53); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[1 << 52]); + } + assert_eq!(res.exp, 12); + assert_eq!(res.non_zero_reminder, false); + } - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b110_1110), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(3); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b101]); - // } - // assert_eq!(res.exp, 1); - // assert_eq!(res.non_zero_reminder, true); + #[test] + #[wasm_bindgen_test] + fn test_rounding() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(5); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b11001]); + } + assert_eq!(res.exp, -1); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(4); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b1100]); + } + assert_eq!(res.exp, 0); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1000_0001), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b110]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + } - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Positive, 0b1001_0110), - // sign: Sign::Positive, - // exp: -1, - // non_zero_reminder: false, - // } - // .to_bin(3); - // let any = A::move_from(res.significand.to_ref()); - // assert_eq!(any.get_type(), Type::Bigint); - // { - // let o = any.try_move::().unwrap(); - // assert_eq!(o.sign(), Sign::Positive); - // assert_eq!(o.items(), &[0b111]); - // } - // assert_eq!(res.exp, 1); - // assert_eq!(res.non_zero_reminder, true); - // } + #[test] + #[wasm_bindgen_test] + fn test_rounding_half() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b101_1010), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b100]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b101_1011), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b100]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b110_1101), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b101]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b110_1110), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b101]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b1001_0110), + sign: Sign::Positive, + exp: -1, + non_zero_reminder: false, + } + .to_bin(3); + let any = A::move_from(res.significand.to_ref()); + assert_eq!(any.get_type(), Type::Bigint); + { + let o = any.try_move::().unwrap(); + assert_eq!(o.sign(), Sign::Positive); + assert_eq!(o.items(), &[0b111]); + } + assert_eq!(res.exp, 1); + assert_eq!(res.non_zero_reminder, true); + } // #[test] // #[wasm_bindgen_test] From ea7ad298f9b9cf056f2d82e0dbebd983e2f2c82b Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 30 Nov 2024 14:56:35 +0200 Subject: [PATCH 23/28] big_float negative zero case --- nanvm-lib/src/big_numbers/big_float.rs | 48 ++++++++++++++------------ 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index c04c7f57..ecbc9093 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -643,30 +643,32 @@ mod test { assert_eq!(res.non_zero_reminder, true); } - // #[test] - // #[wasm_bindgen_test] - // fn test_zero_to_f64() { - // type A = Any; - // type BigintRef = JsBigintRef; - - // let res = BigFloat { - // manager: GLOBAL, - // significand: zero(GLOBAL), - // exp: 100, - // non_zero_reminder: false, - // }.to_f64(); - // assert_eq!(res, 0.0); - // assert!(res.is_sign_positive()); + #[test] + #[wasm_bindgen_test] + fn test_zero_to_f64() { + type A = Any; + type BigintRef = JsBigintRef; - // let res = BigFloat { - // manager: GLOBAL, - // significand: from_u64(GLOBAL, Sign::Negative, 0), - // exp: 100, - // non_zero_reminder: false, - // }.to_f64(); - // assert_eq!(res, 0.0); - // assert!(res.is_sign_negative()); - // } + let res = BigFloat { + manager: GLOBAL, + significand: zero(GLOBAL), + sign: Sign::Positive, + exp: 100, + non_zero_reminder: false, + }.to_f64(); + assert_eq!(res, 0.0); + assert!(res.is_sign_positive()); + + let res = BigFloat { + manager: GLOBAL, + significand: zero(GLOBAL), + sign: Sign::Negative, + exp: 100, + non_zero_reminder: false, + }.to_f64(); + assert_eq!(res, 0.0); + assert!(res.is_sign_negative()); + } // #[test] // #[wasm_bindgen_test] From 2028865b57b8957fc2b8bbdf27d908e5505c4a63 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 30 Nov 2024 15:53:09 +0200 Subject: [PATCH 24/28] big_float tests --- nanvm-lib/src/big_numbers/big_float.rs | 305 ++++++++++++++----------- 1 file changed, 174 insertions(+), 131 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index ecbc9093..6d9202b6 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -655,7 +655,8 @@ mod test { sign: Sign::Positive, exp: 100, non_zero_reminder: false, - }.to_f64(); + } + .to_f64(); assert_eq!(res, 0.0); assert!(res.is_sign_positive()); @@ -665,155 +666,197 @@ mod test { sign: Sign::Negative, exp: 100, non_zero_reminder: false, - }.to_f64(); + } + .to_f64(); assert_eq!(res, 0.0); assert!(res.is_sign_negative()); } - // #[test] - // #[wasm_bindgen_test] - // fn test_normal_to_f64() { - // let a = BigFloat { - // significand: BigInt::from_u64(1), - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 1.0); + #[test] + #[wasm_bindgen_test] + fn test_normal_to_f64() { + type A = Any; + type BigintRef = JsBigintRef; - // let a = BigFloat { - // significand: BigInt::from_i64(-3), - // exp: -1, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, -1.5); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 1.0); - // let a = BigFloat { - // significand: BigInt::from_i64(1), - // exp: -1022, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 2.0f64.powf(-1022.0)); - // assert!(res.is_normal()); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 3), + sign: Sign::Negative, + exp: -1, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, -1.5); - // let a = BigFloat { - // significand: BigInt::from_u64(1 << 59), - // exp: -1022 - 59, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 2.0f64.powf(-1022.0)); - // assert!(res.is_normal()); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: -1022, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 2.0f64.powf(-1022.0)); + assert!(res.is_normal()); - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 60) - 1), - // exp: -1022 - 60, - // non_zero_reminder: true, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 2.0f64.powf(-1022.0)); - // assert!(res.is_normal()); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1 << 59), + sign: Sign::Positive, + exp: -1022 - 59, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 2.0f64.powf(-1022.0)); + assert!(res.is_normal()); - // let a = BigFloat { - // significand: BigInt::from_i64(1), - // exp: 1023, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 2.0f64.powf(1023.0)); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 60) - 1), + sign: Sign::Positive, + exp: -1022 - 60, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 2.0f64.powf(-1022.0)); + assert!(res.is_normal()); - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 52) - 1), - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 4503599627370495f64); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: 1023, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 2.0f64.powf(1023.0)); - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 53) - 1), - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 9007199254740991f64); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 52) - 1), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 4503599627370495f64); - // #[test] - // #[wasm_bindgen_test] - // fn test_normal_to_f64_rounding() { - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 54) - 1), //111111111111111111111111111111111111111111111111111111 - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 18014398509481984f64); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 53) - 1), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 9007199254740991f64); + } - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 54) - 2), //111111111111111111111111111111111111111111111111111110 - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 18014398509481982f64); + #[test] + #[wasm_bindgen_test] + fn test_normal_to_f64_rounding() { + type A = Any; + type BigintRef = JsBigintRef; - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 - // exp: 0, - // non_zero_reminder: true, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 18014398509481982f64); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 54) - 1), //111111111111111111111111111111111111111111111111111111 + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 18014398509481984f64); - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 - // exp: 0, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 18014398509481980f64); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 54) - 2), //111111111111111111111111111111111111111111111111111110 + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 18014398509481982f64); - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 54) - 1), - // exp: 969, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert!(res.is_normal()); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 + sign: Sign::Positive, + exp: 0, + non_zero_reminder: true, + } + .to_f64(); + assert_eq!(res, 18014398509481982f64); - // let a = BigFloat { - // significand: BigInt::from_u64((1 << 54) - 1), - // exp: 970, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert!(res.is_infinite()); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 54) - 3), //111111111111111111111111111111111111111111111111111101 + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 18014398509481980f64); - // #[test] - // #[wasm_bindgen_test] - // fn test_infinity_to_f64() { - // let a = BigFloat { - // significand: BigInt::from_i64(1), - // exp: 1024, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert!(res.is_infinite()); - // assert!(res.is_sign_positive()); + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 54) - 1), + sign: Sign::Positive, + exp: 969, + non_zero_reminder: false, + } + .to_f64(); + assert!(res.is_normal()); - // let a = BigFloat { - // significand: BigInt::from_i64(-1), - // exp: 1024, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert!(res.is_infinite()); - // assert!(res.is_sign_negative()); - // } + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, (1 << 54) - 1), + sign: Sign::Positive, + exp: 970, + non_zero_reminder: false, + } + .to_f64(); + assert!(res.is_infinite()); + } + + #[test] + #[wasm_bindgen_test] + fn test_infinity_to_f64() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: 1024, + non_zero_reminder: false, + } + .to_f64(); + assert!(res.is_infinite()); + assert!(res.is_sign_positive()); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Negative, + exp: 1024, + non_zero_reminder: false, + } + .to_f64(); + assert!(res.is_infinite()); + assert!(res.is_sign_negative()); + } // #[test] // #[wasm_bindgen_test] From 00edba0d751e31e6bf8a9d955387dc9bd7af326b Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 30 Nov 2024 16:07:48 +0200 Subject: [PATCH 25/28] big_float tests --- nanvm-lib/src/big_numbers/big_float.rs | 337 ++++++++++++++----------- 1 file changed, 186 insertions(+), 151 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index 6d9202b6..587e7713 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -858,155 +858,190 @@ mod test { assert!(res.is_sign_negative()); } - // #[test] - // #[wasm_bindgen_test] - // fn test_subnormal_to_f64() { - // let a = BigFloat { - // significand: BigInt::from_u64(1), - // exp: -1023, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 2.0f64.powf(-1023.0)); - // assert!(res.is_subnormal()); - - // let a = BigFloat { - // significand: BigInt::from_i64(-1), - // exp: -1023, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, -(2.0f64.powf(-1023.0))); - // assert!(res.is_subnormal()); - - // let a = BigFloat { - // significand: BigInt::from_u64(1), - // exp: -1074, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 1); - // assert_eq!(res, 2.0f64.powf(-1074.0)); - // assert!(res.is_subnormal()); - - // let a = BigFloat { - // significand: BigInt::from_u64(1), - // exp: -1075, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res, 0.0); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_subnormal_to_f64_rounding() { - // 0.0 => 0 - // let a = BigFloat { - // significand: BigInt::from_u64(0b100), - // exp: -1075, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b10); - // assert_eq!(res, 2.0f64.powf(-1073.0)); - // assert!(res.is_subnormal()); - - // 0.0+ => 0 - // let a = BigFloat { - // significand: BigInt::from_u64(0b100), - // exp: -1075, - // non_zero_reminder: true, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b10); - // assert_eq!(res, 2.0f64.powf(-1073.0)); - // assert!(res.is_subnormal()); - - // 0.1 => 0 - // let a = BigFloat { - // significand: BigInt::from_u64(0b101), - // exp: -1075, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b10); - // assert_eq!(res, 2.0f64.powf(-1073.0)); - // assert!(res.is_subnormal()); - - // 0.1+ => 1 - // let a = BigFloat { - // significand: BigInt::from_u64(0b101), - // exp: -1075, - // non_zero_reminder: true, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b11); - // assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); - // assert!(res.is_subnormal()); - - // 1.0 => 1 - // let a = BigFloat { - // significand: BigInt::from_u64(0b110), - // exp: -1075, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b11); - // assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); - // assert!(res.is_subnormal()); - - // 1.0+ => 1 - // let a = BigFloat { - // significand: BigInt::from_u64(0b110), - // exp: -1075, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b11); - // assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); - // assert!(res.is_subnormal()); - - // 1.1 => 2 - // let a = BigFloat { - // significand: BigInt::from_u64(0b111), - // exp: -1075, - // non_zero_reminder: false, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b100); - // assert_eq!(res, 2.0f64.powf(-1072.0)); - // assert!(res.is_subnormal()); - - // 1.1+ => 2 - // let a = BigFloat { - // significand: BigInt::from_u64(0b111), - // exp: -1075, - // non_zero_reminder: true, - // }; - // let res = a.to_f64(); - // assert_eq!(res.to_bits(), 0b100); - // assert_eq!(res, 2.0f64.powf(-1072.0)); - // assert!(res.is_subnormal()); - // } - - // #[test] - // #[wasm_bindgen_test] - // fn test_rust_cast() { - // test(18014398509481981); - // test(18014398509481982); - // test(18014398509481983); - // test(18014398509481984); - // test(18014398509481985); - - // fn test(n: u64) { - // let big_float = BigFloat { - // significand: BigInt::from_u64(n), - // exp: 0, - // non_zero_reminder: false, - // }; - // let f64 = big_float.to_f64(); - // assert_eq!(f64, n as f64); - // } - // } + #[test] + #[wasm_bindgen_test] + fn test_subnormal_to_f64() { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: -1023, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 2.0f64.powf(-1023.0)); + assert!(res.is_subnormal()); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Negative, + exp: -1023, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, -(2.0f64.powf(-1023.0))); + assert!(res.is_subnormal()); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: -1074, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res.to_bits(), 1); + assert_eq!(res, 2.0f64.powf(-1074.0)); + assert!(res.is_subnormal()); + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 1), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, 0.0); + } + + #[test] + #[wasm_bindgen_test] + fn test_subnormal_to_f64_rounding() { + type A = Any; + type BigintRef = JsBigintRef; + + // 0.0 => 0 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b100), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b10); + assert_eq!(res, 2.0f64.powf(-1073.0)); + assert!(res.is_subnormal()); + + // 0.0+ => 0 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b100), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: true, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b10); + assert_eq!(res, 2.0f64.powf(-1073.0)); + assert!(res.is_subnormal()); + + // 0.1 => 0 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b101), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b10); + assert_eq!(res, 2.0f64.powf(-1073.0)); + assert!(res.is_subnormal()); + + // 0.1+ => 1 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b101), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: true, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b11); + assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); + assert!(res.is_subnormal()); + + // 1.0 => 1 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b110), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b11); + assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); + assert!(res.is_subnormal()); + + // 1.0+ => 1 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b110), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: true, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b11); + assert_eq!(res, 1.5f64 * 2.0f64.powf(-1073.0)); + assert!(res.is_subnormal()); + + // 1.1 => 2 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b111), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b100); + assert_eq!(res, 2.0f64.powf(-1072.0)); + assert!(res.is_subnormal()); + + // 1.1+ => 2 + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, 0b111), + sign: Sign::Positive, + exp: -1075, + non_zero_reminder: true, + } + .to_f64(); + assert_eq!(res.to_bits(), 0b100); + assert_eq!(res, 2.0f64.powf(-1072.0)); + assert!(res.is_subnormal()); + } + + #[test] + #[wasm_bindgen_test] + fn test_rust_cast() { + test(18014398509481981); + test(18014398509481982); + test(18014398509481983); + test(18014398509481984); + test(18014398509481985); + + fn test(n: u64) { + type A = Any; + type BigintRef = JsBigintRef; + + let res = BigFloat { + manager: GLOBAL, + significand: from_u64(GLOBAL, Sign::Positive, n), + sign: Sign::Positive, + exp: 0, + non_zero_reminder: false, + } + .to_f64(); + assert_eq!(res, n as f64); + } + } } From 15d883389d0edfcaae336a3f1776d5609d63b6a8 Mon Sep 17 00:00:00 2001 From: Shandar Denys Date: Sat, 30 Nov 2024 18:32:53 +0200 Subject: [PATCH 26/28] fix bigint parsing --- nanvm-lib/src/tokenizer/mod.rs | 328 +++++++++++++++++---------------- 1 file changed, 165 insertions(+), 163 deletions(-) diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index 2980d394..a222ab7a 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -5,9 +5,14 @@ use std::{ }; use crate::{ - big_numbers::{self, big_float::BigFloat, big_int::BigInt, big_uint::BigUint}, + big_numbers::{ + self, + big_float::BigFloat, + big_int::{BigInt, Sign}, + big_uint::BigUint, + }, common::{cast::Cast, default::default}, - js::js_bigint::{self, add, equals, from_u64, mul, JsBigintMutRef}, + js::js_bigint::{self, add, equals, from_u64, mul, negative, JsBigintMutRef}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State}, }; @@ -262,17 +267,10 @@ impl IntState { } } - fn into_old_bigint(self) -> BigInt { - let deref = self.b.deref(); - let sign = match deref.sign() { - js_bigint::Sign::Positive => big_numbers::big_int::Sign::Positive, - js_bigint::Sign::Negative => big_numbers::big_int::Sign::Negative, - }; - BigInt { - sign, - value: BigUint { - value: deref.items().to_vec(), - }, + fn into_bigint_state>(self, m: M) -> JsBigintMutRef { + match self.s { + js_bigint::Sign::Positive => self.b, + js_bigint::Sign::Negative => negative(m, self.b.deref()), } } } @@ -758,8 +756,11 @@ fn create_int_transactions() -> TransitionMap Date: Sat, 30 Nov 2024 19:30:51 +0200 Subject: [PATCH 27/28] fix clippy --- nanvm-lib/src/big_numbers/big_float.rs | 48 ++++++------ nanvm-lib/src/parser/parser.rs | 11 +-- nanvm-lib/src/tokenizer/mod.rs | 104 ++++++++----------------- 3 files changed, 58 insertions(+), 105 deletions(-) diff --git a/nanvm-lib/src/big_numbers/big_float.rs b/nanvm-lib/src/big_numbers/big_float.rs index 587e7713..77e01ade 100644 --- a/nanvm-lib/src/big_numbers/big_float.rs +++ b/nanvm-lib/src/big_numbers/big_float.rs @@ -1,9 +1,8 @@ use std::{cmp::Ordering, ops::Deref}; use crate::{ - common::cast::Cast, js::js_bigint::{ - div_mod, from_u64, is_zero, mul, pow, pow_u64, shl, shl_on_u64, shr_on_u64, zero, JsBigint, + div_mod, from_u64, is_zero, mul, pow_u64, shl, shl_on_u64, shr_on_u64, zero, JsBigint, JsBigintMutRef, Sign, }, mem::manager::Manager, @@ -231,16 +230,13 @@ impl BigFloat<2, M> { #[cfg(test)] mod test { - use std::ops::Deref; - use wasm_bindgen_test::wasm_bindgen_test; use crate::{ big_numbers::big_float::float_zero, - common::cast::Cast, js::{ any::Any, - js_bigint::{self, from_u64, new_bigint, zero, JsBigintRef, Sign}, + js_bigint::{from_u64, new_bigint, zero, JsBigintRef, Sign}, type_::Type, }, mem::global::{Global, GLOBAL}, @@ -263,7 +259,7 @@ mod test { assert!(o.items().is_empty()); } assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -281,7 +277,7 @@ mod test { assert!(o.items().is_empty()); } assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -299,7 +295,7 @@ mod test { assert!(o.items().is_empty()); } assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); } #[test] @@ -324,7 +320,7 @@ mod test { assert_eq!(o.items(), &[100]); } assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -342,7 +338,7 @@ mod test { assert_eq!(o.items(), &[10 << 60]); } assert_eq!(res.exp, -60); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -360,7 +356,7 @@ mod test { assert_eq!(o.items(), &[10000 << 50]); } assert_eq!(res.exp, -50); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -378,7 +374,7 @@ mod test { assert_eq!(o.items(), &[256]); } assert_eq!(res.exp, -1); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); } #[test] @@ -403,7 +399,7 @@ mod test { assert_eq!(o.items(), &[8]); } assert_eq!(res.exp, 4); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -421,7 +417,7 @@ mod test { assert_eq!(o.items(), &[8]); } assert_eq!(res.exp, 4); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); } #[test] @@ -446,7 +442,7 @@ mod test { assert_eq!(o.items(), &[20]); } assert_eq!(res.exp, -1); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -464,7 +460,7 @@ mod test { assert_eq!(o.items(), &[(1 << 63) + (1 << 61)]); } assert_eq!(res.exp, -60); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -482,7 +478,7 @@ mod test { assert_eq!(o.items(), &[1 << 52]); } assert_eq!(res.exp, 12); - assert_eq!(res.non_zero_reminder, false); + assert!(!res.non_zero_reminder); } #[test] @@ -507,7 +503,7 @@ mod test { assert_eq!(o.items(), &[0b11001]); } assert_eq!(res.exp, -1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -525,7 +521,7 @@ mod test { assert_eq!(o.items(), &[0b1100]); } assert_eq!(res.exp, 0); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -543,7 +539,7 @@ mod test { assert_eq!(o.items(), &[0b110]); } assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); } #[test] @@ -568,7 +564,7 @@ mod test { assert_eq!(o.items(), &[0b100]); } assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -586,7 +582,7 @@ mod test { assert_eq!(o.items(), &[0b100]); } assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -604,7 +600,7 @@ mod test { assert_eq!(o.items(), &[0b101]); } assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -622,7 +618,7 @@ mod test { assert_eq!(o.items(), &[0b101]); } assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); let res = BigFloat { manager: GLOBAL, @@ -640,7 +636,7 @@ mod test { assert_eq!(o.items(), &[0b111]); } assert_eq!(res.exp, 1); - assert_eq!(res.non_zero_reminder, true); + assert!(res.non_zero_reminder); } #[test] diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index 5807cf58..d21d4108 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -365,9 +365,8 @@ mod test { use crate::{ common::default::default, js::{ - self, js_array::JsArrayRef, - js_bigint::{from_u64, new_bigint, JsBigintRef}, + js_bigint::{from_u64, new_bigint, JsBigintRef, Sign}, js_object::JsObjectRef, js_string::JsStringRef, type_::Type, @@ -907,11 +906,7 @@ mod test { let items = result.items(); assert_eq!(items, [0x61, 0x62, 0x63]); - let tokens = [JsonToken::BigInt(from_u64( - manager, - js::js_bigint::Sign::Positive, - 1, - ))]; + let tokens = [JsonToken::BigInt(from_u64(manager, Sign::Positive, 1))]; let result = parse_with_virtual_io(manager, tokens.into_iter()); assert!(result.is_ok()); let result = result.unwrap().any.try_move::>(); @@ -923,7 +918,7 @@ mod test { let tokens = [JsonToken::BigInt(new_bigint( manager, - js::js_bigint::Sign::Negative, + Sign::Negative, [2, 3], ))]; let result = parse_with_virtual_io(manager, tokens.into_iter()); diff --git a/nanvm-lib/src/tokenizer/mod.rs b/nanvm-lib/src/tokenizer/mod.rs index a222ab7a..e27a88b0 100644 --- a/nanvm-lib/src/tokenizer/mod.rs +++ b/nanvm-lib/src/tokenizer/mod.rs @@ -5,14 +5,9 @@ use std::{ }; use crate::{ - big_numbers::{ - self, - big_float::BigFloat, - big_int::{BigInt, Sign}, - big_uint::BigUint, - }, + big_numbers::big_float::BigFloat, common::{cast::Cast, default::default}, - js::js_bigint::{self, add, equals, from_u64, mul, negative, JsBigintMutRef}, + js::js_bigint::{add, equals, from_u64, mul, negative, JsBigintMutRef, Sign}, mem::manager::{Dealloc, Manager}, range_map::{from_one, from_range, merge, merge_list, RangeMap, State}, }; @@ -70,7 +65,7 @@ pub enum TokenizerState { ParseEscapeChar(String), ParseUnicodeChar(ParseUnicodeCharState), ParseMinus, - ParseZero(js_bigint::Sign), + ParseZero(Sign), ParseInt(IntState), ParseFracBegin(IntState), ParseFrac(FloatState), @@ -217,32 +212,20 @@ pub fn bigfloat_to_f64(bf_10: BigFloat<10, M>) -> f64 { bf_2.to_f64() } -impl BigUint { - fn add_digit(mut self, c: char) -> BigUint { - self = &(&self * &BigUint::from_u64(10)) + &BigUint::from_u64(digit_to_number(c)); - self - } -} - pub struct IntState { b: JsBigintMutRef, - s: js_bigint::Sign, + s: Sign, } impl JsBigintMutRef { fn from_digit>(m: M, c: char) -> JsBigintMutRef { - from_u64(m, js_bigint::Sign::Positive, digit_to_number(c)) + from_u64(m, Sign::Positive, digit_to_number(c)) } fn add_digit>(self, m: M, c: char) -> JsBigintMutRef { add( m, - mul( - m, - self.deref(), - from_u64(m, js_bigint::Sign::Positive, 10).deref(), - ) - .deref(), + mul(m, self.deref(), from_u64(m, Sign::Positive, 10).deref()).deref(), Self::from_digit(m, c).deref(), ) } @@ -262,15 +245,15 @@ impl IntState { b: self.b, s: self.s, fe: 0, - es: js_bigint::Sign::Positive, + es: Sign::Positive, e: 0, } } fn into_bigint_state>(self, m: M) -> JsBigintMutRef { match self.s { - js_bigint::Sign::Positive => self.b, - js_bigint::Sign::Negative => negative(m, self.b.deref()), + Sign::Positive => self.b, + Sign::Negative => negative(m, self.b.deref()), } } } @@ -290,7 +273,7 @@ fn int_state_into_number_token( pub struct FloatState { b: JsBigintMutRef, - s: js_bigint::Sign, + s: Sign, fe: i64, } @@ -306,7 +289,7 @@ impl FloatState { b: self.b, s: self.s, fe: self.fe, - es: js_bigint::Sign::Positive, + es: Sign::Positive, e: 0, } } @@ -327,9 +310,9 @@ fn float_state_into_token( pub struct ExpState { b: JsBigintMutRef, - s: js_bigint::Sign, + s: Sign, fe: i64, - es: js_bigint::Sign, + es: Sign, e: i64, } @@ -346,8 +329,8 @@ fn exp_state_into_token( ) -> JsonToken { let exp = state.fe + match state.es { - js_bigint::Sign::Positive => state.e, - js_bigint::Sign::Negative => -state.e, + Sign::Positive => state.e, + Sign::Negative => -state.e, }; JsonToken::Number(bigfloat_to_f64(BigFloat { manager, @@ -405,7 +388,7 @@ const fn digit_to_number(c: char) -> u64 { c as u64 - CP_0 as u64 } -fn start_number(manager: M, s: js_bigint::Sign, c: char) -> IntState { +fn start_number(manager: M, s: Sign, c: char) -> IntState { IntState { b: JsBigintMutRef::from_digit(manager, c), s, @@ -448,7 +431,7 @@ pub struct TransitionMaps { string: TransitionMap, escape_char: TransitionMap, unicode_char: TransitionMap, - zero: TransitionMap, + zero: TransitionMap, int: TransitionMap, M>, minus: TransitionMap<(), M>, frac_begin: TransitionMap, M>, @@ -519,21 +502,14 @@ fn create_initial_transitions() -> TransitionMap<(), M> { from_range('1'..='9', |manager: M, _, c, _| { ( default(), - TokenizerState::ParseInt(start_number( - manager, - js_bigint::Sign::Positive, - c, - )), + TokenizerState::ParseInt(start_number(manager, Sign::Positive, c)), ) }), from_one('"', |_, _, _, _| { (default(), TokenizerState::ParseString(String::default())) }), from_one('0', |_, _, _, _| { - ( - default(), - TokenizerState::ParseZero(js_bigint::Sign::Positive), - ) + (default(), TokenizerState::ParseZero(Sign::Positive)) }), from_one('-', |_, _, _, _| (default(), TokenizerState::ParseMinus)), create_range_map(id_start(), |_, _, c, _| { @@ -678,8 +654,8 @@ fn create_unicode_char_transactions( } } -fn create_zero_transactions() -> TransitionMap { - type Func = TransitionFunc; +fn create_zero_transactions() -> TransitionMap { + type Func = TransitionFunc; TransitionMap { def: (|manager, _, c, maps| tokenize_invalid_number(manager, c, maps)) as Func, rm: merge_list( @@ -690,7 +666,7 @@ fn create_zero_transactions() -> TransitionMap() -> TransitionMap() -> TransitionMap<(), M> { rm: merge( from_one( '0', - (|_, _, _, _| { - ( - default(), - TokenizerState::ParseZero(js_bigint::Sign::Negative), - ) - }) as Func, + (|_, _, _, _| (default(), TokenizerState::ParseZero(Sign::Negative))) as Func, ), from_range('1'..='9', |manager, _, c, _| { ( default(), - TokenizerState::ParseInt(start_number(manager, js_bigint::Sign::Negative, c)), + TokenizerState::ParseInt(start_number(manager, Sign::Negative, c)), ) }), ), @@ -866,7 +837,7 @@ fn create_exp_begin_transactions() -> TransitionMap Date: Sat, 30 Nov 2024 19:46:06 +0200 Subject: [PATCH 28/28] resolve merge conflicts --- nanvm-lib/src/parser/any_state.rs | 6 +++--- nanvm-lib/src/parser/const_state.rs | 2 +- nanvm-lib/src/parser/json_state.rs | 2 +- nanvm-lib/src/parser/parser.rs | 6 +++--- nanvm-lib/src/parser/root_state.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nanvm-lib/src/parser/any_state.rs b/nanvm-lib/src/parser/any_state.rs index f5a22a11..59716bb4 100644 --- a/nanvm-lib/src/parser/any_state.rs +++ b/nanvm-lib/src/parser/any_state.rs @@ -70,7 +70,7 @@ impl AnyState { pub fn parse( self, manager: M, - token: JsonToken, + token: JsonToken, ) -> ( /*any_result:*/ AnyResult, /*module_name:*/ Option, @@ -95,7 +95,7 @@ impl AnyState { pub fn parse_for_module( self, manager: M, - token: JsonToken, + token: JsonToken, ) -> ( /*json_state:*/ JsonState, /*module_name:*/ Option, @@ -142,7 +142,7 @@ impl AnyState { fn parse_import_value( self, - token: JsonToken, + token: JsonToken, ) -> ( /*any_result:*/ AnyResult, /*module_name:*/ Option, diff --git a/nanvm-lib/src/parser/const_state.rs b/nanvm-lib/src/parser/const_state.rs index 62c2f7d8..9f9b0576 100644 --- a/nanvm-lib/src/parser/const_state.rs +++ b/nanvm-lib/src/parser/const_state.rs @@ -11,7 +11,7 @@ pub struct ConstState { } impl ConstState { - pub fn parse(self, manager: M, token: JsonToken) -> JsonState { + pub fn parse(self, manager: M, token: JsonToken) -> JsonState { match token { JsonToken::Semicolon => todo!(), _ => { diff --git a/nanvm-lib/src/parser/json_state.rs b/nanvm-lib/src/parser/json_state.rs index a14413b4..af16605c 100644 --- a/nanvm-lib/src/parser/json_state.rs +++ b/nanvm-lib/src/parser/json_state.rs @@ -18,7 +18,7 @@ impl JsonState { pub fn push( self, manager: M, - token: JsonToken, + token: JsonToken, ) -> ( /*json_state:*/ JsonState, /*import:*/ Option<(/*id:*/ String, /*module:*/ String)>, diff --git a/nanvm-lib/src/parser/parser.rs b/nanvm-lib/src/parser/parser.rs index 73a05b1b..ef2584d4 100644 --- a/nanvm-lib/src/parser/parser.rs +++ b/nanvm-lib/src/parser/parser.rs @@ -149,10 +149,10 @@ fn any_state_parse( } } -fn root_state_parse( +fn root_state_parse( root_state: RootState, context: &mut Context, - token: JsonToken, + token: JsonToken, ) -> JsonState { let (json_state, import) = root_state.parse(context.manager, token); match import { @@ -175,7 +175,7 @@ fn root_state_parse( let read_result = context.io.read_to_string(current_path.as_str()); match read_result { Ok(s) => { - let tokens = tokenize(s); + let tokens = tokenize(context.manager, s); let res = parse_with_tokens(context, tokens.into_iter()); match res { Ok(r) => { diff --git a/nanvm-lib/src/parser/root_state.rs b/nanvm-lib/src/parser/root_state.rs index 38dcf1e1..1b6c1224 100644 --- a/nanvm-lib/src/parser/root_state.rs +++ b/nanvm-lib/src/parser/root_state.rs @@ -27,7 +27,7 @@ impl RootState { pub fn parse( self, manager: M, - token: JsonToken, + token: JsonToken, ) -> ( /*json_state:*/ JsonState, /*import:*/ Option<(/*id:*/ String, /*module:*/ String)>,