Tokenizer bigint

nanvm-lib/src/js/js_bigint.rs

@@ -15,12 +15,13 @@ use crate::{
 
 use super::{bitset::BIGINT, ref_cast::RefCast};
 
-#[derive(Debug, PartialEq, Clone, Eq)]
+#[derive(Debug, PartialEq, Clone, Copy, Eq)]
 pub enum Sign {
     Positive = 1,
     Negative = -1,
 }
 
+#[derive(Debug)]
 pub struct JsBigintHeader {
     len: isize,
 }
@@ -125,6 +126,23 @@ pub fn is_zero(value: &JsBigint) -> bool {
     value.items().is_empty()
 }
 
+impl JsBigint {
+    pub fn compare(&self, other: &Self) -> Ordering {
+        match self.header_len().cmp(&other.header_len()) {
+            Ordering::Equal => cmp_vec(self.items(), other.items()),
+            Ordering::Less => Ordering::Less,
+            Ordering::Greater => Ordering::Greater,
+        }
+    }
+
+    pub fn get_last_bit(&self) -> u64 {
+        if is_zero(self) {
+            return 0;
+        }
+        self.items()[0] & 1
+    }
+}
+
 pub fn negative<M: Manager>(m: M, value: &JsBigint) -> JsBigintMutRef<M::Dealloc> {
     if is_zero(value) {
         return zero(m);
@@ -210,7 +228,7 @@ pub fn div_mod<M: Manager>(
             let mut result: Vec<u64> = default();
             loop {
                 if cmp_vec(&a, b) == Ordering::Less {
-                    return (new_bigint(m, sign.clone(), result), new_bigint(m, sign, a));
+                    return (new_bigint(m, sign, result), new_bigint(m, sign, a));
                 }
                 let a_high_digit = a.len() - 1;
                 let b_high_digit = b.len() - 1;
@@ -356,6 +374,13 @@ pub fn shr<M: Manager>(m: M, lhs: &JsBigint, rhs: &JsBigint) -> JsBigintMutRef<M
     }
 }
 
+pub fn equals(lhs: &JsBigint, rhs: &JsBigint) -> bool {
+    if lhs.sign() != rhs.sign() {
+        return false;
+    }
+    return cmp_vec(lhs.items(), rhs.items()) == Ordering::Equal;
+}
+
 fn to_twos_complement(value: &JsBigint) -> TwosComplement {
     TwosComplement {
         sign: value.sign(),
@@ -449,7 +474,7 @@ fn twos_complement_zip<'a>(
 }
 
 impl JsBigint {
-    fn sign(&self) -> Sign {
+    pub fn sign(&self) -> Sign {
         if self.header.len < 0 {
             Sign::Negative
         } else {
@@ -522,7 +547,7 @@ fn cmp_vec(lhs: &[u64], rhs: &[u64]) -> Ordering {
     Ordering::Equal
 }
 
-fn shl_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
+pub fn shl_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
     let mut vec = lhs.items().to_vec();
     let shift_mod = rhs & ((1 << 6) - 1);
     if shift_mod > 0 {
@@ -547,7 +572,7 @@ fn shl_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::D
     new_bigint(m, lhs.sign(), vec)
 }
 
-fn shr_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
+pub fn shr_on_u64<M: Manager>(m: M, lhs: &JsBigint, rhs: u64) -> JsBigintMutRef<M::Dealloc> {
     let number_to_remove = (rhs / 64) as usize;
     if number_to_remove >= lhs.items().len() {
         return shr_on_big(m, lhs.sign());
@@ -586,7 +611,7 @@ fn shr_on_big<M: Manager>(m: M, sign: Sign) -> JsBigintMutRef<M::Dealloc> {
 
 #[cfg(test)]
 mod test {
-    use std::ops::Deref;
+    use std::{cmp::Ordering, ops::Deref};
 
     use wasm_bindgen_test::wasm_bindgen_test;
 
@@ -736,6 +761,37 @@ mod test {
         }
     }
 
+    #[test]
+    #[wasm_bindgen_test]
+    fn test_cmp() {
+        type A = Any<Global>;
+        type BigintRef = JsBigintRef<Global>;
+
+        let a_ref = from_u64(Global(), Sign::Positive, 1);
+        let b_ref = from_u64(Global(), Sign::Positive, 1);
+        let a = a_ref.deref();
+        let b = b_ref.deref();
+        assert_eq!(a.compare(b), Ordering::Equal);
+
+        let a_ref = from_u64(Global(), Sign::Positive, 1);
+        let b_ref = from_u64(Global(), Sign::Positive, 2);
+        let a = a_ref.deref();
+        let b = b_ref.deref();
+        assert_eq!(a.compare(b), Ordering::Less);
+
+        let a_ref = from_u64(Global(), Sign::Positive, 1);
+        let b_ref = from_u64(Global(), Sign::Negative, 2);
+        let a = a_ref.deref();
+        let b = b_ref.deref();
+        assert_eq!(a.compare(b), Ordering::Greater);
+
+        let a_ref = new_bigint(Global(), Sign::Positive, [1, 2]);
+        let b_ref = new_bigint(Global(), Sign::Positive, [2, 1]);
+        let a = a_ref.deref();
+        let b = b_ref.deref();
+        assert_eq!(a.compare(b), Ordering::Greater);
+    }
+
     #[test]
     #[wasm_bindgen_test]
     fn test_shl_zero() {

nanvm-lib/src/mem/flexible_array/mod.rs

@@ -12,6 +12,7 @@ use self::header::FlexibleArrayHeader;
 use super::{field_layout::FieldLayout, object::Object};
 
 #[repr(transparent)]
+#[derive(Debug)]
 pub struct FlexibleArray<I, T: FlexibleArrayHeader = usize> {
     pub header: T,
     _0: PhantomData<I>,

nanvm-lib/src/parser/analyzer.rs

@@ -1,7 +1,7 @@
 use super::shared::DataType;
 use crate::ast::Module;
 use crate::common::default::default;
-use crate::mem::manager::Dealloc;
+use crate::mem::manager::{Dealloc, Manager};
 use crate::tokenizer::{create_transition_maps, JsonToken, TokenizerState, TransitionMaps};
 
 #[derive(Default)]
@@ -21,17 +21,17 @@ pub struct AnalyzerResults<D: Dealloc> {
     pub diagnostics: Vec<AnalyzerDiagnostic>,
 }
 
-pub struct AnalyzerState<D: Dealloc> {
+pub struct AnalyzerState<M: Manager> {
     parameters: AnalyzerParameters,
-    tokenizer_state: TokenizerState,
-    tokenizer_maps: TransitionMaps,
+    tokenizer_state: TokenizerState<M::Dealloc>,
+    tokenizer_maps: TransitionMaps<M>,
     diagnostics_len: usize,
     // TODO: add line number, column number tracking fields (needed for diagnostics).
-    module: Module<D>,
+    module: Module<M::Dealloc>,
     diagnostics: Vec<AnalyzerDiagnostic>,
 }
 
-impl<D: Dealloc> AnalyzerState<D> {
+impl<M: Manager + 'static> AnalyzerState<M> {
     /// Creates a new analyzer staring state. The caller should check `diagnostics` for errors
     /// immediately after creation (since `parameters` value can be inconsistent).
     pub fn new(parameters: AnalyzerParameters) -> Self {
@@ -48,8 +48,11 @@ impl<D: Dealloc> AnalyzerState<D> {
     /// Updates analyzer state with a next input character; the result is the increment in the count
     /// of `diagnostics`. It's up to the caller to check what was added at the end of `diagnostics`
     ///  - are there any fatal errors, from the point of view of the current parsing session?
-    pub fn push_mut(&mut self, c: char) -> usize {
-        for token in self.tokenizer_state.push_mut(c, &self.tokenizer_maps) {
+    pub fn push_mut(&mut self, manager: M, c: char) -> usize {
+        for token in self
+            .tokenizer_state
+            .push_mut(manager, c, &self.tokenizer_maps)
+        {
             self.process_token(token);
         }
         let prior_diagnostics_len = self.diagnostics_len;
@@ -58,13 +61,13 @@ impl<D: Dealloc> AnalyzerState<D> {
     }
 
     /// Completes the analysis.
-    pub fn end(self) -> AnalyzerResults<D> {
+    pub fn end(self) -> AnalyzerResults<M::Dealloc> {
         // TODO: in case the current state is not a valid end state, add an error to self.diagnostics.
         AnalyzerResults {
             module: self.module,
             diagnostics: self.diagnostics,
         }
     }
 
-    fn process_token(&mut self, _token: JsonToken) {}
+    fn process_token(&mut self, _token: JsonToken<M::Dealloc>) {}
 }

nanvm-lib/src/parser/any_state.rs

@@ -70,7 +70,7 @@ impl<M: Manager> AnyState<M> {
     pub fn parse(
         self,
         manager: M,
-        token: JsonToken,
+        token: JsonToken<M::Dealloc>,
     ) -> (
         /*any_result:*/ AnyResult<M>,
         /*module_name:*/ Option<String>,
@@ -95,7 +95,7 @@ impl<M: Manager> AnyState<M> {
     pub fn parse_for_module(
         self,
         manager: M,
-        token: JsonToken,
+        token: JsonToken<M::Dealloc>,
     ) -> (
         /*json_state:*/ JsonState<M>,
         /*module_name:*/ Option<String>,
@@ -123,7 +123,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_import_begin(self, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_import_begin(self, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::OpeningParenthesis => AnyResult::Continue(AnyState {
                 status: ParsingStatus::ImportValue,
@@ -133,7 +133,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_import_end(self, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_import_end(self, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::ClosingParenthesis => self.end_import(),
             _ => AnyResult::Error(ParseError::WrongRequireStatement),
@@ -142,7 +142,7 @@ impl<M: Manager> AnyState<M> {
 
     fn parse_import_value(
         self,
-        token: JsonToken,
+        token: JsonToken<M::Dealloc>,
     ) -> (
         /*any_result:*/ AnyResult<M>,
         /*module_name:*/ Option<String>,
@@ -189,7 +189,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_value(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_value(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::ArrayBegin => self.begin_array(),
             JsonToken::ObjectBegin => self.begin_object(),
@@ -258,7 +258,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_array_comma(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_array_comma(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::ArrayBegin => self.begin_array(),
             JsonToken::ObjectBegin => self.begin_object(),
@@ -276,7 +276,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_array_begin(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_array_begin(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::ArrayBegin => self.begin_array(),
             JsonToken::ArrayEnd => self.end_array(manager),
@@ -291,7 +291,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_array_value(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_array_value(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::ArrayEnd => self.end_array(manager),
             JsonToken::Comma => AnyResult::Continue(AnyState {
@@ -329,7 +329,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_object_begin(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_object_begin(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::String(s) => self.push_key(s),
             JsonToken::Id(s) if self.data_type.is_djs() => self.push_key(s),
@@ -338,7 +338,7 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_object_next(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_object_next(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::ObjectEnd => self.end_object(manager),
             JsonToken::Comma => AnyResult::Continue(AnyState {
@@ -349,15 +349,15 @@ impl<M: Manager> AnyState<M> {
         }
     }
 
-    pub fn parse_object_comma(self, manager: M, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_object_comma(self, manager: M, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::String(s) => self.push_key(s),
             JsonToken::ObjectEnd => self.end_object(manager),
             _ => AnyResult::Error(ParseError::UnexpectedToken),
         }
     }
 
-    pub fn parse_object_key(self, token: JsonToken) -> AnyResult<M> {
+    pub fn parse_object_key(self, token: JsonToken<M::Dealloc>) -> AnyResult<M> {
         match token {
             JsonToken::Colon => AnyResult::Continue(AnyState {
                 status: ParsingStatus::ObjectColon,

nanvm-lib/src/parser/const_state.rs

@@ -11,7 +11,7 @@ pub struct ConstState<M: Manager> {
 }
 
 impl<M: Manager> ConstState<M> {
-    pub fn parse(self, manager: M, token: JsonToken) -> JsonState<M> {
+    pub fn parse(self, manager: M, token: JsonToken<M::Dealloc>) -> JsonState<M> {
         match token {
             JsonToken::Semicolon => todo!(),
             _ => {

nanvm-lib/src/parser/json_state.rs

@@ -18,12 +18,12 @@ impl<M: Manager> JsonState<M> {
     pub fn push(
         self,
         manager: M,
-        token: JsonToken,
+        token: JsonToken<M::Dealloc>,
     ) -> (
         /*json_state:*/ JsonState<M>,
         /*import:*/ Option<(/*id:*/ String, /*module:*/ String)>,
     ) {
-        if token == JsonToken::NewLine {
+        if let JsonToken::NewLine = token {
             return match self {
                 JsonState::ParseRoot(state) => state.parse(manager, token),
                 _ => (self, None),