Merge branch 'main' into generic-parselets

Cargo.toml

@@ -39,7 +39,7 @@ indexmap = "1.9"
 num = "0.4"
 num-bigint = "0.4"
 rustyline = "9"
-tokay-macros = "0.3"  # use crates.io-version
-#tokay-macros = { version = "0.3", path = "macros" }  # use local version
+#tokay-macros = "0.3"  # use crates.io-version
+tokay-macros = { version = "0.4", path = "macros" }  # use local version
 num-parse = "0.1"  # use crates.io-version
 #num-parse = { version = "0.1", path = "../num-parse" }  # use local version

build.rs

@@ -47,29 +47,34 @@ fn main() {
                             let module = path
                                 .iter()
                                 .enumerate()
-                                .map(|(i, part)| {
+                                .filter_map(|(i, part)| {
                                     let mut part = part.to_str().unwrap();
 
+                                    //println!("part = {:?}", part);
+
                                     if i == 0 {
-                                        return "crate".to_string();
+                                        return Some("crate".to_string());
                                     } else if part.ends_with(".rs")
                                     // fixme: can the be done better?
                                     {
                                         // cut away the ".rs" here...
                                         part = &part[..part.len() - 3];
 
+                                        if part == "mod" {
+                                            return None;
+                                        }
+
                                         // create "type::Type" here in case it's a method
                                         if kind == "method" {
-                                            return format!(
-                                                "{}::{}{}",
+                                            return Some(format!(
+                                                "{}::{}",
                                                 part,
-                                                &part[0..1].to_uppercase(),
-                                                &part[1..]
-                                            );
+                                                func["impl"].borrow().to_string()
+                                            ));
                                         }
                                     }
 
-                                    part.to_string()
+                                    Some(part.to_string())
                                 })
                                 .collect::<Vec<String>>()
                                 .join("::");

build.tok

@@ -1,10 +1,16 @@
 # Tokay code generator for generating a builtin registration function in Rust written in Tokay v0.4
 
+'impl' _ Identifier _ '{' {
+    impl = $3
+    accept
+}
+
 'tokay_' kind => {
     ''function''
     ''method''
     ''token''
-} '!' _ '(' _ '"' _ name => Identifier #{
+} '!' _ '(' _ '"' _ name => Identifier {
+    kind => $kind, name => $name, impl => impl
     #accept "register(\"" + $name + "\", tokay_" + $kind + "_" + $name.lower() + ")"
     #print("register(\"" + $name + "\", tokay_" + $kind + "_" + $name.lower() + ")")
-#}
+}

macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokay-macros"
-version = "0.3.0"
+version = "0.4.0"
 description = "Macro support for Rust-native builtin functions for the Tokay programming language."
 authors = ["Jan Max Meyer <jmm@phorward.de>"]
 homepage = "https://tokay.dev/"

macros/src/lib.rs

@@ -214,6 +214,10 @@ pub fn tokay_method(input: TokenStream) -> TokenStream {
     let def = syn::parse_macro_input!(input as BuiltinDef);
 
     let name = def.name;
+    let internal = syn::Ident::new(
+        &format!("{}_internal", name.to_string()),
+        proc_macro2::Span::call_site(),
+    );
     let callable = syn::Ident::new(
         &format!("tokay_method_{}", name.to_string()),
         proc_macro2::Span::call_site(),
@@ -238,7 +242,8 @@ pub fn tokay_method(input: TokenStream) -> TokenStream {
     // The direct usage function will return an Result<RefValue, Error>
     // instead of an Result<Accept, Reject>.
     let gen = quote! {
-        pub fn #name(
+        fn #internal(
+            context: Option<&mut tokay::Context>,
             mut args: Vec<tokay::RefValue>,
             mut nargs: Option<tokay::Dict>
         ) -> Result<tokay::RefValue, tokay::Error> {
@@ -252,12 +257,19 @@ pub fn tokay_method(input: TokenStream) -> TokenStream {
             #body
         }
 
+        pub fn #name(
+            args: Vec<tokay::RefValue>,
+            nargs: Option<tokay::Dict>
+        ) -> Result<tokay::RefValue, tokay::Error> {
+            Self::#internal(None, args, nargs)
+        }
+
         pub fn #callable(
-            _context: Option<&mut tokay::Context>,
+            context: Option<&mut tokay::Context>,
             args: Vec<tokay::RefValue>,
             nargs: Option<tokay::Dict>
         ) -> Result<tokay::Accept, tokay::Reject> {
-            let ret = Self::#name(args, nargs)?;
+            let ret = Self::#internal(context, args, nargs)?;
             Ok(tokay::Accept::Push(tokay::Capture::Value(ret, None, 10)))
         }
     };

src/_builtins.rs

@@ -5,7 +5,7 @@ DON'T CHANGE THIS FILE MANUALLY, IT WILL GO AWAY!!!
 */
 use crate::builtin::Builtin;
 
-pub static BUILTINS: [Builtin; 53] = [
+pub static BUILTINS: [Builtin; 60] = [
     Builtin {
         name: "Float",
         func: crate::value::token::tokay_token_float,
@@ -86,10 +86,6 @@ pub static BUILTINS: [Builtin; 53] = [
         name: "dict_set_item",
         func: crate::value::dict::Dict::tokay_method_dict_set_item,
     },
-    Builtin {
-        name: "dict_values",
-        func: crate::value::dict::Dict::tokay_method_dict_values,
-    },
     Builtin {
         name: "error",
         func: crate::error::tokay_function_error,
@@ -114,6 +110,34 @@ pub static BUILTINS: [Builtin; 53] = [
         name: "int",
         func: crate::value::value::Value::tokay_method_int,
     },
+    Builtin {
+        name: "iter",
+        func: crate::value::iter::iter::Iter::tokay_method_iter,
+    },
+    Builtin {
+        name: "iter_collect",
+        func: crate::value::iter::iter::Iter::tokay_method_iter_collect,
+    },
+    Builtin {
+        name: "iter_enum",
+        func: crate::value::iter::enumiter::EnumIter::tokay_method_iter_enum,
+    },
+    Builtin {
+        name: "iter_len",
+        func: crate::value::iter::iter::Iter::tokay_method_iter_len,
+    },
+    Builtin {
+        name: "iter_map",
+        func: crate::value::iter::mapiter::MapIter::tokay_method_iter_map,
+    },
+    Builtin {
+        name: "iter_next",
+        func: crate::value::iter::iter::Iter::tokay_method_iter_next,
+    },
+    Builtin {
+        name: "iter_rev",
+        func: crate::value::iter::iter::Iter::tokay_method_iter_rev,
+    },
     Builtin {
         name: "list",
         func: crate::value::list::List::tokay_method_list,
@@ -158,6 +182,10 @@ pub static BUILTINS: [Builtin; 53] = [
         name: "print",
         func: crate::builtin::tokay_function_print,
     },
+    Builtin {
+        name: "range",
+        func: crate::builtin::range::tokay_function_range,
+    },
     Builtin {
         name: "repr",
         func: crate::builtin::tokay_function_repr,

src/builtin.rs → src/builtin/mod.rs

@@ -6,6 +6,7 @@ use crate::{Accept, Context, Reject};
 use std::io::{self, Write};
 extern crate self as tokay;
 use tokay_macros::tokay_function;
+pub mod range;
 
 // Abstraction of a built-in function
 pub struct Builtin {
@@ -86,6 +87,15 @@ impl Object for BuiltinRef {
     }
 
     fn call(
+        &self,
+        context: Option<&mut Context>,
+        args: Vec<RefValue>,
+        nargs: Option<Dict>,
+    ) -> Result<Accept, Reject> {
+        (self.0.func)(context, args, nargs)
+    }
+
+    fn call_direct(
         &self,
         context: &mut Context,
         args: usize,

src/builtin/range.rs

@@ -0,0 +1,81 @@
+use crate::value::{Iter, Object, RefValue, RefValueIter};
+use crate::{Context, Error};
+use num::{One, Zero};
+use num_bigint::BigInt;
+use tokay_macros::tokay_function;
+extern crate self as tokay;
+
+#[derive(Clone)]
+struct RangeIter {
+    next: Option<BigInt>,
+    stop: BigInt,
+    step: BigInt,
+}
+
+impl RefValueIter for RangeIter {
+    fn next(&mut self, _context: Option<&mut Context>) -> Option<RefValue> {
+        if let Some(next) = self.next.as_mut() {
+            if *next != self.stop {
+                let ret = next.clone();
+                *next += &self.step;
+                return Some(RefValue::from(ret));
+            }
+
+            self.next = None;
+        }
+
+        None
+    }
+
+    fn repr(&self) -> String {
+        if self.step.is_one() {
+            format!(
+                "range({}, {})",
+                self.next.as_ref().unwrap_or(&self.stop),
+                self.stop
+            )
+        } else {
+            format!(
+                "range({}, {}, {})",
+                self.next.as_ref().unwrap_or(&self.stop),
+                self.stop,
+                self.step
+            )
+        }
+    }
+
+    fn rev(&mut self) -> Result<(), Error> {
+        self.step = -self.step.clone();
+        let next = self.next.as_ref().unwrap_or(&self.stop).clone();
+        (self.next, self.stop) = (Some(self.stop.clone() + &self.step), next + &self.step);
+        Ok(())
+    }
+}
+
+tokay_function!("range : @start, stop=void, step=1", {
+    let start = if stop.is_void() {
+        stop = start;
+        BigInt::from(0)
+    } else {
+        start.to_bigint()?
+    };
+
+    let stop = stop.to_bigint()?;
+    let step = step.to_bigint()?;
+
+    if step.is_zero() {
+        return Error::from(format!("{} argument 'step' may not be 0", __function)).into();
+    }
+
+    RefValue::from(Iter::new(Box::new(RangeIter {
+        next: if (step > BigInt::zero() && start > stop) || (step < BigInt::zero() && stop > start)
+        {
+            None
+        } else {
+            Some(start)
+        },
+        stop,
+        step,
+    })))
+    .into()
+});

src/compiler/ast.rs

@@ -1349,33 +1349,53 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
                 "for" => {
                     let children = node["children"].borrow();
                     let children = children.object::<List>().unwrap();
+                    assert!(children.len() == 3);
 
-                    let (initial, condition, increment, body) = (
+                    let (var, iter_expr, body) = (
                         &children[0].borrow(),
                         &children[1].borrow(),
                         &children[2].borrow(),
-                        &children[3].borrow(),
                     );
 
-                    let initial = initial.object::<Dict>().unwrap();
-                    let condition = condition.object::<Dict>().unwrap();
-                    let increment = increment.object::<Dict>().unwrap();
+                    let var = var.object::<Dict>().unwrap();
+                    let iter_expr = iter_expr.object::<Dict>().unwrap();
                     let body = body.object::<Dict>().unwrap();
 
-                    // Initial
-                    let initial = traverse_node_rvalue(compiler, initial, Rvalue::CallOrLoad);
-                    let condition = traverse_node_rvalue(compiler, condition, Rvalue::CallOrLoad);
-                    let increment = traverse_node_rvalue(compiler, increment, Rvalue::CallOrLoad);
+                    let temp = compiler.pop_temp(); // Borrow a temporary, unnamed variable
+
+                    // Create an iter() on the iter expression
+                    let initial = ImlOp::from(vec![
+                        traverse_node_rvalue(compiler, iter_expr, Rvalue::CallOrLoad),
+                        ImlOp::call(
+                            None,
+                            compiler.get_builtin("iter").unwrap(),
+                            Some((1, false)),
+                        ),
+                        ImlOp::from(Op::StoreFast(temp)),
+                    ]);
+
+                    // Create the condition, which calls iter_next() until void is returned
+                    let condition = ImlOp::from(vec![
+                        ImlOp::from(Op::LoadFast(temp)),
+                        ImlOp::call(
+                            None,
+                            compiler.get_builtin("iter_next").unwrap(),
+                            Some((1, false)),
+                        ),
+                        traverse_node_lvalue(
+                            compiler, var, true, true, //hold for preceding loop break check
+                        ),
+                    ]);
 
                     compiler.loop_push();
                     let body = traverse_node_rvalue(compiler, body, Rvalue::Load);
                     compiler.loop_pop();
+                    compiler.push_temp(temp); // Give temp variable back for possible reuse.
 
                     ImlOp::Loop {
-                        consuming: None,
+                        iterator: true,
                         initial: Box::new(initial),
                         condition: Box::new(condition),
-                        increment: Box::new(increment),
                         body: Box::new(body),
                     }
                 }
@@ -1390,10 +1410,9 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
                             let body = &children[0].borrow();
 
                             ImlOp::Loop {
-                                consuming: None,
+                                iterator: false,
                                 initial: Box::new(ImlOp::Nop),
                                 condition: Box::new(ImlOp::Nop),
-                                increment: Box::new(ImlOp::Nop),
                                 body: Box::new(traverse_node_rvalue(
                                     compiler,
                                     body.object::<Dict>().unwrap(),
@@ -1405,14 +1424,13 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
                             let (condition, body) = (&children[0].borrow(), &children[1].borrow());
 
                             ImlOp::Loop {
-                                consuming: None,
+                                iterator: false,
                                 initial: Box::new(ImlOp::Nop),
                                 condition: Box::new(traverse_node_rvalue(
                                     compiler,
                                     condition.object::<Dict>().unwrap(),
                                     Rvalue::CallOrLoad,
                                 )),
-                                increment: Box::new(ImlOp::Nop),
                                 body: Box::new(traverse_node_rvalue(
                                     compiler,
                                     body.object::<Dict>().unwrap(),