Update to Pest grammar for hello world.

Cargo.toml

@@ -3,10 +3,6 @@ name = "deimos"
 version = "0.1.0"
 edition = "2021"
 
-[[bin]]
-name = "dmc"
-path = "src/bin/compiler/main.rs"
-
 [[bin]]
 name = "dm"
 path = "src/bin/dvm/main.rs"

dm_std_lib/std/core/array.dm

@@ -0,0 +1,5 @@
+ns std::core
+
+pub int Array<T>
+
+impl ConstantByteArray(fld raw_address: USize, fld raw_size: USize) : Array<Byte>

dm_std_lib/std/core/string.dm

@@ -0,0 +1,5 @@
+ns std::core
+
+pub int String
+
+impl StringImpl(fld bytes: Array<Byte>) : String

sketching/hello_world.dm

@@ -0,0 +1 @@
+println "Hello, World!"

src/bin/compiler/main.rs

@@ -1,22 +0,0 @@
-use deimos::lexer::tokenize;
-use deimos::parser::parse;
-use std::process::exit;
-
-fn main() {
-    let src = String::from("print 42");
-    let tokenize_result = tokenize(&src);
-    if let Err(e) = tokenize_result {
-        eprintln!("{}", e);
-        exit(1);
-    }
-    let tokens = tokenize_result.unwrap();
-    println!("{:?}", tokens);
-    let parse_result = parse(&tokens);
-    if let Err(e) = parse_result {
-        eprintln!("{}", e);
-        exit(1);
-    }
-    let compilation_unit = parse_result.unwrap();
-    println!("{:?}", compilation_unit);
-    // TODO: compilation_unit to DmModule
-}

src/compiler/mod.rs

@@ -0,0 +1,8 @@
+use crate::parser::{DeimosParser, Rule};
+use crate::vm::lib::DmLib;
+use pest::Parser;
+
+pub fn compile(src_unit: &str) -> DmLib {
+    let p = DeimosParser::parse(Rule::compilation_unit, src_unit).unwrap();
+    todo!()
+}

src/lexer/mod.rs

@@ -1,254 +0,0 @@
-use std::iter::Peekable;
-use std::str::Chars;
-
-#[derive(Debug, PartialEq, Eq)]
-pub enum Token {
-    Namespace,
-    Identifier(String),
-    Public,
-    Module,
-    CurlyOpen,
-    CurlyClose,
-    Interface,
-    Colon,
-    Function,
-    ParenOpen,
-    ParenClose,
-    Enum,
-    LessThan,
-    GreaterThan,
-    Intersection,
-    Union,
-    And,
-    Or,
-    Equals,
-    BigArrow,
-    LittleArrow,
-    Plus,
-    Minus,
-    Dot,
-    Ellipsis,
-    Abstract,
-    NumberLiteral(String),
-}
-
-pub fn tokenize(input: &String) -> Result<Vec<Token>, String> {
-    let mut tokens: Vec<Token> = Vec::new();
-    let mut peekable = input.chars().peekable();
-    while let Some(c) = peekable.next() {
-        match c {
-            ' ' | '\n' | '\r' | '\t' => { /* ignore */ }
-            '{' => tokens.push(Token::CurlyOpen),
-            '}' => tokens.push(Token::CurlyClose),
-            ':' => tokens.push(Token::Colon),
-            '(' => tokens.push(Token::ParenOpen),
-            ')' => tokens.push(Token::ParenClose),
-            '<' => tokens.push(Token::LessThan),
-            '>' => tokens.push(Token::GreaterThan),
-            '&' => match peekable.peek() {
-                Some('&') => {
-                    let _ = peekable.next();
-                    tokens.push(Token::And);
-                }
-                Some(_) | None => tokens.push(Token::Intersection),
-            },
-            '|' => match peekable.next_if_eq(&'|') {
-                Some(_) => tokens.push(Token::Or),
-                None => tokens.push(Token::Union),
-            },
-            '=' => match peekable.next_if_eq(&'>') {
-                Some(_) => tokens.push(Token::BigArrow),
-                None => tokens.push(Token::Equals),
-            },
-            '+' => tokens.push(Token::Plus),
-            '-' => match peekable.next_if_eq(&'>') {
-                Some(_) => tokens.push(Token::LittleArrow),
-                None => tokens.push(Token::Minus),
-            },
-            '.' => {
-                let mut count = 1;
-                while let Some(_) = peekable.next_if_eq(&'.') {
-                    count += 1;
-                }
-                match count {
-                    1 => tokens.push(Token::Dot),
-                    3 => tokens.push(Token::Ellipsis),
-                    _ => return Err(String::from("Unexpected number of tokens after '.'")),
-                }
-            }
-            '0'..='9' => {
-                let mut buffer = String::new();
-                buffer.push(c);
-                while let Some(num_char) = peekable.next_if(|c| {
-                    c.is_digit(10)
-                        || match c {
-                            '_' | 'x' | 'L' | 'd' => true,
-                            _ => false,
-                        }
-                }) {
-                    buffer.push(num_char);
-                }
-                tokens.push(Token::NumberLiteral(buffer));
-            }
-            _ => {
-                if let Some(token) = match_identifier_or_keyword(c, &mut peekable) {
-                    tokens.push(token);
-                } else {
-                    return Err(String::from(format!("Unexpected token: {}", c)));
-                }
-            }
-        }
-    }
-    Ok(tokens)
-}
-
-fn match_identifier_or_keyword(start_char: char, peekable: &mut Peekable<Chars>) -> Option<Token> {
-    if !is_valid_identifier_start_char(start_char) {
-        return None;
-    }
-
-    // append start char
-    let mut buffer = String::new();
-    buffer.push(start_char);
-
-    // munch while we have valid identifier chars
-    while let Some(c) = peekable.next_if(|next_char| is_valid_identifier_char(*next_char)) {
-        buffer.push(c);
-    }
-
-    // match to a keyword if possible, else identifier
-    match buffer.as_str() {
-        "abs" => Some(Token::Abstract),
-        "enum" => Some(Token::Enum),
-        "fn" => Some(Token::Function),
-        "int" => Some(Token::Interface),
-        "mod" => Some(Token::Module),
-        "ns" => Some(Token::Namespace),
-        "pub" => Some(Token::Public),
-        _ => Some(Token::Identifier(buffer)),
-    }
-}
-
-fn is_valid_identifier_start_char(c: char) -> bool {
-    match c {
-        'a'..='z' | 'A'..='Z' | '_' => true,
-        _ => false,
-    }
-}
-
-fn is_valid_identifier_char(c: char) -> bool {
-    match c {
-        'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => true,
-        _ => false,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::File;
-    use std::io::Read;
-    use std::path::Path;
-
-    #[test]
-    fn simple_ns() {
-        let result = tokenize(&String::from("ns simple")).unwrap();
-        assert_eq!(Token::Namespace, result[0]);
-        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)]
-    fn simple_ns_file() {
-        let mut src_file = File::open(Path::new("test-data/lexer/simple_ns.dm")).unwrap();
-        let mut src = String::new();
-        let _ = src_file.read_to_string(&mut src);
-        let result = tokenize(&src).unwrap();
-        assert_eq!(Token::Namespace, result[0]);
-        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
-    }
-
-    #[test]
-    fn pub_mod_simple() {
-        let result = tokenize(&String::from("pub mod simple")).unwrap();
-        assert_eq!(Token::Public, result[0]);
-        assert_eq!(Token::Module, result[1]);
-        assert_eq!(Token::Identifier(String::from("simple")), result[2]);
-    }
-
-    #[test]
-    fn curly_open_and_close() {
-        let result = tokenize(&String::from("{ }")).unwrap();
-        assert_eq!(Token::CurlyOpen, result[0]);
-        assert_eq!(Token::CurlyClose, result[1]);
-    }
-
-    #[test]
-    fn simple_int() {
-        let result = tokenize(&String::from("int simple")).unwrap();
-        assert_eq!(Token::Interface, result[0]);
-        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
-    }
-
-    #[test]
-    fn ns_pub_mod_simple() {
-        let result = tokenize(&String::from("ns simple_ns\npub mod simple { }")).unwrap();
-        assert_eq!(Token::Namespace, result[0]);
-        assert_eq!(Token::Identifier(String::from("simple_ns")), result[1]);
-        assert_eq!(Token::Public, result[2]);
-        assert_eq!(Token::Module, result[3]);
-        assert_eq!(Token::Identifier(String::from("simple")), result[4]);
-        assert_eq!(Token::CurlyOpen, result[5]);
-        assert_eq!(Token::CurlyClose, result[6]);
-    }
-
-    #[test]
-    fn curly_open_and_close_no_space() {
-        let result = tokenize(&String::from("{}")).unwrap();
-        assert_eq!(Token::CurlyOpen, result[0]);
-        assert_eq!(Token::CurlyClose, result[1]);
-    }
-
-    #[test]
-    fn interface_function() {
-        let result = tokenize(&String::from("fn test(): Test")).unwrap();
-        assert_eq!(Token::Function, result[0]);
-        assert_eq!(Token::Identifier(String::from("test")), result[1]);
-        assert_eq!(Token::ParenOpen, result[2]);
-        assert_eq!(Token::ParenClose, result[3]);
-        assert_eq!(Token::Colon, result[4]);
-        assert_eq!(Token::Identifier(String::from("Test")), result[5]);
-    }
-
-    #[test]
-    fn interface_prop() {
-        let result = tokenize(&String::from("test: Test")).unwrap();
-        assert_eq!(Token::Identifier(String::from("test")), result[0]);
-        assert_eq!(Token::Colon, result[1]);
-        assert_eq!(Token::Identifier(String::from("Test")), result[2]);
-    }
-
-    #[test]
-    fn enum_decl() {
-        let result = tokenize(&String::from("enum Test {}")).unwrap();
-        assert_eq!(Token::Enum, result[0]);
-        assert_eq!(Token::Identifier(String::from("Test")), result[1]);
-        assert_eq!(Token::CurlyOpen, result[2]);
-        assert_eq!(Token::CurlyClose, result[3]);
-    }
-
-    #[test]
-    fn spread_operator() {
-        let result = tokenize(&String::from("{ ...props }")).unwrap();
-        assert_eq!(Token::CurlyOpen, result[0]);
-        assert_eq!(Token::Ellipsis, result[1]);
-        assert_eq!(Token::Identifier(String::from("props")), result[2]);
-        assert_eq!(Token::CurlyClose, result[3]);
-    }
-
-    #[test]
-    fn simple_number() {
-        let result = tokenize(&String::from("123456")).unwrap();
-        assert_eq!(Token::NumberLiteral(String::from("123456")), result[0]);
-    }
-}

src/lib.rs

@@ -1,4 +1,4 @@
-pub mod lexer;
+mod compiler;
 pub mod parser;
 mod util;
 pub mod vm;

src/parser/deimos.pest

@@ -1,24 +1,31 @@
 compilation_unit = { SOI ~ namespace? ~ declaration* ~ EOI }
 namespace = { "ns" ~ fqn }
 
-fqn = { identifier ~ ( "::" ~ fqn )* }
+fqn = { identifier ~ ( "::" ~ identifier )* }
 identifier = @{ identifier_start_char ~ identifier_char* }
 identifier_start_char = { 'a'..'z' | 'A'..'Z' | "_" }
 identifier_char = { 'a'..'z' | 'A'..'Z' | '0'..'9' | "_" }
 
-declaration = { "decl"? ~ "pub"? ~ ( interface | module | property | function ) }
+declaration = { decl? ~ pub? ~ ( interface | implementation | module | function ) }
+decl = { "decl" }
+pub = { "pub" }
 
-interface = { "int" ~ identifier ~ generics_declaration? ~ interface_extends_list? ~ "{" ~ declaration* ~ "}"}
+interface = { "int" ~ identifier ~ generics_declaration? ~ extends_list? ~ ( "{" ~ declaration* ~ "}" )? }
-interface_extends_list = { ":" ~ type ~ ( "+" ~ type )* }
+extends_list = { ":" ~ type ~ ( "+" ~ type )* }
+
+implementation = { "impl" ~ identifier ~ generics_declaration? ~ impl_ctor? ~ extends_list? }
+impl_ctor = { "(" ~ impl_ctor_args? ~ ")" }
+impl_ctor_args = { impl_ctor_arg ~ ( "," ~ impl_ctor_arg )* }
+impl_ctor_arg = { fld? ~ identifier ~ ":" ~ type }
+fld = { "fld" }
 
 module = { "mod" ~ identifier ~ "{" ~ declaration* ~ "}" }
 
 property = { identifier ~ ":" ~ type }
 
-function = { "fn" ~ generics_declaration? ~ identifier ~ "(" ~ args_list? ~ ")" ~ ( ":" ~ type )? ~ ( "{" ~ "}" )? }
+function = { "fn" ~ generics_declaration? ~ identifier ~ "(" ~ args_list? ~ ")" ~ ( ":" ~ type )? ~ ( function_body | function_equals_body ) }
-
 function_equals_body = { "=" ~ expression }
-function_body = { "{" ~ "}" }
+function_body = { "{" ~ statement* ~ "}" }
 
 type = { identifier ~ generics_declaration? }
 generics_declaration = { "<" ~ identifier ~ ( "," ~ identifier )* ~ ">"}
@@ -26,12 +33,29 @@ generics_declaration = { "<" ~ identifier ~ ( "," ~ identifier )* ~ ">"}
 args_list = { arg ~ ( "," ~ arg )* }
 arg = { identifier ~ ( ":" ~ "..."? ~ type )? }
 
-expression = { literal }
+statement = { call_stmt }
 
-literal = { number_literal }
+call_stmt = { call_expr }
+
+call_expr = { call_expr_with_parens | call_expr_no_parens }
+call_expr_with_parens = { fqn ~ "(" ~ call_args? ~ ")" }
+call_expr_no_parens = { fqn ~ call_args }
+call_args = { call_arg ~ ( "," ~ call_arg )* }
+call_arg = { expression }
+
+expression = { literal | identifier }
+
+literal = { number_literal | string_literal }
 
 number_literal = { int_literal }
-
 int_literal = { '0'..'9'+ }
 
-WHITESPACE = { " " | "\t" | "\n" | "\r" }
+string_literal = ${ "\"" ~ string_inner ~ "\"" }
+string_inner = @{ string_char* }
+string_char = {
+    !( "\"" | "\\" ) ~ ANY
+    | "\\" ~ ( "\"" | "\\" | "/" | "b" | "f" | "n" | "r" | "t" )
+    | "\\" ~ ( "u" ~ ASCII_HEX_DIGIT{4} )
+}
+
+WHITESPACE = _{ " " | "\t" | "\n" | "\r" }

src/parser/mod.rs

@@ -1,18 +1,5 @@
-mod types;
-
-use crate::lexer::Token;
-use crate::parser::types::AstNode;
-use pest::Parser;
 use pest_derive::Parser;
 
 #[derive(Parser)]
 #[grammar = "parser/deimos.pest"]
-struct DeimosParser;
+pub struct DeimosParser;
-
-pub fn parse(tokens: &Vec<Token>) -> Result<AstNode, String> {
-    let p = DeimosParser::parse(Rule::compilation_unit, "ns std::core")
-        .expect("unable to parse")
-        .next()
-        .unwrap();
-    todo!()
-}

src/parser/types.rs

@@ -1,11 +0,0 @@
-use std::fmt::Debug;
-
-pub type NodeChildren = Vec<Box<AstNode>>;
-
-#[derive(Debug)]
-pub enum AstNode {
-    CompilationUnit(NodeChildren),
-    BlockStatement(NodeChildren),
-    Statement(NodeChildren),
-    Expression(NodeChildren),
-}