Cargo.toml

@@ -3,6 +3,10 @@ name = "deimos"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "dmc"
+path = "src/bin/compiler/main.rs"
+
 [[bin]]
 name = "dm"
 path = "src/bin/dvm/main.rs"

dm_std_lib/std/core/array.dm

@@ -1,5 +0,0 @@
-ns std::core
-
-pub int Array<T>
-
-impl ConstantByteArray(fld raw_address: USize, fld raw_size: USize) : Array<Byte>

dm_std_lib/std/core/string.dm

@@ -1,5 +0,0 @@
-ns std::core
-
-pub int String
-
-impl StringImpl(fld bytes: Array<Byte>) : String

sketching/hello_world.dm

@@ -1 +0,0 @@
-println "Hello, World!"

src/bin/compiler/main.rs

@@ -0,0 +1,22 @@
+use deimos::lexer::tokenize;
+use deimos::parser::parse;
+use std::process::exit;
+
+fn main() {
+    let src = String::from("print 42");
+    let tokenize_result = tokenize(&src);
+    if let Err(e) = tokenize_result {
+        eprintln!("{}", e);
+        exit(1);
+    }
+    let tokens = tokenize_result.unwrap();
+    println!("{:?}", tokens);
+    let parse_result = parse(&tokens);
+    if let Err(e) = parse_result {
+        eprintln!("{}", e);
+        exit(1);
+    }
+    let compilation_unit = parse_result.unwrap();
+    println!("{:?}", compilation_unit);
+    // TODO: compilation_unit to DmModule
+}

src/compiler/mod.rs

@@ -1,8 +0,0 @@
-use crate::parser::{DeimosParser, Rule};
-use crate::vm::lib::DmLib;
-use pest::Parser;
-
-pub fn compile(src_unit: &str) -> DmLib {
-    let p = DeimosParser::parse(Rule::compilation_unit, src_unit).unwrap();
-    todo!()
-}

src/lexer/mod.rs

@@ -0,0 +1,254 @@
+use std::iter::Peekable;
+use std::str::Chars;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum Token {
+    Namespace,
+    Identifier(String),
+    Public,
+    Module,
+    CurlyOpen,
+    CurlyClose,
+    Interface,
+    Colon,
+    Function,
+    ParenOpen,
+    ParenClose,
+    Enum,
+    LessThan,
+    GreaterThan,
+    Intersection,
+    Union,
+    And,
+    Or,
+    Equals,
+    BigArrow,
+    LittleArrow,
+    Plus,
+    Minus,
+    Dot,
+    Ellipsis,
+    Abstract,
+    NumberLiteral(String),
+}
+
+pub fn tokenize(input: &String) -> Result<Vec<Token>, String> {
+    let mut tokens: Vec<Token> = Vec::new();
+    let mut peekable = input.chars().peekable();
+    while let Some(c) = peekable.next() {
+        match c {
+            ' ' | '\n' | '\r' | '\t' => { /* ignore */ }
+            '{' => tokens.push(Token::CurlyOpen),
+            '}' => tokens.push(Token::CurlyClose),
+            ':' => tokens.push(Token::Colon),
+            '(' => tokens.push(Token::ParenOpen),
+            ')' => tokens.push(Token::ParenClose),
+            '<' => tokens.push(Token::LessThan),
+            '>' => tokens.push(Token::GreaterThan),
+            '&' => match peekable.peek() {
+                Some('&') => {
+                    let _ = peekable.next();
+                    tokens.push(Token::And);
+                }
+                Some(_) | None => tokens.push(Token::Intersection),
+            },
+            '|' => match peekable.next_if_eq(&'|') {
+                Some(_) => tokens.push(Token::Or),
+                None => tokens.push(Token::Union),
+            },
+            '=' => match peekable.next_if_eq(&'>') {
+                Some(_) => tokens.push(Token::BigArrow),
+                None => tokens.push(Token::Equals),
+            },
+            '+' => tokens.push(Token::Plus),
+            '-' => match peekable.next_if_eq(&'>') {
+                Some(_) => tokens.push(Token::LittleArrow),
+                None => tokens.push(Token::Minus),
+            },
+            '.' => {
+                let mut count = 1;
+                while let Some(_) = peekable.next_if_eq(&'.') {
+                    count += 1;
+                }
+                match count {
+                    1 => tokens.push(Token::Dot),
+                    3 => tokens.push(Token::Ellipsis),
+                    _ => return Err(String::from("Unexpected number of tokens after '.'")),
+                }
+            }
+            '0'..='9' => {
+                let mut buffer = String::new();
+                buffer.push(c);
+                while let Some(num_char) = peekable.next_if(|c| {
+                    c.is_digit(10)
+                        || match c {
+                            '_' | 'x' | 'L' | 'd' => true,
+                            _ => false,
+                        }
+                }) {
+                    buffer.push(num_char);
+                }
+                tokens.push(Token::NumberLiteral(buffer));
+            }
+            _ => {
+                if let Some(token) = match_identifier_or_keyword(c, &mut peekable) {
+                    tokens.push(token);
+                } else {
+                    return Err(String::from(format!("Unexpected token: {}", c)));
+                }
+            }
+        }
+    }
+    Ok(tokens)
+}
+
+fn match_identifier_or_keyword(start_char: char, peekable: &mut Peekable<Chars>) -> Option<Token> {
+    if !is_valid_identifier_start_char(start_char) {
+        return None;
+    }
+
+    // append start char
+    let mut buffer = String::new();
+    buffer.push(start_char);
+
+    // munch while we have valid identifier chars
+    while let Some(c) = peekable.next_if(|next_char| is_valid_identifier_char(*next_char)) {
+        buffer.push(c);
+    }
+
+    // match to a keyword if possible, else identifier
+    match buffer.as_str() {
+        "abs" => Some(Token::Abstract),
+        "enum" => Some(Token::Enum),
+        "fn" => Some(Token::Function),
+        "int" => Some(Token::Interface),
+        "mod" => Some(Token::Module),
+        "ns" => Some(Token::Namespace),
+        "pub" => Some(Token::Public),
+        _ => Some(Token::Identifier(buffer)),
+    }
+}
+
+fn is_valid_identifier_start_char(c: char) -> bool {
+    match c {
+        'a'..='z' | 'A'..='Z' | '_' => true,
+        _ => false,
+    }
+}
+
+fn is_valid_identifier_char(c: char) -> bool {
+    match c {
+        'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => true,
+        _ => false,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+    use std::io::Read;
+    use std::path::Path;
+
+    #[test]
+    fn simple_ns() {
+        let result = tokenize(&String::from("ns simple")).unwrap();
+        assert_eq!(Token::Namespace, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
+    }
+
+    #[test]
+    #[cfg_attr(miri, ignore)]
+    fn simple_ns_file() {
+        let mut src_file = File::open(Path::new("test-data/lexer/simple_ns.dm")).unwrap();
+        let mut src = String::new();
+        let _ = src_file.read_to_string(&mut src);
+        let result = tokenize(&src).unwrap();
+        assert_eq!(Token::Namespace, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
+    }
+
+    #[test]
+    fn pub_mod_simple() {
+        let result = tokenize(&String::from("pub mod simple")).unwrap();
+        assert_eq!(Token::Public, result[0]);
+        assert_eq!(Token::Module, result[1]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[2]);
+    }
+
+    #[test]
+    fn curly_open_and_close() {
+        let result = tokenize(&String::from("{ }")).unwrap();
+        assert_eq!(Token::CurlyOpen, result[0]);
+        assert_eq!(Token::CurlyClose, result[1]);
+    }
+
+    #[test]
+    fn simple_int() {
+        let result = tokenize(&String::from("int simple")).unwrap();
+        assert_eq!(Token::Interface, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
+    }
+
+    #[test]
+    fn ns_pub_mod_simple() {
+        let result = tokenize(&String::from("ns simple_ns\npub mod simple { }")).unwrap();
+        assert_eq!(Token::Namespace, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple_ns")), result[1]);
+        assert_eq!(Token::Public, result[2]);
+        assert_eq!(Token::Module, result[3]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[4]);
+        assert_eq!(Token::CurlyOpen, result[5]);
+        assert_eq!(Token::CurlyClose, result[6]);
+    }
+
+    #[test]
+    fn curly_open_and_close_no_space() {
+        let result = tokenize(&String::from("{}")).unwrap();
+        assert_eq!(Token::CurlyOpen, result[0]);
+        assert_eq!(Token::CurlyClose, result[1]);
+    }
+
+    #[test]
+    fn interface_function() {
+        let result = tokenize(&String::from("fn test(): Test")).unwrap();
+        assert_eq!(Token::Function, result[0]);
+        assert_eq!(Token::Identifier(String::from("test")), result[1]);
+        assert_eq!(Token::ParenOpen, result[2]);
+        assert_eq!(Token::ParenClose, result[3]);
+        assert_eq!(Token::Colon, result[4]);
+        assert_eq!(Token::Identifier(String::from("Test")), result[5]);
+    }
+
+    #[test]
+    fn interface_prop() {
+        let result = tokenize(&String::from("test: Test")).unwrap();
+        assert_eq!(Token::Identifier(String::from("test")), result[0]);
+        assert_eq!(Token::Colon, result[1]);
+        assert_eq!(Token::Identifier(String::from("Test")), result[2]);
+    }
+
+    #[test]
+    fn enum_decl() {
+        let result = tokenize(&String::from("enum Test {}")).unwrap();
+        assert_eq!(Token::Enum, result[0]);
+        assert_eq!(Token::Identifier(String::from("Test")), result[1]);
+        assert_eq!(Token::CurlyOpen, result[2]);
+        assert_eq!(Token::CurlyClose, result[3]);
+    }
+
+    #[test]
+    fn spread_operator() {
+        let result = tokenize(&String::from("{ ...props }")).unwrap();
+        assert_eq!(Token::CurlyOpen, result[0]);
+        assert_eq!(Token::Ellipsis, result[1]);
+        assert_eq!(Token::Identifier(String::from("props")), result[2]);
+        assert_eq!(Token::CurlyClose, result[3]);
+    }
+
+    #[test]
+    fn simple_number() {
+        let result = tokenize(&String::from("123456")).unwrap();
+        assert_eq!(Token::NumberLiteral(String::from("123456")), result[0]);
+    }
+}

src/lib.rs

@@ -1,4 +1,4 @@
-mod compiler;
+pub mod lexer;
 pub mod parser;
 mod util;
 pub mod vm;

src/parser/deimos.pest

@@ -1,31 +1,24 @@
 compilation_unit = { SOI ~ namespace? ~ declaration* ~ EOI }
 namespace = { "ns" ~ fqn }
 
-fqn = { identifier ~ ( "::" ~ identifier )* }
+fqn = { identifier ~ ( "::" ~ fqn )* }
 identifier = @{ identifier_start_char ~ identifier_char* }
 identifier_start_char = { 'a'..'z' | 'A'..'Z' | "_" }
 identifier_char = { 'a'..'z' | 'A'..'Z' | '0'..'9' | "_" }
 
-declaration = { decl? ~ pub? ~ ( interface | implementation | module | function ) }
+declaration = { "decl"? ~ "pub"? ~ ( interface | module | property | function ) }
-decl = { "decl" }
-pub = { "pub" }
 
-interface = { "int" ~ identifier ~ generics_declaration? ~ extends_list? ~ ( "{" ~ declaration* ~ "}" )? }
+interface = { "int" ~ identifier ~ generics_declaration? ~ interface_extends_list? ~ "{" ~ declaration* ~ "}"}
-extends_list = { ":" ~ type ~ ( "+" ~ type )* }
+interface_extends_list = { ":" ~ type ~ ( "+" ~ type )* }
-
-implementation = { "impl" ~ identifier ~ generics_declaration? ~ impl_ctor? ~ extends_list? }
-impl_ctor = { "(" ~ impl_ctor_args? ~ ")" }
-impl_ctor_args = { impl_ctor_arg ~ ( "," ~ impl_ctor_arg )* }
-impl_ctor_arg = { fld? ~ identifier ~ ":" ~ type }
-fld = { "fld" }
 
 module = { "mod" ~ identifier ~ "{" ~ declaration* ~ "}" }
 
 property = { identifier ~ ":" ~ type }
 
-function = { "fn" ~ generics_declaration? ~ identifier ~ "(" ~ args_list? ~ ")" ~ ( ":" ~ type )? ~ ( function_body | function_equals_body ) }
+function = { "fn" ~ generics_declaration? ~ identifier ~ "(" ~ args_list? ~ ")" ~ ( ":" ~ type )? ~ ( "{" ~ "}" )? }
+
 function_equals_body = { "=" ~ expression }
-function_body = { "{" ~ statement* ~ "}" }
+function_body = { "{" ~ "}" }
 
 type = { identifier ~ generics_declaration? }
 generics_declaration = { "<" ~ identifier ~ ( "," ~ identifier )* ~ ">"}
@@ -33,29 +26,12 @@ generics_declaration = { "<" ~ identifier ~ ( "," ~ identifier )* ~ ">"}
 args_list = { arg ~ ( "," ~ arg )* }
 arg = { identifier ~ ( ":" ~ "..."? ~ type )? }
 
-statement = { call_stmt }
+expression = { literal }
 
-call_stmt = { call_expr }
+literal = { number_literal }
-
-call_expr = { call_expr_with_parens | call_expr_no_parens }
-call_expr_with_parens = { fqn ~ "(" ~ call_args? ~ ")" }
-call_expr_no_parens = { fqn ~ call_args }
-call_args = { call_arg ~ ( "," ~ call_arg )* }
-call_arg = { expression }
-
-expression = { literal | identifier }
-
-literal = { number_literal | string_literal }
 
 number_literal = { int_literal }
+
 int_literal = { '0'..'9'+ }
 
-string_literal = ${ "\"" ~ string_inner ~ "\"" }
+WHITESPACE = { " " | "\t" | "\n" | "\r" }
-string_inner = @{ string_char* }
-string_char = {
-    !( "\"" | "\\" ) ~ ANY
-    | "\\" ~ ( "\"" | "\\" | "/" | "b" | "f" | "n" | "r" | "t" )
-    | "\\" ~ ( "u" ~ ASCII_HEX_DIGIT{4} )
-}
-
-WHITESPACE = _{ " " | "\t" | "\n" | "\r" }

src/parser/mod.rs

@@ -1,5 +1,18 @@
+mod types;
+
+use crate::lexer::Token;
+use crate::parser::types::AstNode;
+use pest::Parser;
 use pest_derive::Parser;
 
 #[derive(Parser)]
 #[grammar = "parser/deimos.pest"]
-pub struct DeimosParser;
+struct DeimosParser;
+
+pub fn parse(tokens: &Vec<Token>) -> Result<AstNode, String> {
+    let p = DeimosParser::parse(Rule::compilation_unit, "ns std::core")
+        .expect("unable to parse")
+        .next()
+        .unwrap();
+    todo!()
+}

src/parser/types.rs

@@ -0,0 +1,11 @@
+use std::fmt::Debug;
+
+pub type NodeChildren = Vec<Box<AstNode>>;
+
+#[derive(Debug)]
+pub enum AstNode {
+    CompilationUnit(NodeChildren),
+    BlockStatement(NodeChildren),
+    Statement(NodeChildren),
+    Expression(NodeChildren),
+}