Very skeleton work on parser.

src/bin/compiler/main.rs

@@ -1,13 +1,22 @@
 use deimos::lexer::tokenize;
 use std::process::exit;
+use deimos::parser::parse;
 
 fn main() {
-    let src = String::from("print 'Hello, World!'");
+    let src = String::from("print 42");
-    let r = tokenize(&src);
+    let tokenize_result = tokenize(&src);
-    if let Err(e) = r {
+    if let Err(e) = tokenize_result {
         eprintln!("{}", e);
         exit(1);
     }
-    let tokens = r.unwrap();
+    let tokens = tokenize_result.unwrap();
     println!("{:?}", tokens);
+    let parse_result = parse(&tokens);
+    if let Err(e) = parse_result {
+        eprintln!("{}", e);
+        exit(1);
+    }
+    let compilation_unit = parse_result.unwrap();
+    println!("{:?}", compilation_unit);
+    // TODO: compilation_unit to DmModule
 }

src/lexer/mod.rs

@@ -29,6 +29,7 @@ pub enum Token {
     Dot,
     Ellipsis,
     Abstract,
+    NumberLiteral(String),
 }
 
 pub fn tokenize(input: &String) -> Result<Vec<Token>, String> {
@@ -75,6 +76,19 @@ pub fn tokenize(input: &String) -> Result<Vec<Token>, String> {
                     _ => return Err(String::from("Unexpected number of tokens after '.'")),
                 }
             }
+            '0'..='9' => {
+                let mut buffer = String::new();
+                buffer.push(c);
+                while let Some(num_char) = peekable.next_if(|c| {
+                    c.is_digit(10) || match c {
+                        '_' | 'x' | 'L' | 'd' => true,
+                        _ => false,
+                    }
+                }) {
+                    buffer.push(num_char);
+                }
+                tokens.push(Token::NumberLiteral(buffer));
+            }
             _ => {
                 if let Some(token) = match_identifier_or_keyword(c, &mut peekable) {
                     tokens.push(token);
@@ -230,4 +244,10 @@ mod tests {
         assert_eq!(Token::Identifier(String::from("props")), result[2]);
         assert_eq!(Token::CurlyClose, result[3]);
     }
+
+    #[test]
+    fn simple_number() {
+        let result = tokenize(&String::from("123456")).unwrap();
+        assert_eq!(Token::NumberLiteral(String::from("123456")), result[0]);
+    }
 }

src/lib.rs

@@ -1,2 +1,3 @@
 pub mod lexer;
 pub mod vm;
+pub mod parser;

src/parser/mod.rs

@@ -0,0 +1,8 @@
+mod types;
+
+use crate::lexer::Token;
+use crate::parser::types::AstNode;
+
+pub fn parse(tokens: &Vec<Token>) -> Result<AstNode, String> {
+    todo!()
+}

src/parser/types.rs

@@ -0,0 +1,11 @@
+use std::fmt::Debug;
+
+pub type NodeChildren = Vec<Box<AstNode>>;
+
+#[derive(Debug)]
+pub enum AstNode {
+    CompilationUnit(NodeChildren),
+    BlockStatement(NodeChildren),
+    Statement(NodeChildren),
+    Expression(NodeChildren),
+}