Initial commit. Very rudimentary lexer.

.gitignore

@@ -0,0 +1,2 @@
+/target
+.idea

Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "deimos-lang"
+version = "0.1.0"

Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "deimos-lang"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]

src/lexer/mod.rs

@@ -0,0 +1,103 @@
+#[derive(Debug, PartialEq, Eq)]
+pub enum Token {
+    Namespace,
+    Identifier(String),
+    Public,
+    Module,
+    CurlyOpen,
+    CurlyClose,
+    Interface,
+}
+
+pub fn tokenize(input: &String) -> Vec<Token> {
+    let mut tokens: Vec<Token> = Vec::new();
+    let mut buffer = String::new();
+    let mut peekable = input.chars().peekable();
+    while let Some(c) = peekable.next() {
+        // if not whitespace, append to buffer
+        // else, we should match a token
+        if !c.is_whitespace() {
+            buffer.push(c);
+        } else {
+            tokens.push(match_buffer(&mut buffer));
+            buffer.clear();
+        }
+
+        // check if eof
+        if peekable.peek().is_none() {
+            tokens.push(match_buffer(&mut buffer));
+        }
+    }
+    tokens
+}
+
+fn match_buffer(buffer: &mut String) -> Token {
+    match buffer.as_str() {
+        "int" => Token::Interface,
+        "mod" => Token::Module,
+        "ns" => Token::Namespace,
+        "pub" => Token::Public,
+        "{" => Token::CurlyOpen,
+        "}" => Token::CurlyClose,
+        identifier => Token::Identifier(identifier.to_string()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+    use std::io::Read;
+    use std::path::Path;
+
+    #[test]
+    fn simple_ns() {
+        let result = tokenize(&String::from("ns simple"));
+        assert_eq!(Token::Namespace, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
+    }
+
+    #[test]
+    fn simple_ns_file() {
+        let mut src_file = File::open(Path::new("test-data/lexer/simple_ns.dm")).unwrap();
+        let mut src = String::new();
+        let _ = src_file.read_to_string(&mut src);
+        let result = tokenize(&src);
+        assert_eq!(Token::Namespace, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
+    }
+
+    #[test]
+    fn pub_mod_simple() {
+        let result = tokenize(&String::from("pub mod simple"));
+        assert_eq!(Token::Public, result[0]);
+        assert_eq!(Token::Module, result[1]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[2]);
+    }
+
+    #[test]
+    fn curly_open_and_close() {
+        let result = tokenize(&String::from("{ }"));
+        assert_eq!(Token::CurlyOpen, result[0]);
+        assert_eq!(Token::CurlyClose, result[1]);
+    }
+
+    #[test]
+    fn simple_int() {
+        let result = tokenize(&String::from("int simple"));
+        assert_eq!(Token::Interface, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[1]);
+    }
+
+    #[test]
+    fn ns_pub_mod_simple() {
+        let result = tokenize(&String::from("ns simple_ns\npub mod simple { }"));
+        assert_eq!(Token::Namespace, result[0]);
+        assert_eq!(Token::Identifier(String::from("simple_ns")), result[1]);
+        assert_eq!(Token::Public, result[2]);
+        assert_eq!(Token::Module, result[3]);
+        assert_eq!(Token::Identifier(String::from("simple")), result[4]);
+        assert_eq!(Token::CurlyOpen, result[5]);
+        assert_eq!(Token::CurlyClose, result[6]);
+    }
+}

src/lib.rs

@@ -0,0 +1 @@
+mod lexer;

test-data/lexer/simple_ns.dm

@@ -0,0 +1 @@
+ns simple