jessebrault/deimos-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor lexer for readability and less method calls.

Browse Source
This commit is contained in:
Jesse Brault 2026-03-23 10:47:23 -05:00
parent 916b6377ac
commit 41e798d8a6
1 changed files with 105 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
dmc-lib/src/lexer.rs
View File

@ -34,49 +34,40 @@ impl<'a> Lexer<'a> {
} }
} }
let token = if chunk.starts_with("->") { let mut chars = chunk.chars();
Token::new(self.position, self.position + 2, TokenKind::RightArrow) let current = chars.next().unwrap(); // safe because we return None if chunk is empty above
} else if chunk.starts_with("-") { let peek = chars.next();
Token::new(self.position, self.position + 1, TokenKind::Minus)
} else if chunk.starts_with("(") { let (end, kind) = match current {
Token::new(self.position, self.position + 1, TokenKind::LeftParentheses) '-' => {
} else if chunk.starts_with(")") { if let Some(peek) = peek {
Token::new( if peek == '>' {
self.position, (self.position + 2, TokenKind::RightArrow)
self.position + 1,
TokenKind::RightParentheses,
)
} else if chunk.starts_with("*") {
Token::new(self.position, self.position + 1, TokenKind::Star)
} else if chunk.starts_with("/") {
Token::new(self.position, self.position + 1, TokenKind::Slash)
} else if chunk.starts_with("%") {
Token::new(self.position, self.position + 1, TokenKind::Modulo)
} else if chunk.starts_with("+") {
Token::new(self.position, self.position + 1, TokenKind::Plus)
} else if chunk.starts_with("&") {
Token::new(self.position, self.position + 1, TokenKind::Ampersand)
} else if chunk.starts_with("^") {
Token::new(self.position, self.position + 1, TokenKind::Caret)
} else if chunk.starts_with("|") {
Token::new(self.position, self.position + 1, TokenKind::Bar)
} else if chunk.starts_with("=") {
Token::new(self.position, self.position + 1, TokenKind::Equals)
} else if chunk.starts_with(",") {
Token::new(self.position, self.position + 1, TokenKind::Comma)
} else if chunk.starts_with(":") {
Token::new(self.position, self.position + 1, TokenKind::Colon)
} else if chunk.starts_with(".") {
Token::new(self.position, self.position + 1, TokenKind::Dot)
} else if chunk.starts_with("[") {
Token::new(self.position, self.position + 1, TokenKind::LeftSquare)
} else if chunk.starts_with("]") {
Token::new(self.position, self.position + 1, TokenKind::RightSquare)
} else if chunk.starts_with("<") {
Token::new(self.position, self.position + 1, TokenKind::Lt)
} else if chunk.starts_with(">") {
Token::new(self.position, self.position + 1, TokenKind::Gt)
} else { } else {
(self.position + 1, TokenKind::Minus)
}
} else {
(self.position + 1, TokenKind::Minus)
}
}
'(' => (self.position + 1, TokenKind::LeftParentheses),
')' => (self.position + 1, TokenKind::RightParentheses),
'[' => (self.position + 1, TokenKind::LeftSquare),
']' => (self.position + 1, TokenKind::RightSquare),
'<' => (self.position + 1, TokenKind::Lt),
'>' => (self.position + 1, TokenKind::Gt),
'*' => (self.position + 1, TokenKind::Star),
'/' => (self.position + 1, TokenKind::Slash),
'%' => (self.position + 1, TokenKind::Modulo),
'+' => (self.position + 1, TokenKind::Plus),
'&' => (self.position + 1, TokenKind::Ampersand),
'^' => (self.position + 1, TokenKind::Caret),
'|' => (self.position + 1, TokenKind::Bar),
'=' => (self.position + 1, TokenKind::Equals),
',' => (self.position + 1, TokenKind::Comma),
'.' => (self.position + 1, TokenKind::Dot),
':' => (self.position + 1, TokenKind::Colon),
_ => {
// more than one char token // more than one char token
if chunk.starts_with(|c: char| c.is_ascii_digit()) { if chunk.starts_with(|c: char| c.is_ascii_digit()) {
// number literal // number literal
@ -105,12 +96,12 @@ impl<'a> Lexer<'a> {
} }
} }
if found_fraction { if found_fraction {
Token::new(self.position, end, TokenKind::DoubleLiteral) (end, TokenKind::DoubleLiteral)
} else { } else {
Token::new(self.position, end, TokenKind::IntegerLiteral) (end, TokenKind::IntegerLiteral)
} }
} else { } else {
Token::new(self.position, end, TokenKind::IntegerLiteral) (end, TokenKind::IntegerLiteral)
} }
} else if chunk.starts_with("\"") { } else if chunk.starts_with("\"") {
// string literal // string literal
@ -129,7 +120,7 @@ impl<'a> Lexer<'a> {
if !terminated { if !terminated {
return Some(Err(LexerError::new(LexerErrorKind::UnterminatedString))); return Some(Err(LexerError::new(LexerErrorKind::UnterminatedString)));
} }
Token::new(self.position, end, TokenKind::String) (end, TokenKind::String)
} else { } else {
// keyword or identifier // keyword or identifier
let mut prefix = String::new(); let mut prefix = String::new();
@ -159,9 +150,11 @@ impl<'a> Lexer<'a> {
"ctor" => TokenKind::Ctor, "ctor" => TokenKind::Ctor,
_ => TokenKind::Identifier, _ => TokenKind::Identifier,
}; };
Token::new(self.position, self.position + prefix.len(), token_kind) (self.position + prefix.len(), token_kind)
}
} }
}; };
let token = Token::new(self.position, end, kind);
self.position += token.end() - token.start(); self.position += token.end() - token.start();
Some(Ok(token)) Some(Ok(token))
} }