jessebrault/deimos-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor lexer for readability and less method calls.

Browse Source
This commit is contained in:
Jesse Brault 2026-03-23 10:47:23 -05:00
parent 916b6377ac
commit 41e798d8a6
1 changed files with 105 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
dmc-lib/src/lexer.rs
View File

@ -34,49 +34,40 @@ impl<'a> Lexer<'a> {
}
}
let token = if chunk.starts_with("->") {
Token::new(self.position, self.position + 2, TokenKind::RightArrow)
} else if chunk.starts_with("-") {
Token::new(self.position, self.position + 1, TokenKind::Minus)
} else if chunk.starts_with("(") {
Token::new(self.position, self.position + 1, TokenKind::LeftParentheses)
} else if chunk.starts_with(")") {
Token::new(
self.position,
self.position + 1,
TokenKind::RightParentheses,
)
} else if chunk.starts_with("*") {
Token::new(self.position, self.position + 1, TokenKind::Star)
} else if chunk.starts_with("/") {
Token::new(self.position, self.position + 1, TokenKind::Slash)
} else if chunk.starts_with("%") {
Token::new(self.position, self.position + 1, TokenKind::Modulo)
} else if chunk.starts_with("+") {
Token::new(self.position, self.position + 1, TokenKind::Plus)
} else if chunk.starts_with("&") {
Token::new(self.position, self.position + 1, TokenKind::Ampersand)
} else if chunk.starts_with("^") {
Token::new(self.position, self.position + 1, TokenKind::Caret)
} else if chunk.starts_with("|") {
Token::new(self.position, self.position + 1, TokenKind::Bar)
} else if chunk.starts_with("=") {
Token::new(self.position, self.position + 1, TokenKind::Equals)
} else if chunk.starts_with(",") {
Token::new(self.position, self.position + 1, TokenKind::Comma)
} else if chunk.starts_with(":") {
Token::new(self.position, self.position + 1, TokenKind::Colon)
} else if chunk.starts_with(".") {
Token::new(self.position, self.position + 1, TokenKind::Dot)
} else if chunk.starts_with("[") {
Token::new(self.position, self.position + 1, TokenKind::LeftSquare)
} else if chunk.starts_with("]") {
Token::new(self.position, self.position + 1, TokenKind::RightSquare)
} else if chunk.starts_with("<") {
Token::new(self.position, self.position + 1, TokenKind::Lt)
} else if chunk.starts_with(">") {
Token::new(self.position, self.position + 1, TokenKind::Gt)
let mut chars = chunk.chars();
let current = chars.next().unwrap(); // safe because we return None if chunk is empty above
let peek = chars.next();
let (end, kind) = match current {
'-' => {
if let Some(peek) = peek {
if peek == '>' {
(self.position + 2, TokenKind::RightArrow)
} else {
(self.position + 1, TokenKind::Minus)
}
} else {
(self.position + 1, TokenKind::Minus)
}
}
'(' => (self.position + 1, TokenKind::LeftParentheses),
')' => (self.position + 1, TokenKind::RightParentheses),
'[' => (self.position + 1, TokenKind::LeftSquare),
']' => (self.position + 1, TokenKind::RightSquare),
'<' => (self.position + 1, TokenKind::Lt),
'>' => (self.position + 1, TokenKind::Gt),
'*' => (self.position + 1, TokenKind::Star),
'/' => (self.position + 1, TokenKind::Slash),
'%' => (self.position + 1, TokenKind::Modulo),
'+' => (self.position + 1, TokenKind::Plus),
'&' => (self.position + 1, TokenKind::Ampersand),
'^' => (self.position + 1, TokenKind::Caret),
'|' => (self.position + 1, TokenKind::Bar),
'=' => (self.position + 1, TokenKind::Equals),
',' => (self.position + 1, TokenKind::Comma),
'.' => (self.position + 1, TokenKind::Dot),
':' => (self.position + 1, TokenKind::Colon),
_ => {
// more than one char token
if chunk.starts_with(|c: char| c.is_ascii_digit()) {
// number literal
@ -105,12 +96,12 @@ impl<'a> Lexer<'a> {
}
}
if found_fraction {
Token::new(self.position, end, TokenKind::DoubleLiteral)
(end, TokenKind::DoubleLiteral)
} else {
Token::new(self.position, end, TokenKind::IntegerLiteral)
(end, TokenKind::IntegerLiteral)
}
} else {
Token::new(self.position, end, TokenKind::IntegerLiteral)
(end, TokenKind::IntegerLiteral)
}
} else if chunk.starts_with("\"") {
// string literal
@ -129,7 +120,7 @@ impl<'a> Lexer<'a> {
if !terminated {
return Some(Err(LexerError::new(LexerErrorKind::UnterminatedString)));
}
Token::new(self.position, end, TokenKind::String)
(end, TokenKind::String)
} else {
// keyword or identifier
let mut prefix = String::new();
@ -159,9 +150,11 @@ impl<'a> Lexer<'a> {
"ctor" => TokenKind::Ctor,
_ => TokenKind::Identifier,
};
Token::new(self.position, self.position + prefix.len(), token_kind)
(self.position + prefix.len(), token_kind)
}
}
};
let token = Token::new(self.position, end, kind);
self.position += token.end() - token.start();
Some(Ok(token))
}