deimos-lang/dmc-lib/src/parser.rs

1528 lines
48 KiB
Rust

use crate::FileId;
use crate::ast::NodeId;
use crate::ast::assign_statement::AssignStatement;
use crate::ast::binary_expression::{BinaryExpression, BinaryOperation};
use crate::ast::call::Call;
use crate::ast::class::Class;
use crate::ast::compilation_unit::CompilationUnit;
use crate::ast::constructor::Constructor;
use crate::ast::double_literal::DoubleLiteral;
use crate::ast::expression::Expression;
use crate::ast::expression_statement::ExpressionStatement;
use crate::ast::extern_function::ExternFunction;
use crate::ast::field::Field;
use crate::ast::function::Function;
use crate::ast::generic_parameter::GenericParameter;
use crate::ast::identifier::Identifier;
use crate::ast::integer_literal::IntegerLiteral;
use crate::ast::let_statement::LetStatement;
use crate::ast::negative_expression::NegativeExpression;
use crate::ast::parameter::Parameter;
use crate::ast::path::Path;
use crate::ast::statement::Statement;
use crate::ast::string_literal::StringLiteral;
use crate::ast::type_use::TypeUse;
use crate::diagnostic::{Diagnostic, Diagnostics};
use crate::error_codes::{LEXER_ERROR, PARSE_ERROR};
use crate::lexer::{Lexer, LexerErrorKind};
use crate::source_range::SourceRange;
use crate::token::{Token, TokenKind};
use std::str::FromStr;
pub struct ParseResult<T>(pub T, pub Diagnostics);
impl<T> ParseResult<T> {
pub fn is_ok(&self) -> bool {
self.1.is_empty()
}
pub fn to_result(self) -> Result<T, Diagnostics> {
if self.is_ok() {
Ok(self.0)
} else {
Err(self.1)
}
}
}
pub fn get_compilation_unit(
input: &str,
file_id: Option<FileId>,
) -> Result<CompilationUnit, Diagnostics> {
parse_compilation_unit(input, file_id).to_result()
}
pub fn parse_compilation_unit(
input: &str,
file_id: Option<FileId>,
) -> ParseResult<CompilationUnit> {
let mut parser = Parser::new(input);
parser.advance();
parser.compilation_unit(file_id)
}
pub fn parse_statement(input: &str) -> ParseResult<Option<Statement>> {
let mut parser = Parser::new(input);
parser.advance();
ParseResult(parser.statement(), parser.diagnostics)
}
pub fn parse_expression(input: &str) -> ParseResult<Option<Expression>> {
let mut parser = Parser::new(input);
parser.advance();
ParseResult(parser.expression(), parser.diagnostics)
}
macro_rules! matches_expression_first {
( $token_kind : expr ) => {
matches!(
$token_kind,
TokenKind::IntegerLiteral
| TokenKind::DoubleLiteral
| TokenKind::LongLiteral
| TokenKind::String
| TokenKind::Minus
| TokenKind::SelfKw
| TokenKind::Identifier
)
};
}
const EXPRESSION_FIRSTS: [TokenKind; 7] = [
TokenKind::IntegerLiteral,
TokenKind::DoubleLiteral,
TokenKind::LongLiteral,
TokenKind::String,
TokenKind::Minus,
TokenKind::SelfKw,
TokenKind::Identifier,
];
macro_rules! matches_statement_first {
( $token_kind : expr ) => {
matches!($token_kind, TokenKind::Let) || matches_expression_first!($token_kind)
};
}
const STATEMENT_FIRSTS: [TokenKind; 8] = {
let mut firsts: [TokenKind; 8] = [TokenKind::Let; 8];
let mut i = 0;
while i < EXPRESSION_FIRSTS.len() {
firsts[i + 1] = EXPRESSION_FIRSTS[i];
i += 1;
}
firsts
};
macro_rules! matches_type_use_first {
( $token_kind: expr ) => {
matches!($token_kind, TokenKind::LeftSquare | TokenKind::Identifier)
};
}
fn join_kinds(kinds: &[TokenKind]) -> String {
kinds
.iter()
.map(|kind| format!("{:?}", kind))
.collect::<Vec<_>>()
.join(", ")
}
fn get_expected_but_found(kinds: &[TokenKind], found: &Token) -> Diagnostic {
Diagnostic::new(
&format!("Unexpected token: {:?}.", found.kind()),
found.start(),
found.end(),
)
.with_error_code(PARSE_ERROR)
.with_primary_label_message(&format!("Expected {}.", join_kinds(kinds)))
}
fn get_expected_but_found_eoi(kinds: &[TokenKind], position: usize) -> Diagnostic {
Diagnostic::new("Unexpected end-of-input.", position, position)
.with_error_code(PARSE_ERROR)
.with_primary_label_message(&format!("Expected {}.", join_kinds(kinds)))
}
struct Parser<'a> {
input: &'a str,
lexer: Lexer<'a>,
current: Option<Token>,
lookahead: Option<Token>,
current_node_id: NodeId,
diagnostics: Diagnostics,
}
impl<'a> Parser<'a> {
fn new(input: &'a str) -> Self {
Self {
input,
lexer: Lexer::new(input),
current: None,
lookahead: None,
current_node_id: 0,
diagnostics: Diagnostics::new(),
}
}
fn advance(&mut self) {
fn fetch(lexer: &mut Lexer, diagnostics: &mut Diagnostics) -> Option<Token> {
let mut maybe_token: Option<Token> = None;
while let Some(lexer_result) = lexer.next() {
match lexer_result {
Ok(token) => {
maybe_token = Some(token);
break;
}
Err(lexer_error) => {
let diagnostic = match lexer_error.kind() {
LexerErrorKind::UnterminatedString => Diagnostic::new(
"Unterminated string literal.",
lexer_error.start(),
lexer_error.end(),
)
.with_error_code(LEXER_ERROR),
LexerErrorKind::UnrecognizedCharacter(c) => Diagnostic::new(
&format!("Unrecognized character: {}", c),
lexer_error.start(),
lexer_error.end(),
)
.with_error_code(LEXER_ERROR),
};
diagnostics.push(diagnostic);
}
}
}
maybe_token
}
if self.lookahead.is_some() {
// we've advanced at least once
self.current = self.lookahead.take();
self.lookahead = fetch(&mut self.lexer, &mut self.diagnostics);
} else if self.lookahead.is_none() && self.current.is_some() {
// we're on the last token
self.current = None;
} else {
// we've not yet advanced, so fetch both
self.current = fetch(&mut self.lexer, &mut self.diagnostics);
self.lookahead = fetch(&mut self.lexer, &mut self.diagnostics);
}
}
fn expect_advance(&mut self, token_kind: TokenKind) -> Option<Token> {
match self.current.take() {
None => {
self.diagnostics
.push(get_expected_but_found_eoi(&[token_kind], self.input.len()));
None
}
Some(token) => {
if token.kind() == token_kind {
self.advance();
Some(token)
} else {
self.diagnostics
.push(get_expected_but_found(&[token_kind], &token));
None
}
}
}
}
fn expect_immediately_after_advance(
&mut self,
token_kind: TokenKind,
previous_token: &Token,
) -> Option<Token> {
let maybe_matched = self.expect_advance(token_kind);
match maybe_matched {
None => None,
Some(matched) => {
if matched.start() == previous_token.end() {
Some(matched)
} else {
self.diagnostics.push(
Diagnostic::new(
&format!(
"Expected {:?} immediately after previous token.",
token_kind
),
matched.start(),
matched.end(),
)
.with_error_code(PARSE_ERROR),
);
None
}
}
}
}
fn peek_current(&self, token_kind: TokenKind) -> bool {
match &self.current {
None => panic!("BUG: peeked when current is None"),
Some(token) => token.kind() == token_kind,
}
}
fn sample_input(&self, start: usize, end: usize) -> &'a str {
&self.input[start..end]
}
fn token_text(&self, token: &Token) -> &'a str {
self.sample_input(token.start(), token.end())
}
fn next_node_id(&mut self) -> NodeId {
let node_id = self.current_node_id;
self.current_node_id += 1;
node_id
}
fn compilation_unit(&mut self, file_id: Option<FileId>) -> ParseResult<CompilationUnit> {
let mut functions: Vec<Function> = vec![];
let mut extern_functions: Vec<ExternFunction> = vec![];
let mut classes: Vec<Class> = vec![];
while self.current.is_some() {
let current = self.current.as_ref().unwrap(); // loop checks for Some
match current.kind() {
TokenKind::Fn | TokenKind::Extern | TokenKind::Class => {
self.module_level_declaration(
&mut functions,
&mut extern_functions,
&mut classes,
);
}
_ => {
self.diagnostics.push(get_expected_but_found(
&[TokenKind::Fn, TokenKind::Extern, TokenKind::Class],
current,
));
}
}
}
ParseResult(
CompilationUnit::new(file_id, functions, extern_functions, classes),
std::mem::take(&mut self.diagnostics),
)
}
fn module_level_declaration(
&mut self,
functions: &mut Vec<Function>,
extern_functions: &mut Vec<ExternFunction>,
classes: &mut Vec<Class>,
) {
let current = match self.current.as_ref() {
None => {
self.diagnostics.push(get_expected_but_found_eoi(
&[TokenKind::Fn, TokenKind::Extern, TokenKind::Class],
self.input.len(),
));
return;
}
Some(current) => current,
};
match current.kind() {
TokenKind::Fn => {
if let Some(function) = self.function() {
functions.push(function);
}
}
TokenKind::Extern => {
if let Some(extern_function) = self.extern_function() {
extern_functions.push(extern_function);
}
}
TokenKind::Class => {
if let Some(class) = self.class() {
classes.push(class);
}
}
_ => unreachable!(),
}
}
fn function(&mut self) -> Option<Function> {
let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) {
self.advance(); // pub
true
} else {
false
};
self.expect_advance(TokenKind::Fn);
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
self.expect_advance(TokenKind::LeftParentheses);
let parameters = self.parameter_list();
self.expect_advance(TokenKind::RightParentheses);
let return_type = if self.current.is_some() && self.peek_current(TokenKind::RightArrow) {
self.return_type()
} else {
None
};
let mut statements = vec![];
while self.current.is_some() && !self.peek_current(TokenKind::End) {
if let Some(statement) = self.statement() {
statements.push(statement);
}
}
self.expect_advance(TokenKind::End);
if let Some(identifier) = maybe_identifier {
let function = Function::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
is_public,
parameters,
return_type,
statements,
);
Some(function)
} else {
None
}
}
fn extern_function(&mut self) -> Option<ExternFunction> {
self.expect_advance(TokenKind::Extern);
self.expect_advance(TokenKind::Fn);
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
self.expect_advance(TokenKind::LeftParentheses);
let parameters = self.parameter_list();
self.expect_advance(TokenKind::RightParentheses);
let maybe_return_type = self.return_type();
if let Some(identifier) = maybe_identifier
&& let Some(type_use) = maybe_return_type
{
let extern_function = ExternFunction::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
parameters,
type_use,
);
Some(extern_function)
} else {
None
}
}
fn class(&mut self) -> Option<Class> {
self.expect_advance(TokenKind::Class);
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
let generic_parameters = if self.current.is_some() && self.peek_current(TokenKind::Lt) {
self.generic_parameters()
} else {
Vec::new()
};
let mut fields = vec![];
let mut functions = vec![];
let mut constructor: Option<Constructor> = None;
while self.current.is_some() && !self.peek_current(TokenKind::End) {
match self.current.as_ref().unwrap().kind() {
// checked in loop
TokenKind::Public => {
self.public_class_member(&mut fields, &mut functions, &mut constructor);
}
TokenKind::Mut | TokenKind::Identifier => {
if let Some(field) = self.field() {
fields.push(field);
}
}
TokenKind::Fn => {
if let Some(function) = self.function() {
functions.push(function);
}
}
TokenKind::Ctor => {
constructor = self.constructor();
}
_ => unreachable!(),
}
}
self.expect_advance(TokenKind::End);
if let Some(identifier) = maybe_identifier {
let class = Class::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
generic_parameters,
constructor,
fields,
functions,
);
Some(class)
} else {
None
}
}
fn parameter_list(&mut self) -> Vec<Parameter> {
let mut parameters = vec![];
while self.current.is_some() && self.peek_current(TokenKind::Identifier) {
if let Some(parameter) = self.parameter() {
parameters.push(parameter);
}
if self.current.is_some() && self.peek_current(TokenKind::Comma) {
self.advance();
}
}
parameters
}
fn parameter(&mut self) -> Option<Parameter> {
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
self.expect_advance(TokenKind::Colon);
let maybe_type_use = self.type_use();
if let Some(identifier) = maybe_identifier
&& let Some(type_use) = maybe_type_use
{
let parameter = Parameter::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
type_use,
);
Some(parameter)
} else {
None
}
}
fn return_type(&mut self) -> Option<TypeUse> {
self.expect_advance(TokenKind::RightArrow);
self.type_use()
}
fn type_use(&mut self) -> Option<TypeUse> {
if self.current.is_some() {
let current = self.current.as_ref().unwrap(); // loop checks for Some
match current.kind() {
TokenKind::LeftSquare => {
todo!()
}
TokenKind::Identifier => {
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
let generic_arguments =
if self.current.is_some() && self.peek_current(TokenKind::Lt) {
self.advance(); // <
let generic_arguments = self.generic_arguments_list();
self.expect_advance(TokenKind::Gt); // >
generic_arguments
} else {
vec![]
};
if let Some(identifier) = maybe_identifier {
let type_use = TypeUse::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
generic_arguments,
);
Some(type_use)
} else {
None
}
}
_ => {
self.diagnostics.push(get_expected_but_found(
&[TokenKind::LeftSquare, TokenKind::Identifier],
current,
));
None
}
}
} else {
self.diagnostics.push(get_expected_but_found_eoi(
&[TokenKind::LeftSquare, TokenKind::Identifier],
self.input.len(),
));
None
}
}
fn generic_arguments_list(&mut self) -> Vec<TypeUse> {
let mut generic_arguments: Vec<TypeUse> = vec![];
while self.current.is_some()
&& matches_type_use_first!(self.current.as_ref().unwrap().kind())
{
if let Some(type_use) = self.type_use() {
generic_arguments.push(type_use);
}
if self.current.is_some() && self.peek_current(TokenKind::Comma) {
self.advance(); // comma
} else {
break;
}
}
generic_arguments
}
fn generic_parameters(&mut self) -> Vec<GenericParameter> {
self.expect_advance(TokenKind::Lt);
let mut parameters: Vec<GenericParameter> = vec![];
while self.current.is_some() && self.peek_current(TokenKind::Identifier) {
if let Some(generic_parameter) = self.generic_parameter() {
parameters.push(generic_parameter);
}
if self.current.is_some() && self.peek_current(TokenKind::Plus) {
self.advance(); // +
} else {
break;
}
}
self.expect_advance(TokenKind::Gt);
parameters
}
fn generic_parameter(&mut self) -> Option<GenericParameter> {
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
let mut extends_list: Vec<TypeUse> = vec![];
if self.current.is_some() && self.peek_current(TokenKind::Colon) {
self.advance(); // :
while self.current.is_some()
&& matches_type_use_first!(self.current.as_ref().unwrap().kind())
{
if let Some(type_use) = self.type_use() {
extends_list.push(type_use);
}
if self.current.is_some() && self.peek_current(TokenKind::Comma) {
self.advance(); // ,
} else {
break;
}
}
}
if let Some(identifier) = maybe_identifier {
let generic_parameter = GenericParameter::new(
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
extends_list,
);
Some(generic_parameter)
} else {
None
}
}
fn public_class_member(
&mut self,
fields: &mut Vec<Field>,
functions: &mut Vec<Function>,
maybe_constructor: &mut Option<Constructor>,
) {
if self.lookahead.is_some() {
match self.lookahead.as_ref().unwrap().kind() {
TokenKind::Mut | TokenKind::Identifier => {
if let Some(field) = self.field() {
fields.push(field);
}
}
TokenKind::Fn => {
if let Some(function) = self.function() {
functions.push(function);
}
}
TokenKind::Ctor => {
if let Some(constructor) = self.constructor() {
maybe_constructor.replace(constructor);
}
}
_ => {
let lookahead = self.lookahead.as_ref().unwrap();
self.diagnostics.push(get_expected_but_found(
&[TokenKind::Mut, TokenKind::Identifier, TokenKind::Fn],
lookahead,
));
}
}
} else {
let current = self.current.as_ref().unwrap();
self.diagnostics.push(get_expected_but_found(
&[TokenKind::Mut, TokenKind::Identifier, TokenKind::Fn],
current,
));
}
}
fn constructor(&mut self) -> Option<Constructor> {
let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) {
self.advance();
true
} else {
false
};
let maybe_ctor_kw = self.expect_advance(TokenKind::Ctor);
self.expect_advance(TokenKind::LeftParentheses);
let parameters = if self.current.is_some() && self.peek_current(TokenKind::Identifier) {
self.parameter_list()
} else {
vec![]
};
self.expect_advance(TokenKind::RightParentheses);
// statements
let mut statements: Vec<Statement> = vec![];
while self.current.is_some()
&& matches_statement_first!(self.current.as_ref().unwrap().kind())
{
if let Some(statement) = self.statement() {
statements.push(statement);
}
}
self.expect_advance(TokenKind::End);
if let Some(ctor_keyword) = maybe_ctor_kw {
let constructor = Constructor::new(
self.next_node_id(),
is_public,
SourceRange::new(ctor_keyword.start(), ctor_keyword.end()),
parameters,
statements,
);
Some(constructor)
} else {
None
}
}
fn field(&mut self) -> Option<Field> {
let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) {
self.advance();
true
} else {
false
};
let is_mut = if self.current.is_some() && self.peek_current(TokenKind::Mut) {
self.advance();
true
} else {
false
};
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
let maybe_declared_type = if self.current.is_some() && self.peek_current(TokenKind::Colon) {
self.advance(); // colon
self.type_use()
} else {
None
};
let maybe_initializer = if self.current.is_some() && self.peek_current(TokenKind::Equals) {
self.advance(); // equals
self.expression()
} else {
None
};
if let Some(identifier) = maybe_identifier {
Some(Field::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
is_public,
is_mut,
maybe_declared_type,
maybe_initializer,
))
} else {
None
}
}
fn statement(&mut self) -> Option<Statement> {
let current = match self.current.as_ref() {
None => {
self.diagnostics.push(get_expected_but_found_eoi(
&STATEMENT_FIRSTS,
self.input.len(),
));
return None;
}
Some(current) => current,
};
match current.kind() {
TokenKind::Let => self
.let_statement()
.map(|let_statement| Statement::Let(let_statement)),
_ => self.expression_statement_or_assign_statement(),
}
}
fn let_statement(&mut self) -> Option<LetStatement> {
self.expect_advance(TokenKind::Let);
let is_mut = if self.current.is_some() && self.peek_current(TokenKind::Mut) {
self.advance();
true
} else {
false
};
let maybe_identifier = self.expect_advance(TokenKind::Identifier);
self.expect_advance(TokenKind::Equals);
let maybe_expression = self.expression();
let identifier = match maybe_identifier {
None => {
return None;
}
Some(identifier) => identifier,
};
maybe_expression.map(|expression| {
LetStatement::new(
self.next_node_id(),
self.token_text(&identifier),
SourceRange::new(identifier.start(), identifier.end()),
is_mut,
expression,
)
})
}
fn expression_statement_or_assign_statement(&mut self) -> Option<Statement> {
let expression = self.expression();
if self.current.is_some() && self.peek_current(TokenKind::Equals) {
expression.and_then(|expression| {
self.assign_rhs(expression)
.map(|assign_statement| Statement::Assign(assign_statement))
})
} else {
expression.map(|expression| Statement::Expression(ExpressionStatement::new(expression)))
}
}
fn assign_rhs(&mut self, destination: Expression) -> Option<AssignStatement> {
self.expect_advance(TokenKind::Equals);
let maybe_expression = self.expression();
maybe_expression.map(|expression| AssignStatement::new(destination, expression))
}
fn expression(&mut self) -> Option<Expression> {
if self.current.is_some() {
if matches_expression_first!(self.current.as_ref().unwrap().kind()) {
self.bitwise_or_expression()
} else {
self.diagnostics.push(get_expected_but_found(
&EXPRESSION_FIRSTS,
self.current.as_ref().unwrap(),
));
None
}
} else {
self.diagnostics
.push(get_expected_but_found_eoi(&EXPRESSION_FIRSTS, 0));
None
}
}
fn zip_binary_expression(
&mut self,
left: Option<Expression>,
right: Option<Expression>,
operation: BinaryOperation,
) -> Option<Expression> {
left.zip_with(right, |lhs, rhs| {
let source_range =
SourceRange::new(lhs.source_range().start(), rhs.source_range().end());
Expression::Binary(BinaryExpression::new(
self.next_node_id(),
lhs,
rhs,
operation,
source_range,
))
})
}
fn bitwise_or_expression(&mut self) -> Option<Expression> {
let mut base = self.bitwise_xor_expression();
while self.current.is_some() && self.peek_current(TokenKind::Bar) {
self.advance(); // |
let rhs = self.bitwise_xor_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::BitwiseOr);
}
base
}
fn bitwise_xor_expression(&mut self) -> Option<Expression> {
let mut base = self.bitwise_and_expression();
while self.current.is_some() && self.peek_current(TokenKind::Caret) {
self.advance(); // ^
let rhs = self.bitwise_and_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::BitwiseXor);
}
base
}
fn bitwise_and_expression(&mut self) -> Option<Expression> {
let mut base = self.shift_expression();
while self.current.is_some() && self.peek_current(TokenKind::Ampersand) {
self.advance(); // &
let rhs = self.shift_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::BitwiseAnd);
}
base
}
fn shift_expression(&mut self) -> Option<Expression> {
let mut base = self.additive_expression();
while self.current.is_some() {
let current = self.current.as_ref().unwrap();
match current.kind() {
TokenKind::Lt => {
let previous_cloned = current.clone();
self.advance(); // first <
self.expect_immediately_after_advance(TokenKind::Lt, &previous_cloned); // second <
let rhs = self.additive_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::LeftShift);
}
TokenKind::Gt => {
let previous_cloned = current.clone();
self.advance(); // first >
self.expect_immediately_after_advance(TokenKind::Gt, &previous_cloned); // second >
let rhs = self.additive_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::RightShift);
}
_ => break,
}
}
base
}
fn additive_expression(&mut self) -> Option<Expression> {
let mut base = self.multiplicative_expression();
while self.current.is_some() {
let current = self.current.as_ref().unwrap();
match current.kind() {
TokenKind::Plus => {
self.advance(); // plus
let rhs = self.multiplicative_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::Add);
}
TokenKind::Minus => {
self.advance(); // minus
let rhs = self.multiplicative_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::Subtract);
}
_ => break,
}
}
base
}
fn multiplicative_expression(&mut self) -> Option<Expression> {
let mut base = self.prefix_expression();
while self.current.is_some() {
let current = self.current.as_ref().unwrap();
match current.kind() {
TokenKind::Star => {
self.advance(); // multiply
let rhs = self.prefix_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::Multiply);
}
TokenKind::Slash => {
self.advance(); // slash
let rhs = self.prefix_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::Divide);
}
TokenKind::Modulo => {
self.advance(); // modulo
let rhs = self.prefix_expression();
base = self.zip_binary_expression(base, rhs, BinaryOperation::Modulo);
}
_ => break,
}
}
base
}
fn prefix_expression(&mut self) -> Option<Expression> {
// first, collect all consecutive operators
let mut operator_tokens = vec![];
while self.current.is_some() {
let current = self.current.as_ref().unwrap();
match current.kind() {
TokenKind::Minus => {
operator_tokens.push(current.clone()); // unfortunately necessary
self.advance();
}
_ => break,
}
}
// now go in reverse and build up expressions
// the parser is currently just after the prefix operators, so we need a suffix expression
// as a base
let mut base = self.suffix_expression();
while let Some(operator_token) = operator_tokens.pop() {
match operator_token.kind() {
TokenKind::Minus => {
base = base.map(|expression| {
let source_range = SourceRange::new(
operator_token.start(),
expression.source_range().end(),
);
Expression::Negative(NegativeExpression::new(
self.next_node_id(),
expression,
source_range,
))
});
}
_ => unreachable!(),
}
}
base
}
fn suffix_expression(&mut self) -> Option<Expression> {
if let Some(mut expression) = self.expression_base() {
while self.current.is_some() {
let current = self.current.as_ref().unwrap();
match current.kind() {
TokenKind::LeftParentheses => {
expression = Expression::Call(self.call(expression));
}
TokenKind::Dot => {
self.advance(); // .
if let Some(identifier) = self.identifier() {
let source_range = SourceRange::new(
expression.source_range().start(),
identifier.source_range().end(),
);
expression = Expression::Path(Path::new(
self.next_node_id(),
source_range,
expression,
identifier,
));
}
}
_ => break,
}
}
Some(expression)
} else {
None
}
}
fn identifier(&mut self) -> Option<Identifier> {
if let Some(identifier_token) = self.expect_advance(TokenKind::Identifier) {
Some(Identifier::new(
self.next_node_id(),
self.token_text(&identifier_token),
SourceRange::new(identifier_token.start(), identifier_token.end()),
))
} else {
None
}
}
fn expression_base(&mut self) -> Option<Expression> {
let current = match self.current.as_ref() {
Some(current) => current,
None => {
self.diagnostics.push(get_expected_but_found_eoi(
&EXPRESSION_FIRSTS,
self.input.len(),
));
return None;
}
};
match current.kind() {
TokenKind::IntegerLiteral => {
let raw = self.token_text(&current);
let source_range = SourceRange::new(current.start(), current.end());
self.advance();
Some(Expression::Integer(IntegerLiteral::new(
self.next_node_id(),
i32::from_str(raw).unwrap(),
source_range,
)))
}
TokenKind::DoubleLiteral => {
let raw = self.token_text(&current);
let source_range = SourceRange::new(current.start(), current.end());
self.advance();
Some(Expression::Double(DoubleLiteral::new(
self.next_node_id(),
f64::from_str(raw).unwrap(),
source_range,
)))
}
TokenKind::String => {
let with_quotes = self.token_text(&current);
let source_range = SourceRange::new(current.start(), current.end());
self.advance();
Some(Expression::String(StringLiteral::new(
self.next_node_id(),
&with_quotes[1..with_quotes.len() - 1],
source_range,
)))
}
TokenKind::Identifier => {
let declared_name = self.token_text(&current);
let source_range = SourceRange::new(current.start(), current.end());
self.advance();
Some(Expression::Identifier(Identifier::new(
self.next_node_id(),
declared_name,
source_range,
)))
}
_ => unreachable!("Unreachable token type found: {:?}", current.kind()),
}
}
fn call(&mut self, callee: Expression) -> Call {
self.expect_advance(TokenKind::LeftParentheses);
let mut arguments = vec![];
if let Some(current) = &self.current {
if matches_expression_first!(current.kind()) {
arguments = self.expression_list();
}
}
let maybe_right_parenthesis = self.expect_advance(TokenKind::RightParentheses);
let source_range = if let Some(token) = maybe_right_parenthesis {
SourceRange::new(callee.source_range().start(), token.end())
} else {
// This should be good enough for error reporting if we're missing the right parenthesis
SourceRange::new(callee.source_range().start(), callee.source_range().end())
};
Call::new(self.next_node_id(), callee, arguments, source_range)
}
fn expression_list(&mut self) -> Vec<Expression> {
let mut expressions = vec![];
if let Some(expression) = self.expression() {
expressions.push(expression);
}
while self.current.is_some() && self.peek_current(TokenKind::Comma) {
self.advance();
if let Some(expression) = self.expression() {
expressions.push(expression);
}
}
expressions
}
}
#[cfg(test)]
mod smoke_tests {
use super::*;
fn smoke_test(input: &str) {
let parse_result = parse_compilation_unit(input, None);
if !parse_result.is_ok() {
eprintln!("{:#?}", parse_result.1);
panic!("There were diagnostics during parsing");
}
}
#[test]
fn forty_two() {
smoke_test("fn main() 42 end");
}
#[test]
fn chained_calls() {
smoke_test("fn main() getCl()() end");
}
#[test]
fn extern_fn_with_param() {
smoke_test("extern fn println(message: Any) -> Void");
}
#[test]
fn fn_with_param() {
smoke_test("fn foo(bar: Int) end");
}
#[test]
fn fn_with_params() {
smoke_test("fn foo(bar: Int, baz: Int) end");
}
#[test]
fn return_type() {
smoke_test("fn foo() -> Int end")
}
#[test]
fn extern_return_type() {
smoke_test("extern fn foo() -> Int");
}
#[test]
fn add_two_numbers() {
smoke_test("fn main() 1 + 2 end");
}
#[test]
fn negative_return() {
smoke_test("fn main() -> Int -1 end");
}
#[test]
fn negative_left_add() {
smoke_test("fn main() -> Int -1 + 1 end");
}
#[test]
fn negative_right_add() {
smoke_test("fn main() -> Int 1 + -1 end");
}
#[test]
fn two_negatives() {
smoke_test("fn main() -> Int -1 + -1 end");
}
#[test]
fn minus_positive_number() {
smoke_test("fn main() -> Int 1 - 1 end");
}
#[test]
fn minus_negative_number() {
smoke_test("fn main() -> Int 1 - -1 end");
}
#[test]
fn empty_class() {
smoke_test("class Foo end");
}
#[test]
fn class_with_pub_member() {
smoke_test("class Foo pub bar end");
}
#[test]
fn class_with_mut_member() {
smoke_test("class Foo mut bar end");
}
#[test]
fn class_with_nothing_member() {
smoke_test("class Foo bar end");
}
#[test]
fn class_with_member_type_use() {
smoke_test("class Foo bar: Int end");
}
#[test]
fn class_with_member_init() {
smoke_test("class Foo bar = 42 end");
}
#[test]
fn class_with_member_type_use_and_init() {
smoke_test("class Foo bar: Int = 42 end");
}
#[test]
fn class_with_member_all() {
smoke_test("class Foo pub mut bar: Bar = Baz() end");
}
#[test]
fn class_with_pub_fn() {
smoke_test(
"
class Greeter
pub fn greet() end
end
",
);
}
#[test]
fn simple_assign() {
smoke_test(
"
fn main()
let mut x = 4
x = 42
end
",
);
}
#[test]
fn simple_multiply() {
smoke_test("fn main() 1 * 2 end");
}
#[test]
fn simple_divide() {
smoke_test("fn main() 1 / 2 end");
}
#[test]
fn simple_modulo() {
smoke_test("fn main() 1 % 2 end");
}
#[test]
fn simple_left_shift() {
smoke_test("fn main() 2 << 1 end");
}
#[test]
fn simple_right_shift() {
smoke_test("fn main() 4 >> 1 end");
}
#[test]
fn simple_bitwise_and() {
smoke_test("fn main() 2 & 1 end");
}
#[test]
fn simple_bitwise_xor() {
smoke_test("fn main() 1 ^ 2 end");
}
#[test]
fn simple_bitwise_or() {
smoke_test("fn main() 1 | 2 end");
}
#[test]
fn ops_left_to_right() {
smoke_test(
"
fn main()
1 | 2 ^ 3 & 4 << 5 >> 7 + 8 - 9 * 10 / 11 % 12
end
",
)
}
#[test]
fn array_generic_arg() {
smoke_test("fn main(args: Array<String>) end");
}
#[test]
fn nested_generic_args() {
smoke_test("fn main(foo: Array<Bar<Foo>>) end");
}
#[test]
fn class_with_generic_param() {
smoke_test("class Foo<T> end");
}
#[test]
fn single_member_access() {
smoke_test("fn main(foo: Foo) foo.bar end")
}
#[test]
fn double_member_access() {
smoke_test("fn main(foo: Foo) foo.bar.baz end")
}
#[test]
fn call_member() {
smoke_test("fn main(s: String) s.len() end")
}
}
#[cfg(test)]
mod concrete_tests {
use super::*;
fn report_diagnostics(diagnostics: &[Diagnostic]) -> ! {
for diagnostic in diagnostics {
eprintln!("{:?}", diagnostic);
}
panic!();
}
fn assert_compilation_unit(input: &str) -> CompilationUnit {
let parse_result = parse_compilation_unit(input, None);
if !parse_result.is_ok() {
report_diagnostics(&parse_result.1);
}
parse_result.0
}
fn assert_expression(input: &str) -> Expression {
let parse_result = parse_expression(input);
if parse_result.is_ok() {
parse_result.0.unwrap()
} else {
report_diagnostics(&parse_result.1);
}
}
fn assert_function_in<'a>(
compilation_unit: &'a CompilationUnit,
function_name: &str,
) -> &'a Function {
compilation_unit
.functions()
.iter()
.find(|f| f.declared_name() == function_name)
.unwrap()
}
#[test]
fn parses_extern_fn() {
let compilation_unit = assert_compilation_unit("extern fn println() -> Void");
let extern_functions = compilation_unit.extern_functions();
assert_eq!(extern_functions.len(), 1);
let extern_function = &extern_functions[0];
assert_eq!(extern_function.declared_name(), "println");
}
#[test]
fn hello_world() {
let compilation_unit = assert_compilation_unit("fn main() println(\"Hello, World!\") end");
let function = assert_function_in(&compilation_unit, "main");
let statements = function.statements();
assert_eq!(statements.len(), 1);
if let Statement::Expression(expression_statement) = statements[0] {
if let Expression::Call(call) = expression_statement.expression() {
let callee = call.callee();
match callee {
Expression::Identifier(identifier) => {
assert_eq!(identifier.name(), "println");
}
_ => panic!("Expected identifier"),
}
let arguments = call.arguments();
assert_eq!(arguments.len(), 1);
let first_argument = &arguments[0];
match first_argument {
Expression::String(s) => {
assert_eq!(s.content(), "Hello, World!");
}
_ => panic!("Expected string"),
}
} else {
panic!("Expected call");
}
} else {
panic!("Expected expression");
}
}
#[test]
fn negative_expression() {
let expression = assert_expression("-1");
match expression {
Expression::Negative(negative_expression) => match negative_expression.operand() {
Expression::Integer(integer_literal) => {
assert_eq!(integer_literal.value(), 1);
}
_ => panic!("Expected integer literal"),
},
_ => panic!("Expected negative expression"),
}
}
#[test]
fn add_negative() {
let expression = assert_expression("1 + -1");
match expression {
Expression::Binary(binary_expression) => {
assert!(matches!(binary_expression.op(), BinaryOperation::Add));
match binary_expression.lhs() {
Expression::Integer(integer_literal) => {
assert_eq!(integer_literal.value(), 1);
}
_ => panic!("Expected integer literal"),
}
match binary_expression.rhs() {
Expression::Negative(negative_expression) => {
match negative_expression.operand() {
Expression::Integer(integer_literal) => {
assert_eq!(integer_literal.value(), 1);
}
_ => panic!("Expected integer literal"),
}
}
_ => panic!("Expected negative expression"),
}
}
_ => panic!("Expected additive expression"),
}
}
#[test]
fn simple_subtract() {
let expression = assert_expression("1 - 1");
match expression {
Expression::Binary(binary_expression) => {
assert!(matches!(binary_expression.op(), BinaryOperation::Subtract));
match binary_expression.lhs() {
Expression::Integer(integer_literal) => {
assert_eq!(integer_literal.value(), 1);
}
_ => panic!("Expected integer literal"),
}
match binary_expression.rhs() {
Expression::Integer(integer_literal) => {
assert_eq!(integer_literal.value(), 1);
}
_ => panic!("Expected integer literal"),
}
}
_ => panic!("Expected subtract expression"),
}
}
}