use crate::FileId; use crate::ast::NodeId; use crate::ast::assign_statement::AssignStatement; use crate::ast::binary_expression::{BinaryExpression, BinaryOperation}; use crate::ast::call::Call; use crate::ast::class::Class; use crate::ast::compilation_unit::CompilationUnit; use crate::ast::constructor::Constructor; use crate::ast::double_literal::DoubleLiteral; use crate::ast::expression::Expression; use crate::ast::expression_statement::ExpressionStatement; use crate::ast::extern_function::ExternFunction; use crate::ast::field::Field; use crate::ast::function::Function; use crate::ast::generic_parameter::GenericParameter; use crate::ast::identifier::Identifier; use crate::ast::integer_literal::IntegerLiteral; use crate::ast::let_statement::LetStatement; use crate::ast::negative_expression::NegativeExpression; use crate::ast::parameter::Parameter; use crate::ast::path::Path; use crate::ast::statement::Statement; use crate::ast::string_literal::StringLiteral; use crate::ast::type_use::TypeUse; use crate::diagnostic::{Diagnostic, Diagnostics}; use crate::error_codes::{LEXER_ERROR, PARSE_ERROR}; use crate::lexer::{Lexer, LexerErrorKind}; use crate::source_range::SourceRange; use crate::token::{Token, TokenKind}; use std::str::FromStr; pub struct ParseResult(pub T, pub Diagnostics); impl ParseResult { pub fn is_ok(&self) -> bool { self.1.is_empty() } pub fn to_result(self) -> Result { if self.is_ok() { Ok(self.0) } else { Err(self.1) } } } pub fn get_compilation_unit( input: &str, file_id: Option, ) -> Result { parse_compilation_unit(input, file_id).to_result() } pub fn parse_compilation_unit( input: &str, file_id: Option, ) -> ParseResult { let mut parser = Parser::new(input); parser.advance(); parser.compilation_unit(file_id) } pub fn parse_statement(input: &str) -> ParseResult> { let mut parser = Parser::new(input); parser.advance(); ParseResult(parser.statement(), parser.diagnostics) } pub fn parse_expression(input: &str) -> ParseResult> { let mut parser = Parser::new(input); parser.advance(); ParseResult(parser.expression(), parser.diagnostics) } macro_rules! matches_expression_first { ( $token_kind : expr ) => { matches!( $token_kind, TokenKind::IntegerLiteral | TokenKind::DoubleLiteral | TokenKind::LongLiteral | TokenKind::String | TokenKind::Minus | TokenKind::SelfKw | TokenKind::Identifier ) }; } const EXPRESSION_FIRSTS: [TokenKind; 7] = [ TokenKind::IntegerLiteral, TokenKind::DoubleLiteral, TokenKind::LongLiteral, TokenKind::String, TokenKind::Minus, TokenKind::SelfKw, TokenKind::Identifier, ]; macro_rules! matches_statement_first { ( $token_kind : expr ) => { matches!($token_kind, TokenKind::Let) || matches_expression_first!($token_kind) }; } const STATEMENT_FIRSTS: [TokenKind; 8] = { let mut firsts: [TokenKind; 8] = [TokenKind::Let; 8]; let mut i = 0; while i < EXPRESSION_FIRSTS.len() { firsts[i + 1] = EXPRESSION_FIRSTS[i]; i += 1; } firsts }; macro_rules! matches_type_use_first { ( $token_kind: expr ) => { matches!($token_kind, TokenKind::LeftSquare | TokenKind::Identifier) }; } fn join_kinds(kinds: &[TokenKind]) -> String { kinds .iter() .map(|kind| format!("{:?}", kind)) .collect::>() .join(", ") } fn get_expected_but_found(kinds: &[TokenKind], found: &Token) -> Diagnostic { Diagnostic::new( &format!("Unexpected token: {:?}.", found.kind()), found.start(), found.end(), ) .with_error_code(PARSE_ERROR) .with_primary_label_message(&format!("Expected {}.", join_kinds(kinds))) } fn get_expected_but_found_eoi(kinds: &[TokenKind], position: usize) -> Diagnostic { Diagnostic::new("Unexpected end-of-input.", position, position) .with_error_code(PARSE_ERROR) .with_primary_label_message(&format!("Expected {}.", join_kinds(kinds))) } struct Parser<'a> { input: &'a str, lexer: Lexer<'a>, current: Option, lookahead: Option, current_node_id: NodeId, diagnostics: Diagnostics, } impl<'a> Parser<'a> { fn new(input: &'a str) -> Self { Self { input, lexer: Lexer::new(input), current: None, lookahead: None, current_node_id: 0, diagnostics: Diagnostics::new(), } } fn advance(&mut self) { fn fetch(lexer: &mut Lexer, diagnostics: &mut Diagnostics) -> Option { let mut maybe_token: Option = None; while let Some(lexer_result) = lexer.next() { match lexer_result { Ok(token) => { maybe_token = Some(token); break; } Err(lexer_error) => { let diagnostic = match lexer_error.kind() { LexerErrorKind::UnterminatedString => Diagnostic::new( "Unterminated string literal.", lexer_error.start(), lexer_error.end(), ) .with_error_code(LEXER_ERROR), LexerErrorKind::UnrecognizedCharacter(c) => Diagnostic::new( &format!("Unrecognized character: {}", c), lexer_error.start(), lexer_error.end(), ) .with_error_code(LEXER_ERROR), }; diagnostics.push(diagnostic); } } } maybe_token } if self.lookahead.is_some() { // we've advanced at least once self.current = self.lookahead.take(); self.lookahead = fetch(&mut self.lexer, &mut self.diagnostics); } else if self.lookahead.is_none() && self.current.is_some() { // we're on the last token self.current = None; } else { // we've not yet advanced, so fetch both self.current = fetch(&mut self.lexer, &mut self.diagnostics); self.lookahead = fetch(&mut self.lexer, &mut self.diagnostics); } } fn expect_advance(&mut self, token_kind: TokenKind) -> Option { match self.current.take() { None => { self.diagnostics .push(get_expected_but_found_eoi(&[token_kind], self.input.len())); None } Some(token) => { if token.kind() == token_kind { self.advance(); Some(token) } else { self.diagnostics .push(get_expected_but_found(&[token_kind], &token)); None } } } } fn expect_immediately_after_advance( &mut self, token_kind: TokenKind, previous_token: &Token, ) -> Option { let maybe_matched = self.expect_advance(token_kind); match maybe_matched { None => None, Some(matched) => { if matched.start() == previous_token.end() { Some(matched) } else { self.diagnostics.push( Diagnostic::new( &format!( "Expected {:?} immediately after previous token.", token_kind ), matched.start(), matched.end(), ) .with_error_code(PARSE_ERROR), ); None } } } } fn peek_current(&self, token_kind: TokenKind) -> bool { match &self.current { None => panic!("BUG: peeked when current is None"), Some(token) => token.kind() == token_kind, } } fn sample_input(&self, start: usize, end: usize) -> &'a str { &self.input[start..end] } fn token_text(&self, token: &Token) -> &'a str { self.sample_input(token.start(), token.end()) } fn next_node_id(&mut self) -> NodeId { let node_id = self.current_node_id; self.current_node_id += 1; node_id } fn compilation_unit(&mut self, file_id: Option) -> ParseResult { let mut functions: Vec = vec![]; let mut extern_functions: Vec = vec![]; let mut classes: Vec = vec![]; while self.current.is_some() { let current = self.current.as_ref().unwrap(); // loop checks for Some match current.kind() { TokenKind::Fn | TokenKind::Extern | TokenKind::Class => { self.module_level_declaration( &mut functions, &mut extern_functions, &mut classes, ); } _ => { self.diagnostics.push(get_expected_but_found( &[TokenKind::Fn, TokenKind::Extern, TokenKind::Class], current, )); } } } ParseResult( CompilationUnit::new(file_id, functions, extern_functions, classes), std::mem::take(&mut self.diagnostics), ) } fn module_level_declaration( &mut self, functions: &mut Vec, extern_functions: &mut Vec, classes: &mut Vec, ) { let current = match self.current.as_ref() { None => { self.diagnostics.push(get_expected_but_found_eoi( &[TokenKind::Fn, TokenKind::Extern, TokenKind::Class], self.input.len(), )); return; } Some(current) => current, }; match current.kind() { TokenKind::Fn => { if let Some(function) = self.function() { functions.push(function); } } TokenKind::Extern => { if let Some(extern_function) = self.extern_function() { extern_functions.push(extern_function); } } TokenKind::Class => { if let Some(class) = self.class() { classes.push(class); } } _ => unreachable!(), } } fn function(&mut self) -> Option { let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) { self.advance(); // pub true } else { false }; self.expect_advance(TokenKind::Fn); let maybe_identifier = self.expect_advance(TokenKind::Identifier); self.expect_advance(TokenKind::LeftParentheses); let parameters = self.parameter_list(); self.expect_advance(TokenKind::RightParentheses); let return_type = if self.current.is_some() && self.peek_current(TokenKind::RightArrow) { self.return_type() } else { None }; let mut statements = vec![]; while self.current.is_some() && !self.peek_current(TokenKind::End) { if let Some(statement) = self.statement() { statements.push(statement); } } self.expect_advance(TokenKind::End); if let Some(identifier) = maybe_identifier { let function = Function::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), is_public, parameters, return_type, statements, ); Some(function) } else { None } } fn extern_function(&mut self) -> Option { self.expect_advance(TokenKind::Extern); self.expect_advance(TokenKind::Fn); let maybe_identifier = self.expect_advance(TokenKind::Identifier); self.expect_advance(TokenKind::LeftParentheses); let parameters = self.parameter_list(); self.expect_advance(TokenKind::RightParentheses); let maybe_return_type = self.return_type(); if let Some(identifier) = maybe_identifier && let Some(type_use) = maybe_return_type { let extern_function = ExternFunction::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), parameters, type_use, ); Some(extern_function) } else { None } } fn class(&mut self) -> Option { self.expect_advance(TokenKind::Class); let maybe_identifier = self.expect_advance(TokenKind::Identifier); let generic_parameters = if self.current.is_some() && self.peek_current(TokenKind::Lt) { self.generic_parameters() } else { Vec::new() }; let mut fields = vec![]; let mut functions = vec![]; let mut constructor: Option = None; while self.current.is_some() && !self.peek_current(TokenKind::End) { match self.current.as_ref().unwrap().kind() { // checked in loop TokenKind::Public => { self.public_class_member(&mut fields, &mut functions, &mut constructor); } TokenKind::Mut | TokenKind::Identifier => { if let Some(field) = self.field() { fields.push(field); } } TokenKind::Fn => { if let Some(function) = self.function() { functions.push(function); } } TokenKind::Ctor => { constructor = self.constructor(); } _ => unreachable!(), } } self.expect_advance(TokenKind::End); if let Some(identifier) = maybe_identifier { let class = Class::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), generic_parameters, constructor, fields, functions, ); Some(class) } else { None } } fn parameter_list(&mut self) -> Vec { let mut parameters = vec![]; while self.current.is_some() && self.peek_current(TokenKind::Identifier) { if let Some(parameter) = self.parameter() { parameters.push(parameter); } if self.current.is_some() && self.peek_current(TokenKind::Comma) { self.advance(); } } parameters } fn parameter(&mut self) -> Option { let maybe_identifier = self.expect_advance(TokenKind::Identifier); self.expect_advance(TokenKind::Colon); let maybe_type_use = self.type_use(); if let Some(identifier) = maybe_identifier && let Some(type_use) = maybe_type_use { let parameter = Parameter::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), type_use, ); Some(parameter) } else { None } } fn return_type(&mut self) -> Option { self.expect_advance(TokenKind::RightArrow); self.type_use() } fn type_use(&mut self) -> Option { if self.current.is_some() { let current = self.current.as_ref().unwrap(); // loop checks for Some match current.kind() { TokenKind::LeftSquare => { todo!() } TokenKind::Identifier => { let maybe_identifier = self.expect_advance(TokenKind::Identifier); let generic_arguments = if self.current.is_some() && self.peek_current(TokenKind::Lt) { self.advance(); // < let generic_arguments = self.generic_arguments_list(); self.expect_advance(TokenKind::Gt); // > generic_arguments } else { vec![] }; if let Some(identifier) = maybe_identifier { let type_use = TypeUse::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), generic_arguments, ); Some(type_use) } else { None } } _ => { self.diagnostics.push(get_expected_but_found( &[TokenKind::LeftSquare, TokenKind::Identifier], current, )); None } } } else { self.diagnostics.push(get_expected_but_found_eoi( &[TokenKind::LeftSquare, TokenKind::Identifier], self.input.len(), )); None } } fn generic_arguments_list(&mut self) -> Vec { let mut generic_arguments: Vec = vec![]; while self.current.is_some() && matches_type_use_first!(self.current.as_ref().unwrap().kind()) { if let Some(type_use) = self.type_use() { generic_arguments.push(type_use); } if self.current.is_some() && self.peek_current(TokenKind::Comma) { self.advance(); // comma } else { break; } } generic_arguments } fn generic_parameters(&mut self) -> Vec { self.expect_advance(TokenKind::Lt); let mut parameters: Vec = vec![]; while self.current.is_some() && self.peek_current(TokenKind::Identifier) { if let Some(generic_parameter) = self.generic_parameter() { parameters.push(generic_parameter); } if self.current.is_some() && self.peek_current(TokenKind::Plus) { self.advance(); // + } else { break; } } self.expect_advance(TokenKind::Gt); parameters } fn generic_parameter(&mut self) -> Option { let maybe_identifier = self.expect_advance(TokenKind::Identifier); let mut extends_list: Vec = vec![]; if self.current.is_some() && self.peek_current(TokenKind::Colon) { self.advance(); // : while self.current.is_some() && matches_type_use_first!(self.current.as_ref().unwrap().kind()) { if let Some(type_use) = self.type_use() { extends_list.push(type_use); } if self.current.is_some() && self.peek_current(TokenKind::Comma) { self.advance(); // , } else { break; } } } if let Some(identifier) = maybe_identifier { let generic_parameter = GenericParameter::new( self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), extends_list, ); Some(generic_parameter) } else { None } } fn public_class_member( &mut self, fields: &mut Vec, functions: &mut Vec, maybe_constructor: &mut Option, ) { if self.lookahead.is_some() { match self.lookahead.as_ref().unwrap().kind() { TokenKind::Mut | TokenKind::Identifier => { if let Some(field) = self.field() { fields.push(field); } } TokenKind::Fn => { if let Some(function) = self.function() { functions.push(function); } } TokenKind::Ctor => { if let Some(constructor) = self.constructor() { maybe_constructor.replace(constructor); } } _ => { let lookahead = self.lookahead.as_ref().unwrap(); self.diagnostics.push(get_expected_but_found( &[TokenKind::Mut, TokenKind::Identifier, TokenKind::Fn], lookahead, )); } } } else { let current = self.current.as_ref().unwrap(); self.diagnostics.push(get_expected_but_found( &[TokenKind::Mut, TokenKind::Identifier, TokenKind::Fn], current, )); } } fn constructor(&mut self) -> Option { let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) { self.advance(); true } else { false }; let maybe_ctor_kw = self.expect_advance(TokenKind::Ctor); self.expect_advance(TokenKind::LeftParentheses); let parameters = if self.current.is_some() && self.peek_current(TokenKind::Identifier) { self.parameter_list() } else { vec![] }; self.expect_advance(TokenKind::RightParentheses); // statements let mut statements: Vec = vec![]; while self.current.is_some() && matches_statement_first!(self.current.as_ref().unwrap().kind()) { if let Some(statement) = self.statement() { statements.push(statement); } } self.expect_advance(TokenKind::End); if let Some(ctor_keyword) = maybe_ctor_kw { let constructor = Constructor::new( self.next_node_id(), is_public, SourceRange::new(ctor_keyword.start(), ctor_keyword.end()), parameters, statements, ); Some(constructor) } else { None } } fn field(&mut self) -> Option { let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) { self.advance(); true } else { false }; let is_mut = if self.current.is_some() && self.peek_current(TokenKind::Mut) { self.advance(); true } else { false }; let maybe_identifier = self.expect_advance(TokenKind::Identifier); let maybe_declared_type = if self.current.is_some() && self.peek_current(TokenKind::Colon) { self.advance(); // colon self.type_use() } else { None }; let maybe_initializer = if self.current.is_some() && self.peek_current(TokenKind::Equals) { self.advance(); // equals self.expression() } else { None }; if let Some(identifier) = maybe_identifier { Some(Field::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), is_public, is_mut, maybe_declared_type, maybe_initializer, )) } else { None } } fn statement(&mut self) -> Option { let current = match self.current.as_ref() { None => { self.diagnostics.push(get_expected_but_found_eoi( &STATEMENT_FIRSTS, self.input.len(), )); return None; } Some(current) => current, }; match current.kind() { TokenKind::Let => self .let_statement() .map(|let_statement| Statement::Let(let_statement)), _ => self.expression_statement_or_assign_statement(), } } fn let_statement(&mut self) -> Option { self.expect_advance(TokenKind::Let); let is_mut = if self.current.is_some() && self.peek_current(TokenKind::Mut) { self.advance(); true } else { false }; let maybe_identifier = self.expect_advance(TokenKind::Identifier); self.expect_advance(TokenKind::Equals); let maybe_expression = self.expression(); let identifier = match maybe_identifier { None => { return None; } Some(identifier) => identifier, }; maybe_expression.map(|expression| { LetStatement::new( self.next_node_id(), self.token_text(&identifier), SourceRange::new(identifier.start(), identifier.end()), is_mut, expression, ) }) } fn expression_statement_or_assign_statement(&mut self) -> Option { let expression = self.expression(); if self.current.is_some() && self.peek_current(TokenKind::Equals) { expression.and_then(|expression| { self.assign_rhs(expression) .map(|assign_statement| Statement::Assign(assign_statement)) }) } else { expression.map(|expression| Statement::Expression(ExpressionStatement::new(expression))) } } fn assign_rhs(&mut self, destination: Expression) -> Option { self.expect_advance(TokenKind::Equals); let maybe_expression = self.expression(); maybe_expression.map(|expression| AssignStatement::new(destination, expression)) } fn expression(&mut self) -> Option { if self.current.is_some() { if matches_expression_first!(self.current.as_ref().unwrap().kind()) { self.bitwise_or_expression() } else { self.diagnostics.push(get_expected_but_found( &EXPRESSION_FIRSTS, self.current.as_ref().unwrap(), )); None } } else { self.diagnostics .push(get_expected_but_found_eoi(&EXPRESSION_FIRSTS, 0)); None } } fn zip_binary_expression( &mut self, left: Option, right: Option, operation: BinaryOperation, ) -> Option { left.zip_with(right, |lhs, rhs| { let source_range = SourceRange::new(lhs.source_range().start(), rhs.source_range().end()); Expression::Binary(BinaryExpression::new( self.next_node_id(), lhs, rhs, operation, source_range, )) }) } fn bitwise_or_expression(&mut self) -> Option { let mut base = self.bitwise_xor_expression(); while self.current.is_some() && self.peek_current(TokenKind::Bar) { self.advance(); // | let rhs = self.bitwise_xor_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::BitwiseOr); } base } fn bitwise_xor_expression(&mut self) -> Option { let mut base = self.bitwise_and_expression(); while self.current.is_some() && self.peek_current(TokenKind::Caret) { self.advance(); // ^ let rhs = self.bitwise_and_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::BitwiseXor); } base } fn bitwise_and_expression(&mut self) -> Option { let mut base = self.shift_expression(); while self.current.is_some() && self.peek_current(TokenKind::Ampersand) { self.advance(); // & let rhs = self.shift_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::BitwiseAnd); } base } fn shift_expression(&mut self) -> Option { let mut base = self.additive_expression(); while self.current.is_some() { let current = self.current.as_ref().unwrap(); match current.kind() { TokenKind::Lt => { let previous_cloned = current.clone(); self.advance(); // first < self.expect_immediately_after_advance(TokenKind::Lt, &previous_cloned); // second < let rhs = self.additive_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::LeftShift); } TokenKind::Gt => { let previous_cloned = current.clone(); self.advance(); // first > self.expect_immediately_after_advance(TokenKind::Gt, &previous_cloned); // second > let rhs = self.additive_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::RightShift); } _ => break, } } base } fn additive_expression(&mut self) -> Option { let mut base = self.multiplicative_expression(); while self.current.is_some() { let current = self.current.as_ref().unwrap(); match current.kind() { TokenKind::Plus => { self.advance(); // plus let rhs = self.multiplicative_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::Add); } TokenKind::Minus => { self.advance(); // minus let rhs = self.multiplicative_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::Subtract); } _ => break, } } base } fn multiplicative_expression(&mut self) -> Option { let mut base = self.prefix_expression(); while self.current.is_some() { let current = self.current.as_ref().unwrap(); match current.kind() { TokenKind::Star => { self.advance(); // multiply let rhs = self.prefix_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::Multiply); } TokenKind::Slash => { self.advance(); // slash let rhs = self.prefix_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::Divide); } TokenKind::Modulo => { self.advance(); // modulo let rhs = self.prefix_expression(); base = self.zip_binary_expression(base, rhs, BinaryOperation::Modulo); } _ => break, } } base } fn prefix_expression(&mut self) -> Option { // first, collect all consecutive operators let mut operator_tokens = vec![]; while self.current.is_some() { let current = self.current.as_ref().unwrap(); match current.kind() { TokenKind::Minus => { operator_tokens.push(current.clone()); // unfortunately necessary self.advance(); } _ => break, } } // now go in reverse and build up expressions // the parser is currently just after the prefix operators, so we need a suffix expression // as a base let mut base = self.suffix_expression(); while let Some(operator_token) = operator_tokens.pop() { match operator_token.kind() { TokenKind::Minus => { base = base.map(|expression| { let source_range = SourceRange::new( operator_token.start(), expression.source_range().end(), ); Expression::Negative(NegativeExpression::new( self.next_node_id(), expression, source_range, )) }); } _ => unreachable!(), } } base } fn suffix_expression(&mut self) -> Option { if let Some(mut expression) = self.expression_base() { while self.current.is_some() { let current = self.current.as_ref().unwrap(); match current.kind() { TokenKind::LeftParentheses => { expression = Expression::Call(self.call(expression)); } TokenKind::Dot => { self.advance(); // . if let Some(identifier) = self.identifier() { let source_range = SourceRange::new( expression.source_range().start(), identifier.source_range().end(), ); expression = Expression::Path(Path::new( self.next_node_id(), source_range, expression, identifier, )); } } _ => break, } } Some(expression) } else { None } } fn identifier(&mut self) -> Option { if let Some(identifier_token) = self.expect_advance(TokenKind::Identifier) { Some(Identifier::new( self.next_node_id(), self.token_text(&identifier_token), SourceRange::new(identifier_token.start(), identifier_token.end()), )) } else { None } } fn expression_base(&mut self) -> Option { let current = match self.current.as_ref() { Some(current) => current, None => { self.diagnostics.push(get_expected_but_found_eoi( &EXPRESSION_FIRSTS, self.input.len(), )); return None; } }; match current.kind() { TokenKind::IntegerLiteral => { let raw = self.token_text(¤t); let source_range = SourceRange::new(current.start(), current.end()); self.advance(); Some(Expression::Integer(IntegerLiteral::new( self.next_node_id(), i32::from_str(raw).unwrap(), source_range, ))) } TokenKind::DoubleLiteral => { let raw = self.token_text(¤t); let source_range = SourceRange::new(current.start(), current.end()); self.advance(); Some(Expression::Double(DoubleLiteral::new( self.next_node_id(), f64::from_str(raw).unwrap(), source_range, ))) } TokenKind::String => { let with_quotes = self.token_text(¤t); let source_range = SourceRange::new(current.start(), current.end()); self.advance(); Some(Expression::String(StringLiteral::new( self.next_node_id(), &with_quotes[1..with_quotes.len() - 1], source_range, ))) } TokenKind::Identifier => { let declared_name = self.token_text(¤t); let source_range = SourceRange::new(current.start(), current.end()); self.advance(); Some(Expression::Identifier(Identifier::new( self.next_node_id(), declared_name, source_range, ))) } _ => unreachable!("Unreachable token type found: {:?}", current.kind()), } } fn call(&mut self, callee: Expression) -> Call { self.expect_advance(TokenKind::LeftParentheses); let mut arguments = vec![]; if let Some(current) = &self.current { if matches_expression_first!(current.kind()) { arguments = self.expression_list(); } } let maybe_right_parenthesis = self.expect_advance(TokenKind::RightParentheses); let source_range = if let Some(token) = maybe_right_parenthesis { SourceRange::new(callee.source_range().start(), token.end()) } else { // This should be good enough for error reporting if we're missing the right parenthesis SourceRange::new(callee.source_range().start(), callee.source_range().end()) }; Call::new(self.next_node_id(), callee, arguments, source_range) } fn expression_list(&mut self) -> Vec { let mut expressions = vec![]; if let Some(expression) = self.expression() { expressions.push(expression); } while self.current.is_some() && self.peek_current(TokenKind::Comma) { self.advance(); if let Some(expression) = self.expression() { expressions.push(expression); } } expressions } } #[cfg(test)] mod smoke_tests { use super::*; fn smoke_test(input: &str) { let parse_result = parse_compilation_unit(input, None); if !parse_result.is_ok() { eprintln!("{:#?}", parse_result.1); panic!("There were diagnostics during parsing"); } } #[test] fn forty_two() { smoke_test("fn main() 42 end"); } #[test] fn chained_calls() { smoke_test("fn main() getCl()() end"); } #[test] fn extern_fn_with_param() { smoke_test("extern fn println(message: Any) -> Void"); } #[test] fn fn_with_param() { smoke_test("fn foo(bar: Int) end"); } #[test] fn fn_with_params() { smoke_test("fn foo(bar: Int, baz: Int) end"); } #[test] fn return_type() { smoke_test("fn foo() -> Int end") } #[test] fn extern_return_type() { smoke_test("extern fn foo() -> Int"); } #[test] fn add_two_numbers() { smoke_test("fn main() 1 + 2 end"); } #[test] fn negative_return() { smoke_test("fn main() -> Int -1 end"); } #[test] fn negative_left_add() { smoke_test("fn main() -> Int -1 + 1 end"); } #[test] fn negative_right_add() { smoke_test("fn main() -> Int 1 + -1 end"); } #[test] fn two_negatives() { smoke_test("fn main() -> Int -1 + -1 end"); } #[test] fn minus_positive_number() { smoke_test("fn main() -> Int 1 - 1 end"); } #[test] fn minus_negative_number() { smoke_test("fn main() -> Int 1 - -1 end"); } #[test] fn empty_class() { smoke_test("class Foo end"); } #[test] fn class_with_pub_member() { smoke_test("class Foo pub bar end"); } #[test] fn class_with_mut_member() { smoke_test("class Foo mut bar end"); } #[test] fn class_with_nothing_member() { smoke_test("class Foo bar end"); } #[test] fn class_with_member_type_use() { smoke_test("class Foo bar: Int end"); } #[test] fn class_with_member_init() { smoke_test("class Foo bar = 42 end"); } #[test] fn class_with_member_type_use_and_init() { smoke_test("class Foo bar: Int = 42 end"); } #[test] fn class_with_member_all() { smoke_test("class Foo pub mut bar: Bar = Baz() end"); } #[test] fn class_with_pub_fn() { smoke_test( " class Greeter pub fn greet() end end ", ); } #[test] fn simple_assign() { smoke_test( " fn main() let mut x = 4 x = 42 end ", ); } #[test] fn simple_multiply() { smoke_test("fn main() 1 * 2 end"); } #[test] fn simple_divide() { smoke_test("fn main() 1 / 2 end"); } #[test] fn simple_modulo() { smoke_test("fn main() 1 % 2 end"); } #[test] fn simple_left_shift() { smoke_test("fn main() 2 << 1 end"); } #[test] fn simple_right_shift() { smoke_test("fn main() 4 >> 1 end"); } #[test] fn simple_bitwise_and() { smoke_test("fn main() 2 & 1 end"); } #[test] fn simple_bitwise_xor() { smoke_test("fn main() 1 ^ 2 end"); } #[test] fn simple_bitwise_or() { smoke_test("fn main() 1 | 2 end"); } #[test] fn ops_left_to_right() { smoke_test( " fn main() 1 | 2 ^ 3 & 4 << 5 >> 7 + 8 - 9 * 10 / 11 % 12 end ", ) } #[test] fn array_generic_arg() { smoke_test("fn main(args: Array) end"); } #[test] fn nested_generic_args() { smoke_test("fn main(foo: Array>) end"); } #[test] fn class_with_generic_param() { smoke_test("class Foo end"); } #[test] fn single_member_access() { smoke_test("fn main(foo: Foo) foo.bar end") } #[test] fn double_member_access() { smoke_test("fn main(foo: Foo) foo.bar.baz end") } #[test] fn call_member() { smoke_test("fn main(s: String) s.len() end") } } #[cfg(test)] mod concrete_tests { use super::*; fn report_diagnostics(diagnostics: &[Diagnostic]) -> ! { for diagnostic in diagnostics { eprintln!("{:?}", diagnostic); } panic!(); } fn assert_compilation_unit(input: &str) -> CompilationUnit { let parse_result = parse_compilation_unit(input, None); if !parse_result.is_ok() { report_diagnostics(&parse_result.1); } parse_result.0 } fn assert_expression(input: &str) -> Expression { let parse_result = parse_expression(input); if parse_result.is_ok() { parse_result.0.unwrap() } else { report_diagnostics(&parse_result.1); } } fn assert_function_in<'a>( compilation_unit: &'a CompilationUnit, function_name: &str, ) -> &'a Function { compilation_unit .functions() .iter() .find(|f| f.declared_name() == function_name) .unwrap() } #[test] fn parses_extern_fn() { let compilation_unit = assert_compilation_unit("extern fn println() -> Void"); let extern_functions = compilation_unit.extern_functions(); assert_eq!(extern_functions.len(), 1); let extern_function = &extern_functions[0]; assert_eq!(extern_function.declared_name(), "println"); } #[test] fn hello_world() { let compilation_unit = assert_compilation_unit("fn main() println(\"Hello, World!\") end"); let function = assert_function_in(&compilation_unit, "main"); let statements = function.statements(); assert_eq!(statements.len(), 1); if let Statement::Expression(expression_statement) = statements[0] { if let Expression::Call(call) = expression_statement.expression() { let callee = call.callee(); match callee { Expression::Identifier(identifier) => { assert_eq!(identifier.name(), "println"); } _ => panic!("Expected identifier"), } let arguments = call.arguments(); assert_eq!(arguments.len(), 1); let first_argument = &arguments[0]; match first_argument { Expression::String(s) => { assert_eq!(s.content(), "Hello, World!"); } _ => panic!("Expected string"), } } else { panic!("Expected call"); } } else { panic!("Expected expression"); } } #[test] fn negative_expression() { let expression = assert_expression("-1"); match expression { Expression::Negative(negative_expression) => match negative_expression.operand() { Expression::Integer(integer_literal) => { assert_eq!(integer_literal.value(), 1); } _ => panic!("Expected integer literal"), }, _ => panic!("Expected negative expression"), } } #[test] fn add_negative() { let expression = assert_expression("1 + -1"); match expression { Expression::Binary(binary_expression) => { assert!(matches!(binary_expression.op(), BinaryOperation::Add)); match binary_expression.lhs() { Expression::Integer(integer_literal) => { assert_eq!(integer_literal.value(), 1); } _ => panic!("Expected integer literal"), } match binary_expression.rhs() { Expression::Negative(negative_expression) => { match negative_expression.operand() { Expression::Integer(integer_literal) => { assert_eq!(integer_literal.value(), 1); } _ => panic!("Expected integer literal"), } } _ => panic!("Expected negative expression"), } } _ => panic!("Expected additive expression"), } } #[test] fn simple_subtract() { let expression = assert_expression("1 - 1"); match expression { Expression::Binary(binary_expression) => { assert!(matches!(binary_expression.op(), BinaryOperation::Subtract)); match binary_expression.lhs() { Expression::Integer(integer_literal) => { assert_eq!(integer_literal.value(), 1); } _ => panic!("Expected integer literal"), } match binary_expression.rhs() { Expression::Integer(integer_literal) => { assert_eq!(integer_literal.value(), 1); } _ => panic!("Expected integer literal"), } } _ => panic!("Expected subtract expression"), } } }