deimos-lang/dmc-lib/src/parser.rs

use crate::ast::assign_statement::AssignStatement;
use crate::ast::binary_expression::{BinaryExpression, BinaryOperation};
use crate::ast::call::Call;
use crate::ast::class::Class;
use crate::ast::compilation_unit::CompilationUnit;
use crate::ast::constructor::Constructor;
use crate::ast::double_literal::DoubleLiteral;
use crate::ast::expression::Expression;
use crate::ast::expression_statement::ExpressionStatement;
use crate::ast::extern_function::ExternFunction;
use crate::ast::field::Field;
use crate::ast::function::Function;
use crate::ast::generic_parameter::GenericParameter;
use crate::ast::identifier::Identifier;
use crate::ast::integer_literal::IntegerLiteral;
use crate::ast::let_statement::LetStatement;
use crate::ast::negative_expression::NegativeExpression;
use crate::ast::parameter::Parameter;
use crate::ast::statement::Statement;
use crate::ast::string_literal::StringLiteral;
use crate::ast::type_use::TypeUse;
use crate::diagnostic::Diagnostic;
use crate::lexer::Lexer;
use crate::source_range::SourceRange;
use crate::token::{Token, TokenKind};
use std::str::FromStr;

pub fn parse_compilation_unit(input: &str) -> Result<CompilationUnit, Vec<Diagnostic>> {
    let mut parser = Parser::new(input);
    parser.compilation_unit()
}

pub fn parse_expression(input: &str) -> Result<Expression, Vec<Diagnostic>> {
    let mut parser = Parser::new(input);
    parser.advance(); // get started
    parser.expression()
}

macro_rules! matches_expression_first {
    ( $token_kind : expr ) => {
        matches!(
            $token_kind,
            TokenKind::IntegerLiteral
                | TokenKind::DoubleLiteral
                | TokenKind::LongLiteral
                | TokenKind::String
                | TokenKind::Minus
                | TokenKind::SelfKw
                | TokenKind::Identifier
        )
    };
}

macro_rules! matches_statement_first {
    ( $token_kind : expr ) => {
        matches!($token_kind, TokenKind::Let) || matches_expression_first!($token_kind)
    };
}

macro_rules! matches_type_use_first {
    ( $token_kind: expr ) => {
        matches!($token_kind, TokenKind::LeftSquare | TokenKind::Identifier)
    };
}

struct Parser<'a> {
    input: &'a str,
    lexer: Lexer<'a>,
    current: Option<Token>,
    lookahead: Option<Token>,
}

impl<'a> Parser<'a> {
    fn new(input: &'a str) -> Self {
        Self {
            input,
            lexer: Lexer::new(input),
            current: None,
            lookahead: None,
        }
    }

    fn advance_until(&mut self, token_kinds: &[TokenKind]) {
        while self.current.is_some() {
            self.advance();
            match &self.current {
                None => {
                    // reached eoi
                }
                Some(current) => {
                    if token_kinds.contains(&current.kind()) {
                        break;
                    }
                }
            }
        }
    }

    fn advance(&mut self) {
        if self.lookahead.is_some() {
            // we've advanced at least once
            self.current = self.lookahead.take();
            self.lookahead = match self.lexer.next() {
                None => None,
                Some(result) => match result {
                    Ok(token) => Some(token),
                    Err(lexer_error) => {
                        panic!("{:?}", lexer_error);
                    }
                },
            }
        } else if self.lookahead.is_none() && self.current.is_some() {
            // we're on the last token
            self.current = None;
        } else {
            // we've not yet advanced, so fetch both
            // current
            match self.lexer.next() {
                None => {}
                Some(result) => match result {
                    Ok(token) => {
                        self.current = Some(token);
                    }
                    Err(lexer_error) => {
                        panic!("{:?}", lexer_error);
                    }
                },
            }
            // lookahead
            match self.lexer.next() {
                None => {}
                Some(result) => match result {
                    Ok(token) => {
                        self.lookahead = Some(token);
                    }
                    Err(lexer_error) => {
                        panic!("{:?}", lexer_error);
                    }
                },
            }
        }
    }

    fn expect_advance(&mut self, token_kind: TokenKind) -> Result<Token, Vec<Diagnostic>> {
        match self.current.take() {
            None => Err(vec![
                Diagnostic::new(
                    &format!("Expected {:?} but found end-of-input.", token_kind),
                    self.input.len(),
                    self.input.len(),
                )
                .with_reporter(file!(), line!()),
            ]),
            Some(token) => {
                if token.kind() == token_kind {
                    self.advance();
                    Ok(token)
                } else {
                    self.advance_until(&[token_kind]);
                    Err(vec![
                        Diagnostic::new(
                            &format!("Expected {:?} but found {:?}", token_kind, token.kind()),
                            token.start(),
                            token.end(),
                        )
                        .with_reporter(file!(), line!()),
                    ])
                }
            }
        }
    }

    fn expect_position_advance(
        &mut self,
        token_kind: TokenKind,
        start_position: usize,
    ) -> Result<Token, Vec<Diagnostic>> {
        let matched = self.expect_advance(token_kind)?;
        if matched.start() == start_position {
            Ok(matched)
        } else {
            Err(vec![
                Diagnostic::new(
                    &format!("Expected {:?} but found {:?}", token_kind, matched.kind()),
                    matched.start(),
                    matched.end(),
                )
                .with_reporter(file!(), line!()),
            ])
        }
    }

    fn peek_current(&self, token_kind: TokenKind) -> bool {
        match &self.current {
            None => panic!("Unexpected end of input."),
            Some(token) => token.kind() == token_kind,
        }
    }

    fn get_current(&self) -> &Token {
        match &self.current {
            None => {
                panic!("Unexpected end of input");
            }
            Some(token) => token,
        }
    }

    fn peek_lookahead(&self, token_kind: TokenKind) -> bool {
        match &self.lookahead {
            None => panic!("Unexpected end of input."),
            Some(token) => token.kind() == token_kind,
        }
    }

    fn sample_input(&self, start: usize, end: usize) -> &'a str {
        &self.input[start..end]
    }

    fn token_text(&self, token: &Token) -> &'a str {
        self.sample_input(token.start(), token.end())
    }

    pub fn compilation_unit(&mut self) -> Result<CompilationUnit, Vec<Diagnostic>> {
        let mut functions: Vec<Function> = vec![];
        let mut extern_functions: Vec<ExternFunction> = vec![];
        let mut classes: Vec<Class> = vec![];

        let mut diagnostics = vec![];

        self.advance(); // get started

        while self.current.is_some() {
            let current = self.get_current();
            match current.kind() {
                TokenKind::Fn | TokenKind::Extern | TokenKind::Class => {
                    match self.module_level_declaration(
                        &mut functions,
                        &mut extern_functions,
                        &mut classes,
                    ) {
                        Ok(_) => {}
                        Err(mut declaration_diagnostics) => {
                            diagnostics.append(&mut declaration_diagnostics)
                        }
                    }
                }
                _ => {
                    diagnostics.push(Diagnostic::new(
                        &format!(
                            "Expected any of {:?}; found {:?}",
                            [TokenKind::Fn, TokenKind::Extern, TokenKind::Class],
                            current.kind()
                        ),
                        current.start(),
                        current.end(),
                    ));
                    self.advance_until(&[TokenKind::Fn, TokenKind::Extern]);
                }
            }
        }
        if diagnostics.is_empty() {
            Ok(CompilationUnit::new(functions, extern_functions, classes))
        } else {
            Err(diagnostics)
        }
    }

    fn module_level_declaration(
        &mut self,
        functions: &mut Vec<Function>,
        extern_functions: &mut Vec<ExternFunction>,
        classes: &mut Vec<Class>,
    ) -> Result<(), Vec<Diagnostic>> {
        let current = self.get_current();
        match current.kind() {
            TokenKind::Fn => {
                let function_result = self.function();
                match function_result {
                    Ok(function) => {
                        functions.push(function);
                        Ok(())
                    }
                    Err(function_diagnostics) => Err(function_diagnostics),
                }
            }
            TokenKind::Extern => {
                let extern_function_result = self.extern_function();
                match extern_function_result {
                    Ok(extern_function) => {
                        extern_functions.push(extern_function);
                        Ok(())
                    }
                    Err(extern_function_diagnostics) => Err(extern_function_diagnostics),
                }
            }
            TokenKind::Class => match self.class() {
                Ok(class) => {
                    classes.push(class);
                    Ok(())
                }
                Err(class_diagnostics) => Err(class_diagnostics),
            },
            _ => unreachable!(),
        }
    }

    fn function(&mut self) -> Result<Function, Vec<Diagnostic>> {
        let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) {
            self.advance(); // pub
            true
        } else {
            false
        };

        self.expect_advance(TokenKind::Fn)?;
        let identifier_token = self.expect_advance(TokenKind::Identifier)?;

        self.expect_advance(TokenKind::LeftParentheses)?;
        let parameters = self.parameter_list()?;
        self.expect_advance(TokenKind::RightParentheses)?;

        let mut diagnostics = vec![];

        let return_type = if self.current.is_some() && self.peek_current(TokenKind::RightArrow) {
            match self.return_type() {
                Ok(type_use) => Some(type_use),
                Err(mut return_type_diagnostics) => {
                    diagnostics.append(&mut return_type_diagnostics);
                    None
                }
            }
        } else {
            None
        };

        let mut statements = vec![];
        while self.current.is_some() && !self.peek_current(TokenKind::End) {
            let statement_result = self.statement();
            match statement_result {
                Ok(statement) => {
                    statements.push(statement);
                }
                Err(mut statement_diagnostics) => {
                    diagnostics.append(&mut statement_diagnostics);
                }
            }
        }

        // if we're missing "end", append it to the other statement diagnostics
        let end_result = self.expect_advance(TokenKind::End);
        match end_result {
            Err(mut end_diagnostics) => {
                diagnostics.append(&mut end_diagnostics);
            }
            _ => {}
        }

        if diagnostics.is_empty() {
            Ok(Function::new(
                self.token_text(&identifier_token),
                SourceRange::new(identifier_token.start(), identifier_token.end()),
                is_public,
                parameters,
                return_type,
                statements,
            ))
        } else {
            Err(diagnostics)
        }
    }

    fn extern_function(&mut self) -> Result<ExternFunction, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::Extern)?;
        self.expect_advance(TokenKind::Fn)?;
        let identifier_token = self.expect_advance(TokenKind::Identifier)?;
        self.expect_advance(TokenKind::LeftParentheses)?;

        let mut diagnostics = vec![];

        let mut maybe_parameters: Option<Vec<Parameter>> = None;
        let params_result = self.parameter_list();
        match params_result {
            Ok(parameters) => {
                maybe_parameters = Some(parameters);
            }
            Err(mut parameter_list_diagnostics) => {
                diagnostics.append(&mut parameter_list_diagnostics);
            }
        }

        let right_parentheses_result = self.expect_advance(TokenKind::RightParentheses);
        match right_parentheses_result {
            Err(mut right_parentheses_diagnostics) => {
                diagnostics.append(&mut right_parentheses_diagnostics);
            }
            Ok(_) => {}
        }

        let return_type = self.return_type()?;

        if diagnostics.is_empty() {
            Ok(ExternFunction::new(
                self.token_text(&identifier_token),
                SourceRange::new(identifier_token.start(), identifier_token.end()),
                maybe_parameters.unwrap(),
                return_type,
            ))
        } else {
            Err(diagnostics)
        }
    }

    fn class(&mut self) -> Result<Class, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::Class)?;
        let identifier_token = self.expect_advance(TokenKind::Identifier)?;
        let mut generic_parameters: Vec<GenericParameter> = vec![];
        let mut fields = vec![];
        let mut functions = vec![];
        let mut maybe_constructor: Option<Constructor> = None;

        let mut diagnostics = vec![];

        while self.current.is_some() && !self.peek_current(TokenKind::End) {
            match self.get_current().kind() {
                TokenKind::Public => match self.public_class_member(
                    &mut fields,
                    &mut functions,
                    &mut maybe_constructor,
                ) {
                    Ok(_) => {}
                    Err(mut member_diagnostics) => diagnostics.append(&mut member_diagnostics),
                },
                TokenKind::Mut | TokenKind::Identifier => match self.field() {
                    Ok(field) => fields.push(field),
                    Err(mut field_diagnostics) => diagnostics.append(&mut field_diagnostics),
                },
                TokenKind::Fn => match self.function() {
                    Ok(function) => functions.push(function),
                    Err(mut function_diagnostics) => diagnostics.append(&mut function_diagnostics),
                },
                TokenKind::Ctor => match self.constructor() {
                    Ok(constructor) => {
                        maybe_constructor = Some(constructor);
                    }
                    Err(mut ctor_diagnostics) => diagnostics.append(&mut ctor_diagnostics),
                },
                TokenKind::Lt => match self.generic_parameters() {
                    Ok(parsed_params) => {
                        generic_parameters = parsed_params;
                    }
                    Err(mut generic_parameters_diagnostics) => {
                        diagnostics.append(&mut generic_parameters_diagnostics);
                    }
                },
                _ => unreachable!(),
            }
        }

        self.expect_advance(TokenKind::End)?;

        if diagnostics.is_empty() {
            Ok(Class::new(
                self.token_text(&identifier_token),
                SourceRange::new(identifier_token.start(), identifier_token.end()),
                generic_parameters,
                maybe_constructor,
                fields,
                functions,
            ))
        } else {
            Err(diagnostics)
        }
    }

    fn parameter_list(&mut self) -> Result<Vec<Parameter>, Vec<Diagnostic>> {
        let mut parameters = vec![];
        let mut diagnostics = vec![];
        while self.current.is_some() && self.peek_current(TokenKind::Identifier) {
            let parameter_result = self.parameter();
            match parameter_result {
                Ok(parameter) => {
                    parameters.push(parameter);
                }
                Err(mut parameter_diagnostics) => {
                    diagnostics.append(&mut parameter_diagnostics);
                }
            }
            if self.current.is_some() && self.peek_current(TokenKind::Comma) {
                self.advance();
            }
        }
        if diagnostics.is_empty() {
            Ok(parameters)
        } else {
            Err(diagnostics)
        }
    }

    fn parameter(&mut self) -> Result<Parameter, Vec<Diagnostic>> {
        let identifier_token = self.expect_advance(TokenKind::Identifier)?;
        self.expect_advance(TokenKind::Colon)?;
        let type_use = self.type_use()?;
        Ok(Parameter::new(
            self.token_text(&identifier_token),
            SourceRange::new(identifier_token.start(), identifier_token.end()),
            type_use,
        ))
    }

    fn return_type(&mut self) -> Result<TypeUse, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::RightArrow)?;
        self.type_use()
    }

    fn type_use(&mut self) -> Result<TypeUse, Vec<Diagnostic>> {
        if self.current.is_some() {
            let current = self.get_current();
            return match current.kind() {
                TokenKind::LeftSquare => {
                    self.advance(); // [
                    let inner_type_use = self.type_use()?;
                    self.expect_advance(TokenKind::RightSquare)?;
                    todo!()
                }
                TokenKind::Identifier => {
                    let identifier_token = self.expect_advance(TokenKind::Identifier)?;
                    let generic_arguments =
                        if self.current.is_some() && self.peek_current(TokenKind::Lt) {
                            self.advance(); // <
                            let generic_arguments = self.generic_arguments_list()?;
                            self.expect_advance(TokenKind::Gt)?; // >
                            generic_arguments
                        } else {
                            vec![]
                        };
                    Ok(TypeUse::new(
                        self.token_text(&identifier_token),
                        SourceRange::new(identifier_token.start(), identifier_token.end()),
                        generic_arguments,
                    ))
                }
                _ => Err(vec![Diagnostic::new(
                    &format!(
                        "Expected LeftSquare or Identifier; found: {:?}",
                        current.kind()
                    ),
                    current.start(),
                    current.end(),
                )]),
            };
        }
        Err(vec![Diagnostic::new(
            "Expected LeftSquare or Identifier; found end of input.",
            self.input.len(),
            self.input.len(),
        )])
    }

    fn generic_arguments_list(&mut self) -> Result<Vec<TypeUse>, Vec<Diagnostic>> {
        let mut generic_arguments: Vec<TypeUse> = vec![];
        while self.current.is_some() && matches_type_use_first!(self.get_current().kind()) {
            generic_arguments.push(self.type_use()?);
            if self.current.is_some() && self.peek_current(TokenKind::Comma) {
                self.advance(); // comma
            } else {
                break;
            }
        }
        Ok(generic_arguments)
    }

    fn generic_parameters(&mut self) -> Result<Vec<GenericParameter>, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::Lt)?;
        let mut parameters: Vec<GenericParameter> = vec![];
        while self.current.is_some() && self.peek_current(TokenKind::Identifier) {
            parameters.push(self.generic_parameter()?);
            if self.current.is_some() && self.peek_current(TokenKind::Plus) {
                self.advance(); // +
            } else {
                break;
            }
        }
        self.expect_advance(TokenKind::Gt)?;
        Ok(parameters)
    }

    fn generic_parameter(&mut self) -> Result<GenericParameter, Vec<Diagnostic>> {
        let identifier = self.expect_advance(TokenKind::Identifier)?;
        let mut extends_list: Vec<TypeUse> = vec![];
        if self.current.is_some() && self.peek_current(TokenKind::Colon) {
            self.advance(); // :
            while self.current.is_some() && matches_type_use_first!(self.get_current().kind()) {
                extends_list.push(self.type_use()?);
                if self.current.is_some() && self.peek_current(TokenKind::Comma) {
                    self.advance(); // ,
                } else {
                    break;
                }
            }
        }
        Ok(GenericParameter::new(
            self.token_text(&identifier),
            SourceRange::new(identifier.start(), identifier.end()),
            extends_list,
        ))
    }

    fn public_class_member(
        &mut self,
        fields: &mut Vec<Field>,
        functions: &mut Vec<Function>,
        maybe_ctor: &mut Option<Constructor>,
    ) -> Result<(), Vec<Diagnostic>> {
        if self.lookahead.is_some() {
            match self.lookahead.as_ref().unwrap().kind() {
                TokenKind::Mut | TokenKind::Identifier => {
                    fields.push(self.field()?);
                }
                TokenKind::Fn => functions.push(self.function()?),
                TokenKind::Ctor => {
                    maybe_ctor.replace(self.constructor()?);
                }
                _ => {
                    let lookahead = self.lookahead.as_ref().unwrap();
                    return Err(vec![Diagnostic::new(
                        &format!(
                            "Expected any of {:?}; found {:?}",
                            [TokenKind::Mut, TokenKind::Identifier, TokenKind::Fn],
                            lookahead.kind()
                        ),
                        lookahead.start(),
                        lookahead.end(),
                    )]);
                }
            }
            Ok(())
        } else {
            let current = self.current.as_ref().unwrap();
            Err(vec![Diagnostic::new(
                &format!(
                    "Expected any of {:?}; found end-of-input.",
                    [TokenKind::Mut, TokenKind::Identifier, TokenKind::Fn]
                ),
                current.end(),
                current.end(),
            )])
        }
    }

    fn constructor(&mut self) -> Result<Constructor, Vec<Diagnostic>> {
        let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) {
            self.advance();
            true
        } else {
            false
        };

        let ctor_keyword = self.expect_advance(TokenKind::Ctor)?;
        self.expect_advance(TokenKind::LeftParentheses)?;

        let parameters = if self.current.is_some() && self.peek_current(TokenKind::Identifier) {
            self.parameter_list()?
        } else {
            vec![]
        };

        self.expect_advance(TokenKind::RightParentheses)?;

        // statements
        let mut statements: Vec<Statement> = vec![];
        let mut diagnostics: Vec<Diagnostic> = vec![];

        while self.current.is_some()
            && matches_statement_first!(self.current.as_ref().unwrap().kind())
        {
            match self.statement() {
                Ok(statement) => {
                    statements.push(statement);
                }
                Err(mut statement_diagnostics) => {
                    diagnostics.append(&mut statement_diagnostics);
                }
            }
        }

        self.expect_advance(TokenKind::End)?;

        if diagnostics.is_empty() {
            Ok(Constructor::new(
                is_public,
                SourceRange::new(ctor_keyword.start(), ctor_keyword.end()),
                parameters,
                statements,
            ))
        } else {
            Err(diagnostics)
        }
    }

    fn field(&mut self) -> Result<Field, Vec<Diagnostic>> {
        let is_public = if self.current.is_some() && self.peek_current(TokenKind::Public) {
            self.advance();
            true
        } else {
            false
        };

        let is_mut = if self.current.is_some() && self.peek_current(TokenKind::Mut) {
            self.advance();
            true
        } else {
            false
        };

        let identifier = self.expect_advance(TokenKind::Identifier)?;

        let declared_type = if self.current.is_some() && self.peek_current(TokenKind::Colon) {
            self.advance(); // colon
            Some(self.type_use()?)
        } else {
            None
        };

        let initializer = if self.current.is_some() && self.peek_current(TokenKind::Equals) {
            self.advance(); // equals
            Some(self.expression()?)
        } else {
            None
        };

        Ok(Field::new(
            self.token_text(&identifier),
            SourceRange::new(identifier.start(), identifier.end()),
            is_public,
            is_mut,
            declared_type,
            initializer,
        ))
    }

    fn statement(&mut self) -> Result<Statement, Vec<Diagnostic>> {
        let current = self.get_current();
        match current.kind() {
            TokenKind::Let => Ok(Statement::Let(self.let_statement()?)),
            _ => self.expression_statement_or_assign_statement(),
        }
    }

    fn let_statement(&mut self) -> Result<LetStatement, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::Let)?;

        let is_mut = if self.current.is_some() && self.peek_current(TokenKind::Mut) {
            self.advance();
            true
        } else {
            false
        };

        let identifier = self.expect_advance(TokenKind::Identifier)?;
        self.expect_advance(TokenKind::Equals)?;
        let expression = self.expression()?;
        Ok(LetStatement::new(
            self.token_text(&identifier),
            SourceRange::new(identifier.start(), identifier.end()),
            is_mut,
            expression,
        ))
    }

    fn expression_statement_or_assign_statement(&mut self) -> Result<Statement, Vec<Diagnostic>> {
        let base = self.expression()?;
        if self.current.is_some() && self.peek_current(TokenKind::Equals) {
            Ok(Statement::Assign(self.assign_rhs(base)?))
        } else {
            Ok(Statement::Expression(ExpressionStatement::new(base)))
        }
    }

    fn assign_rhs(&mut self, destination: Expression) -> Result<AssignStatement, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::Equals)?;
        let value = self.expression()?;
        Ok(AssignStatement::new(destination, value))
    }

    fn expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        self.bitwise_or_expression()
    }

    fn bitwise_or_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.bitwise_xor_expression()?;
        while self.current.is_some() && self.peek_current(TokenKind::Bar) {
            self.advance(); // |
            let rhs = self.bitwise_xor_expression()?;
            let source_range =
                SourceRange::new(result.source_range().start(), rhs.source_range().end());
            result = Expression::Binary(BinaryExpression::new(
                result,
                rhs,
                BinaryOperation::BitwiseOr,
                source_range,
            ));
        }
        Ok(result)
    }

    fn bitwise_xor_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.bitwise_and_expression()?;
        while self.current.is_some() && self.peek_current(TokenKind::Caret) {
            self.advance(); // ^
            let rhs = self.bitwise_and_expression()?;
            let source_range =
                SourceRange::new(result.source_range().start(), rhs.source_range().end());
            result = Expression::Binary(BinaryExpression::new(
                result,
                rhs,
                BinaryOperation::BitwiseXor,
                source_range,
            ));
        }
        Ok(result)
    }

    fn bitwise_and_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.shift_expression()?;
        while self.current.is_some() && self.peek_current(TokenKind::Ampersand) {
            self.advance(); // &
            let rhs = self.shift_expression()?;
            let source_range =
                SourceRange::new(result.source_range().start(), rhs.source_range().end());
            result = Expression::Binary(BinaryExpression::new(
                result,
                rhs,
                BinaryOperation::BitwiseAnd,
                source_range,
            ));
        }
        Ok(result)
    }

    fn shift_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.additive_expression()?;
        while self.current.is_some() {
            let current = self.get_current();
            match current.kind() {
                TokenKind::Lt => {
                    let second_lt_start = current.start() + 1;
                    self.advance(); // first <
                    self.expect_position_advance(TokenKind::Lt, second_lt_start)?; // second <
                    let rhs = self.additive_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::LeftShift,
                        source_range,
                    ));
                }
                TokenKind::Gt => {
                    let second_gt_start = current.start() + 1;
                    self.advance(); // first >
                    self.expect_position_advance(TokenKind::Gt, second_gt_start)?; // second gt
                    let rhs = self.additive_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::RightShift,
                        source_range,
                    ));
                }
                _ => break,
            }
        }
        Ok(result)
    }

    fn additive_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.multiplicative_expression()?;
        while self.current.is_some() {
            let current = self.get_current();
            match current.kind() {
                TokenKind::Plus => {
                    self.advance(); // plus
                    let rhs = self.multiplicative_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::Add,
                        source_range,
                    ));
                }
                TokenKind::Minus => {
                    self.advance(); // minus
                    let rhs = self.multiplicative_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::Subtract,
                        source_range,
                    ));
                }
                _ => break,
            }
        }
        Ok(result)
    }

    fn multiplicative_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.prefix_expression()?;
        while self.current.is_some() {
            let current = self.get_current();
            match current.kind() {
                TokenKind::Star => {
                    self.advance(); // multiply
                    let rhs = self.prefix_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::Multiply,
                        source_range,
                    ));
                }
                TokenKind::Slash => {
                    self.advance(); // slash
                    let rhs = self.prefix_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::Divide,
                        source_range,
                    ))
                }
                TokenKind::Modulo => {
                    self.advance(); // modulo
                    let rhs = self.prefix_expression()?;
                    let source_range =
                        SourceRange::new(result.source_range().start(), rhs.source_range().end());
                    result = Expression::Binary(BinaryExpression::new(
                        result,
                        rhs,
                        BinaryOperation::Modulo,
                        source_range,
                    ));
                }
                _ => break,
            }
        }
        Ok(result)
    }

    fn prefix_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        // first, collect all consecutive operators
        let mut operator_tokens = vec![];
        while self.current.is_some() {
            let current = self.get_current();
            match current.kind() {
                TokenKind::Minus => {
                    operator_tokens.push(current.clone()); // unfortunately necessary
                    self.advance();
                }
                _ => break,
            }
        }

        // now go in reverse and build up expressions
        // the parser is currently just after the prefix operators, so we need a suffix expression
        // as a base
        let mut result = self.suffix_expression()?;
        while let Some(operator_token) = operator_tokens.pop() {
            let source_range =
                SourceRange::new(operator_token.start(), result.source_range().end());
            match operator_token.kind() {
                TokenKind::Minus => {
                    result = Expression::Negative(NegativeExpression::new(result, source_range));
                }
                _ => unreachable!(),
            }
        }
        Ok(result)
    }

    fn suffix_expression(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let mut result = self.expression_base()?;
        while self.current.is_some() {
            let current = self.get_current();
            match current.kind() {
                TokenKind::LeftParentheses => {
                    result = Expression::Call(self.call(result)?);
                }
                _ => break,
            }
        }
        Ok(result)
    }

    fn expression_base(&mut self) -> Result<Expression, Vec<Diagnostic>> {
        let current = self.get_current();
        match current.kind() {
            TokenKind::IntegerLiteral => {
                let raw = self.token_text(&current);
                let source_range = SourceRange::new(current.start(), current.end());
                self.advance();
                Ok(Expression::Integer(IntegerLiteral::new(
                    i32::from_str(raw).unwrap(),
                    source_range,
                )))
            }
            TokenKind::DoubleLiteral => {
                let raw = self.token_text(&current);
                let source_range = SourceRange::new(current.start(), current.end());
                self.advance();
                Ok(Expression::Double(DoubleLiteral::new(
                    f64::from_str(raw).unwrap(),
                    source_range,
                )))
            }
            TokenKind::String => {
                let with_quotes = self.token_text(&current);
                let source_range = SourceRange::new(current.start(), current.end());
                self.advance();
                Ok(Expression::String(StringLiteral::new(
                    &with_quotes[1..with_quotes.len() - 1],
                    source_range,
                )))
            }
            TokenKind::Identifier => {
                let declared_name = self.token_text(&current);
                let source_range = SourceRange::new(current.start(), current.end());
                self.advance();
                Ok(Expression::Identifier(Identifier::new(
                    declared_name,
                    source_range,
                )))
            }
            _ => unreachable!("Unreachable token type found: {:?}", current.kind()),
        }
    }

    fn call(&mut self, callee: Expression) -> Result<Call, Vec<Diagnostic>> {
        self.expect_advance(TokenKind::LeftParentheses)?;
        let mut arguments = vec![];
        if let Some(current) = &self.current {
            if matches_expression_first!(current.kind()) {
                arguments.append(&mut self.expression_list()?);
            }
        }
        let right_parentheses_token = self.expect_advance(TokenKind::RightParentheses)?;
        let source_range =
            SourceRange::new(callee.source_range().start(), right_parentheses_token.end());
        Ok(Call::new(callee, arguments, source_range))
    }

    fn expression_list(&mut self) -> Result<Vec<Expression>, Vec<Diagnostic>> {
        let mut expressions = vec![];
        expressions.push(self.expression()?);
        while self.current.is_some() && self.peek_current(TokenKind::Comma) {
            self.advance(); // comma
            expressions.push(self.expression()?);
        }
        Ok(expressions)
    }
}

#[cfg(test)]
mod smoke_tests {
    use super::*;

    fn smoke_test(input: &str) {
        let parse_result = parse_compilation_unit(input);
        match parse_result {
            Ok(_) => {}
            Err(diagnostics) => {
                eprintln!("{:#?}", diagnostics);
                panic!("There were diagnostics during parsing");
            }
        }
    }

    #[test]
    fn forty_two() {
        smoke_test("fn main() 42 end");
    }

    #[test]
    fn chained_calls() {
        smoke_test("fn main() getCl()() end");
    }

    #[test]
    fn extern_fn_with_param() {
        smoke_test("extern fn println(message: Any) -> Void");
    }

    #[test]
    fn fn_with_param() {
        smoke_test("fn foo(bar: Int) end");
    }

    #[test]
    fn fn_with_params() {
        smoke_test("fn foo(bar: Int, baz: Int) end");
    }

    #[test]
    fn return_type() {
        smoke_test("fn foo() -> Int end")
    }

    #[test]
    fn extern_return_type() {
        smoke_test("extern fn foo() -> Int");
    }

    #[test]
    fn add_two_numbers() {
        smoke_test("fn main() 1 + 2 end");
    }

    #[test]
    fn negative_return() {
        smoke_test("fn main() -> Int -1 end");
    }

    #[test]
    fn negative_left_add() {
        smoke_test("fn main() -> Int -1 + 1 end");
    }

    #[test]
    fn negative_right_add() {
        smoke_test("fn main() -> Int 1 + -1 end");
    }

    #[test]
    fn two_negatives() {
        smoke_test("fn main() -> Int -1 + -1 end");
    }

    #[test]
    fn minus_positive_number() {
        smoke_test("fn main() -> Int 1 - 1 end");
    }

    #[test]
    fn minus_negative_number() {
        smoke_test("fn main() -> Int 1 - -1 end");
    }

    #[test]
    fn empty_class() {
        smoke_test("class Foo end");
    }

    #[test]
    fn class_with_pub_member() {
        smoke_test("class Foo pub bar end");
    }

    #[test]
    fn class_with_mut_member() {
        smoke_test("class Foo mut bar end");
    }

    #[test]
    fn class_with_nothing_member() {
        smoke_test("class Foo bar end");
    }

    #[test]
    fn class_with_member_type_use() {
        smoke_test("class Foo bar: Int end");
    }

    #[test]
    fn class_with_member_init() {
        smoke_test("class Foo bar = 42 end");
    }

    #[test]
    fn class_with_member_type_use_and_init() {
        smoke_test("class Foo bar: Int = 42 end");
    }

    #[test]
    fn class_with_member_all() {
        smoke_test("class Foo pub mut bar: Bar = Baz() end");
    }

    #[test]
    fn class_with_pub_fn() {
        smoke_test(
            "
            class Greeter
                pub fn greet() end
            end
            ",
        );
    }

    #[test]
    fn simple_assign() {
        smoke_test(
            "
            fn main()
                let mut x = 4
                x = 42
            end
            ",
        );
    }

    #[test]
    fn simple_multiply() {
        smoke_test("fn main() 1 * 2 end");
    }

    #[test]
    fn simple_divide() {
        smoke_test("fn main() 1 / 2 end");
    }

    #[test]
    fn simple_modulo() {
        smoke_test("fn main() 1 % 2 end");
    }

    #[test]
    fn simple_left_shift() {
        smoke_test("fn main() 2 << 1 end");
    }

    #[test]
    fn simple_right_shift() {
        smoke_test("fn main() 4 >> 1 end");
    }

    #[test]
    fn simple_bitwise_and() {
        smoke_test("fn main() 2 & 1 end");
    }

    #[test]
    fn simple_bitwise_xor() {
        smoke_test("fn main() 1 ^ 2 end");
    }

    #[test]
    fn simple_bitwise_or() {
        smoke_test("fn main() 1 | 2 end");
    }

    #[test]
    fn ops_left_to_right() {
        smoke_test(
            "
            fn main()
                1 | 2 ^ 3 & 4 << 5 >> 7 + 8 - 9 * 10 / 11 % 12
            end
            ",
        )
    }

    #[test]
    fn array_generic_arg() {
        smoke_test("fn main(args: Array<String>) end");
    }

    #[test]
    fn nested_generic_args() {
        smoke_test("fn main(foo: Array<Bar<Foo>>) end");
    }

    #[test]
    fn class_with_generic_param() {
        smoke_test("class Foo<T> end");
    }
}

#[cfg(test)]
mod concrete_tests {
    use super::*;

    fn report_diagnostics(diagnostics: &[Diagnostic]) -> ! {
        for diagnostic in diagnostics {
            eprintln!("{:?}", diagnostic);
        }
        panic!();
    }

    fn assert_compilation_unit(input: &str) -> CompilationUnit {
        let parse_result = parse_compilation_unit(input);
        match parse_result {
            Ok(compilation_unit) => compilation_unit,
            Err(diagnostics) => {
                report_diagnostics(&diagnostics);
            }
        }
    }

    fn assert_expression(input: &str) -> Expression {
        let parse_result = parse_expression(input);
        match parse_result {
            Ok(expression) => expression,
            Err(diagnostics) => {
                report_diagnostics(&diagnostics);
            }
        }
    }

    fn assert_function_in<'a>(
        compilation_unit: &'a CompilationUnit,
        function_name: &str,
    ) -> &'a Function {
        compilation_unit
            .functions()
            .iter()
            .find(|f| f.declared_name() == function_name)
            .unwrap()
    }

    #[test]
    fn parses_extern_fn() {
        let compilation_unit = assert_compilation_unit("extern fn println() -> Void");
        let extern_functions = compilation_unit.extern_functions();
        assert_eq!(extern_functions.len(), 1);
        let extern_function = &extern_functions[0];
        assert_eq!(extern_function.declared_name(), "println");
    }

    #[test]
    fn hello_world() {
        let compilation_unit = assert_compilation_unit("fn main() println(\"Hello, World!\") end");
        let function = assert_function_in(&compilation_unit, "main");
        let statements = function.statements();
        assert_eq!(statements.len(), 1);
        if let Statement::Expression(expression_statement) = statements[0] {
            if let Expression::Call(call) = expression_statement.expression() {
                let callee = call.callee();
                match callee {
                    Expression::Identifier(identifier) => {
                        assert_eq!(identifier.name(), "println");
                    }
                    _ => panic!("Expected identifier"),
                }
                let arguments = call.arguments();
                assert_eq!(arguments.len(), 1);
                let first_argument = arguments[0];
                match first_argument {
                    Expression::String(s) => {
                        assert_eq!(s.content(), "Hello, World!");
                    }
                    _ => panic!("Expected string"),
                }
            } else {
                panic!("Expected call");
            }
        } else {
            panic!("Expected expression");
        }
    }

    #[test]
    fn negative_expression() {
        let expression = assert_expression("-1");
        match expression {
            Expression::Negative(negative_expression) => match negative_expression.operand() {
                Expression::Integer(integer_literal) => {
                    assert_eq!(integer_literal.value(), 1);
                }
                _ => panic!("Expected integer literal"),
            },
            _ => panic!("Expected negative expression"),
        }
    }

    #[test]
    fn add_negative() {
        let expression = assert_expression("1 + -1");
        match expression {
            Expression::Binary(binary_expression) => {
                assert!(matches!(binary_expression.op(), BinaryOperation::Add));
                match binary_expression.lhs() {
                    Expression::Integer(integer_literal) => {
                        assert_eq!(integer_literal.value(), 1);
                    }
                    _ => panic!("Expected integer literal"),
                }
                match binary_expression.rhs() {
                    Expression::Negative(negative_expression) => {
                        match negative_expression.operand() {
                            Expression::Integer(integer_literal) => {
                                assert_eq!(integer_literal.value(), 1);
                            }
                            _ => panic!("Expected integer literal"),
                        }
                    }
                    _ => panic!("Expected negative expression"),
                }
            }
            _ => panic!("Expected additive expression"),
        }
    }

    #[test]
    fn simple_subtract() {
        let expression = assert_expression("1 - 1");
        match expression {
            Expression::Binary(binary_expression) => {
                assert!(matches!(binary_expression.op(), BinaryOperation::Subtract));
                match binary_expression.lhs() {
                    Expression::Integer(integer_literal) => {
                        assert_eq!(integer_literal.value(), 1);
                    }
                    _ => panic!("Expected integer literal"),
                }
                match binary_expression.rhs() {
                    Expression::Integer(integer_literal) => {
                        assert_eq!(integer_literal.value(), 1);
                    }
                    _ => panic!("Expected integer literal"),
                }
            }
            _ => panic!("Expected subtract expression"),
        }
    }
}

#[cfg(test)]
mod parse_failure_tests {
    use super::*;

    #[test]
    fn lone_end() {
        let parse_result = parse_compilation_unit("end");
        match parse_result {
            Err(diagnostics) => {
                assert_eq!(diagnostics.len(), 1);
                for diagnostic in &diagnostics {
                    println!("{:?}", diagnostic)
                }
            }
            Ok(_) => panic!(),
        }
    }

    #[test]
    fn two_ends() {
        let parse_result = parse_compilation_unit("end end");
        match parse_result {
            Err(diagnostics) => {
                // Should only have an error on the first end, since we advance until we find a
                // token we can recover from (fn or extern)
                assert_eq!(diagnostics.len(), 1);
            }
            Ok(_) => panic!(),
        }
    }
}