Working on tools and tests.

This commit is contained in:
JesseBrault0709 2024-05-24 10:37:58 +02:00
parent f0b133bb22
commit 449c83975f
29 changed files with 343 additions and 51 deletions

View File

@ -157,6 +157,7 @@ def toolSpec = { String name, String mainClass ->
}
final List<ToolSpec> toolSpecs = [
toolSpec('astBuilder', 'AstBuilder'),
toolSpec('astFileMaker', 'AstFileMakerCli'), // deprecated
toolSpec('convertToGroovy', 'ConvertToGroovy'),
toolSpec('groovyWvc', 'GroovyWvcCompiler'),

View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
ARGS="-v -d src/test/ast/ast-files -s _ast -e .txt"
if [ "$1" == "--debug" ]; then
shift
bin/astBuilder --debug $ARGS "$@"
else
bin/astBuilder $ARGS "$@"
fi

View File

@ -1,7 +1,7 @@
lexer grammar LexerFragments;
fragment
NL : [\n\r] ;
NL : '\n' | '\r\n' ;
fragment
WS : [ \t] ;

View File

@ -331,7 +331,7 @@ TagError
mode GROOVY_CODE;
PreambleClose
: THREE_DASH { this.inPreamble() && this.getCharPositionInLine() == 3 }? { this.onPreambleClose(); }
: THREE_DASH { this.inPreamble() && this.getCharPositionInLine() == 3 }? WS* NL? { this.onPreambleClose(); }
;
ScriptletClose

View File

@ -50,7 +50,7 @@ private fun doCheck(tree: ParseTree, destination: MutableList<MismatchedComponen
data class MismatchedComponentTypeError(val component: ComponentWithChildrenContext, val message: String)
fun check(tree: ParseTree): List<MismatchedComponentTypeError> {
fun checkForMismatchedComponentTypeErrors(tree: ParseTree): List<MismatchedComponentTypeError> {
val result: MutableList<MismatchedComponentTypeError> = ArrayList()
doCheck(tree, result)
return result

View File

@ -97,7 +97,7 @@ public final class CompilerPipeline {
// check for mismatched type errors
final List<MismatchedComponentTypeError> mismatchedComponentTypeErrors =
MismatchedComponentTypeAnalysis.check(parseResult.getCompilationUnitContext());
MismatchedComponentTypeAnalysis.checkForMismatchedComponentTypeErrors(parseResult.getCompilationUnitContext());
if (!mismatchedComponentTypeErrors.isEmpty()) {
if (mismatchedComponentTypeErrors.size() == 1) {

View File

@ -0,0 +1,72 @@
CompilationUnitNode(1,1..21,1)
PreambleNode(1,1..6,1)
PreambleBreak[1,1](---\n)
GroovyCode[2,1](import some.Thing // a comment...rld!'\n)
PreambleBreak[5,1](---\n)
BodyNode(6,1..20,8)
BodyTextNode(6,1..7,1)
TextNode(6,1..7,1)
RawText[6,1](<!DOCTYPE html>\n)
TypedComponentNode(7,1..20,7)
ComponentArgsNode(7,2..7,6)
StringComponentTypeNode(7,2..7,6)
StringIdentifier[7,2](html)
BodyNode(7,7..19,12)
TypedComponentNode(8,5..8,17)
ComponentArgsNode(8,6..8,10)
StringComponentTypeNode(8,6..8,10)
StringIdentifier[8,6](head)
TypedComponentNode(9,5..19,11)
ComponentArgsNode(9,6..9,10)
StringComponentTypeNode(9,6..9,10)
StringIdentifier[9,6](body)
BodyNode(9,11..19,5)
TypedComponentNode(10,9..10,28)
ComponentArgsNode(10,10..10,12)
StringComponentTypeNode(10,10..10,12)
StringIdentifier[10,10](h1)
BodyNode(10,13..10,23)
BodyTextNode(10,13..10,23)
DollarScriptletNode(10,13..10,23)
DollarScriptletOpen[10,13](${)
GroovyCode[10,15](greeting)
DollarScriptletClose[10,23](})
TypedComponentNode(11,9..18,33)
ComponentArgsNode(11,10..11,32)
ClassComponentTypeNode(11,10..11,32)
TypedIdentifier[11,10](groowt.view.web.Select)
BodyNode(11,33..18,9)
TypedComponentNode(12,13..14,19)
ComponentArgsNode(12,14..12,35)
ClassComponentTypeNode(12,14..12,18)
TypedIdentifier[12,14](Case)
KeyValueAttrNode(12,19..12,35)
KeyNode(12,19..12,23)
AttributeIdentifier[12,19](cond)
Equals[12,23](=)
ClosureValueNode(12,24..12,35)
ClosureAttrValueStart[12,24]({)
GroovyCode[12,25](isItTrue())
ClosureAttrValueEnd[12,35](})
BodyNode(12,37..14,13)
TypedComponentNode(13,17..13,36)
ComponentArgsNode(13,18..13,18)
StringComponentTypeNode(13,18..13,18)
StringIdentifier[13,18](p)
BodyNode(13,20..13,33)
BodyTextNode(13,20..13,33)
TextNode(13,20..13,33)
RawText[13,20](It's true! :))
TypedComponentNode(15,13..17,22)
ComponentArgsNode(15,14..15,21)
ClassComponentTypeNode(15,14..15,21)
TypedIdentifier[15,14](Default)
BodyNode(15,22..17,13)
TypedComponentNode(16,17..16,39)
ComponentArgsNode(16,18..16,18)
StringComponentTypeNode(16,18..16,18)
StringIdentifier[16,18](p)
BodyNode(16,20..16,36)
BodyTextNode(16,20..16,36)
TextNode(16,20..16,36)
RawText[16,20](It's false... :()

View File

@ -0,0 +1,10 @@
CompilationUnitNode(1,1..2,1)
BodyNode(1,1..2,1)
BodyTextNode(1,1..2,1)
TextNode(1,1..1,8)
RawText[1,1](Hello, )
DollarReferenceNode(1,8..1,15)
DollarReferenceStart[1,8]($)
GroovyCode[1,9](target)
TextNode(1,15..2,1)
RawText[1,15](!\n)

View File

@ -0,0 +1,5 @@
CompilationUnitNode(1,1..2,1)
BodyNode(1,1..2,1)
BodyTextNode(1,1..2,1)
TextNode(1,1..2,1)
RawText[1,1](Hello, World!\n)

View File

@ -0,0 +1,10 @@
CompilationUnitNode(1,1..2,1)
BodyNode(1,1..1,35)
TypedComponentNode(1,1..1,34)
ComponentArgsNode(1,2..1,10)
ClassComponentTypeNode(1,2..1,10)
TypedIdentifier[1,2](Greeting)
BodyNode(1,11..1,24)
BodyTextNode(1,11..1,24)
TextNode(1,11..1,24)
RawText[1,11](to the world!)

View File

@ -0,0 +1 @@
Hello, World!

View File

@ -10,7 +10,6 @@ import org.junit.jupiter.api.Test
import static groowt.view.component.web.antlr.TokenUtil.getTokenName
import static groowt.view.component.web.antlr.WebViewComponentsLexer.GroovyCode
import static groowt.view.component.web.antlr.WebViewComponentsLexer.PreambleBreak
import static groowt.view.component.web.antlr.WebViewComponentsLexerBase.RawText
import static org.antlr.v4.runtime.Recognizer.EOF
import static org.junit.jupiter.api.Assertions.*
@ -51,11 +50,11 @@ class WebViewComponentsTokenStreamTests {
def lexer = new WebViewComponentsLexer(input)
def tokenStream = new WebViewComponentsTokenStream(lexer)
def tokens = tokenStream.allTokens
assertTypes([PreambleBreak, GroovyCode, PreambleBreak, RawText, EOF], tokens)
assertTypes([PreambleBreak, GroovyCode, PreambleBreak, EOF], tokens)
assertMergedGroovyCodeToken(tokens[1]) {
assertEquals('println \'Hello, World!\' // comment\n', it.text)
}
assertIterableEquals(0..4, tokens*.tokenIndex)
assertIterableEquals(0..3, tokens*.tokenIndex)
}
}

View File

@ -8,6 +8,7 @@ import groowt.view.component.web.antlr.WebViewComponentsParser
import groowt.view.component.web.antlr.WebViewComponentsTokenStream
import groowt.view.component.web.ast.node.*
import org.antlr.v4.runtime.CharStreams
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import static groowt.view.component.web.antlr.WebViewComponentsParser.CompilationUnitContext
@ -52,6 +53,7 @@ class DefaultAstBuilderVisitorTests {
}
@Test
@Disabled('Move to file tests.')
void helloTarget() {
def (node, tokenList) = this.doBuild('Hello, $target!')
assertNodeWith(CompilationUnitNode, node) {

View File

@ -19,7 +19,7 @@ public class DefaultAstBuilderTests extends AstBuilderTests {
super(
Path.of("src", "test", "ast"),
"*.wvc",
Path.of("src", "test", "ast", "trees"),
Path.of("src", "test", "ast", "ast-files"),
"_ast.txt"
);
}

View File

@ -1,7 +1,7 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](import some.Thing // a comment...rld!'\n)
2: PreambleBreak[5,1](---)
3: RawText[5,4](\n<!DOCTYPE html>\n)
2: PreambleBreak[5,1](---\n)
3: RawText[6,1](<!DOCTYPE html>\n)
4: ComponentOpen[7,1](<)
5: StringIdentifier[7,2](html)
6: ComponentClose[7,6](>)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---)
1: GroovyCode[1,4](-\n)
2: PreambleBreak[2,1](---)
3: RawText[2,4](\n)
2: PreambleBreak[2,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](// ---\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)
2: PreambleBreak[3,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def regex = $/match --- me $ $... / /$\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)
2: PreambleBreak[3,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = "---$test${test('---')}"\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)
2: PreambleBreak[3,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = '---'\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)
2: PreambleBreak[3,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def regex = (/match --- \/ me/)\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)
2: PreambleBreak[3,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = """\n--- $test ${te...\n"""\n)
2: PreambleBreak[5,1](---)
3: RawText[5,4](\n)
2: PreambleBreak[5,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = '''\n---\n'''\n)
2: PreambleBreak[5,1](---)
3: RawText[5,4](\n)
2: PreambleBreak[5,1](---\n)

View File

@ -1,4 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](package test\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)
2: PreambleBreak[3,1](---\n)

View File

@ -1,12 +1,12 @@
compilationUnit[1,1..21,1]
preamble[1,1..5,4]
preamble[1,1..6,1]
PreambleBreak[1,1](---\n)
GroovyCode[2,1](import some.Thing // a comment\n\ndef greeting = 'Hello, World!'\n)
PreambleBreak[5,1](---)
body[5,4..20,8]
bodyText[5,4..7,1]
text[5,4..7,1]
RawText[5,4](\n<!DOCTYPE html>\n)
PreambleBreak[5,1](---\n)
body[6,1..20,8]
bodyText[6,1..7,1]
text[6,1..7,1]
RawText[6,1](<!DOCTYPE html>\n)
component[7,1..20,7]
componentWithChildren[7,1..20,7]
openComponent[7,1..7,6]

View File

@ -80,7 +80,7 @@ final class AstFileMaker extends AbstractOutputFileMaker {
)
}
def mismatchedTypeErrors = MismatchedComponentTypeAnalysis.check(cuContext)
def mismatchedTypeErrors = MismatchedComponentTypeAnalysis.checkForMismatchedComponentTypeErrors(cuContext)
if (!mismatchedTypeErrors.isEmpty()) {
def message = 'There were mismatched type errors: \n' + mismatchedTypeErrors.collect {

View File

@ -0,0 +1,193 @@
package groowt.view.component.web.tools
import groowt.view.component.web.analysis.MismatchedComponentTypeError
import groowt.view.component.web.analysis.checkForMismatchedComponentTypeErrors
import groowt.view.component.web.antlr.*
import groowt.view.component.web.antlr.WebViewComponentsLexerBase.ERROR
import groowt.view.component.web.antlr.WebViewComponentsLexerBase.HIDDEN
import groowt.view.component.web.antlr.WebViewComponentsParser.CompilationUnitContext
import groowt.view.component.web.ast.DefaultAstBuilder
import groowt.view.component.web.ast.DefaultNodeFactory
import groowt.view.component.web.ast.formatAst
import groowt.view.component.web.ast.node.CompilationUnitNode
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.ConsoleErrorListener
import picocli.CommandLine
import picocli.CommandLine.Command
import picocli.CommandLine.Option
import java.nio.file.Path
import kotlin.io.path.nameWithoutExtension
import kotlin.system.exitProcess
@Command(
name = "astBuilder",
description = ["Create an AST from a .wvc file."],
mixinStandardHelpOptions = true,
version = ["0.1.0"]
)
open class AstBuilder : AbstractSourceTransformerCli() {
companion object {
@JvmStatic
fun main(args: Array<String>) {
exitProcess(CommandLine(AstBuilder()).execute(*args))
}
}
@Option(
names = ["-s", "--suffix"],
description = ["The suffix (not extension!) to append to the output file."]
)
protected var suffix: String? = null
@Option(
names = ["-e", "--extension"],
description = ["The extension for output files."],
defaultValue = ".txt"
)
protected lateinit var extension: String
@Option(
names = ["-d", "--output-dir"],
description = ["The output directory."],
defaultValue = ".",
paramLabel = "outputDir"
)
protected lateinit var myOutputDir: Path
@Option(
names = ["--strict"],
description = ["If true, do not recover from syntax errors during parsing."],
negatable = true
)
protected var strict = true
protected open fun onLexerErrors(errors: List<LexerError>): Boolean {
System.err.println("There were lexer errors.")
errors.forEach { System.err.println(formatLexerError(it)) }
return this.getYesNo("Do you wish to try again?", false)
}
protected open fun onParserErrors(errors: List<ParserError>): Boolean {
System.err.println("There were parser errors.")
errors.forEach { System.err.println(formatParserError(it)) }
return this.getYesNo("Do you wish to try again?", false)
}
protected open fun onMismatchedErrors(errors: List<MismatchedComponentTypeError>): Boolean {
System.err.println("There were mismatched component type errors.")
errors.forEach { System.err.println(it.message) }
return getYesNo("Do you wish to try again?", false)
}
protected open fun onException(phase: String, e: Exception): Boolean {
System.err.println("There was an exception during $phase: $e")
if (this.verbose) {
e.printStackTrace(System.err)
}
return this.getYesNo("Do you wish to try again?", false)
}
protected open fun getFullTargetPath(target: Path): Path =
Path.of(target.nameWithoutExtension + (suffix ?: "") + extension)
protected open fun onSuccess(target: Path, tokenList: TokenList, cuNode: CompilationUnitNode): Boolean {
val formatted = formatAst(cuNode, tokenList)
if (interactive) {
println("Please review the following ast:\n$formatted")
} else {
println(formatted)
}
if (getYesNo("Do you wish to write to disk?", true)) {
writeToDisk(getFullTargetPath(target), formatted)
return false
} else {
return getYesNo("Do you wish to redo this file?", false)
}
}
override fun getOutputDir() = myOutputDir
override fun transform(target: Path): Int {
if (interactive) {
println("Building ast for $target")
}
while (true) {
val parseResult: Pair<TokenList, CompilationUnitContext> = try {
val input = CharStreams.fromPath(target)
val lexer = WebViewComponentsLexer(input)
lexer.removeErrorListener(ConsoleErrorListener.INSTANCE)
val lexerErrorListener = LexerErrorListener()
lexer.addErrorListener(lexerErrorListener)
val tokenStream = if (strict) {
WebViewComponentsTokenStream(lexer) // only ignore hidden (default)
} else {
WebViewComponentsTokenStream(lexer, setOf(HIDDEN, ERROR)) // ignore hidden and error
}
val parser = WebViewComponentsParser(tokenStream)
parser.removeErrorListener(ConsoleErrorListener.INSTANCE)
val parserErrorListener = ParserErrorListener()
parser.addErrorListener(parserErrorListener)
val cuContext = parser.compilationUnit()
val lexerErrors = lexerErrorListener.getErrors() + parserErrorListener.getLexerErrors()
val parserErrors = parserErrorListener.getParserErrors()
if (lexerErrors.isNotEmpty()) {
val recover = this.onLexerErrors(lexerErrors)
if (!recover) {
return 1
}
} else if (parserErrors.isNotEmpty()) {
val recover = this.onParserErrors(parserErrors)
if (!recover) {
return 1
}
}
Pair(TokenList(tokenStream), cuContext)
} catch (e: Exception) {
val recover = this.onException("parsing", e)
if (!recover) {
return 1
} else {
continue
}
}
val mismatchedErrors = checkForMismatchedComponentTypeErrors(parseResult.second)
if (mismatchedErrors.isNotEmpty()) {
val tryAgain = this.onMismatchedErrors(mismatchedErrors)
if (!tryAgain) {
return 1
} else {
continue
}
}
val nodeFactory = DefaultNodeFactory(parseResult.first)
val astBuilder = DefaultAstBuilder(nodeFactory)
val cuNode: CompilationUnitNode = try {
astBuilder.buildCompilationUnit(parseResult.second)
} catch (e: Exception) {
val recover = this.onException("ast building", e)
if (!recover) {
return 1
} else {
continue
}
}
val redo = this.onSuccess(target, parseResult.first, cuNode)
if (!redo) {
return 0
}
}
}
}

View File

@ -7,11 +7,18 @@ import groowt.view.component.web.antlr.WebViewComponentsParser.CompilationUnitCo
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.ConsoleErrorListener
import picocli.CommandLine
import picocli.CommandLine.Command
import picocli.CommandLine.Option
import java.nio.file.Path
import kotlin.io.path.nameWithoutExtension
import kotlin.system.exitProcess
@Command(
name = "parseWvc",
description = ["Parse a .wvc file and output an antlr4 parse tree."],
mixinStandardHelpOptions = true,
version = ["0.1.0"]
)
open class ParseWvc : AbstractSourceTransformerCli() {
companion object {
@ -103,7 +110,6 @@ open class ParseWvc : AbstractSourceTransformerCli() {
if (interactive) {
println("Parsing $target")
}
var code = 0
while (true) {
try {
val input = CharStreams.fromPath(target)
@ -132,30 +138,26 @@ open class ParseWvc : AbstractSourceTransformerCli() {
if (lexerErrors.isNotEmpty()) {
val recover = this.onLexerErrors(lexerErrors)
if (!recover) {
code = 1
break
return 1
}
} else if (parserErrors.isNotEmpty()) {
val recover = this.onParserErrors(parserErrors)
if (!recover) {
code = 1
break
return 1
}
} else {
val redo = this.onSuccess(target, parser, cuContext)
if (!redo) {
break
return 0
}
}
} catch (e: Exception) {
val recover = this.onException(e)
if (!recover) {
code = 1
break
return 1
}
}
}
return code
}
}

View File

@ -87,7 +87,6 @@ open class TokenizeWvc : AbstractSourceTransformerCli() {
if (interactive) {
println("Tokenizing $target")
}
var code = 0
while (true) {
try {
val input = CharStreams.fromPath(target)
@ -104,24 +103,21 @@ open class TokenizeWvc : AbstractSourceTransformerCli() {
if (errors.isNotEmpty()) {
val recover = this.onErrors(errors)
if (!recover) {
code = 1
break
return 1
}
} else {
val redo = this.onSuccess(target, allTokens)
if (!redo) {
break
return 0
}
}
} catch (e: Exception) {
val recover = this.onException(e)
if (!recover) {
code = 1
break
return 1
}
}
}
return code
}
}