jessebrault/groowt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lots of refactoring lexer, parser, and tools.

Browse Source
This commit is contained in:
JesseBrault0709 2024-05-23 16:51:13 +02:00
parent bfa2ec6cd7
commit 58d7641ece
96 changed files with 1035 additions and 554 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
buildSrc/src/main/java/groowt/gradle/antlr/GroowtAntlrPlugin.java
View File

@ -3,6 +3,7 @@ package groowt.gradle.antlr;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.plugins.JavaPluginExtension;
import org.gradle.api.tasks.Delete;
import java.io.File;
@ -39,7 +40,7 @@ public final class GroowtAntlrPlugin implements Plugin<Project> {
sourceSet.getJava().srcDir(baseOutputDir);
final var sourceSetTasks = antlrSourceDirectorySet.getFiles().stream()
final var generateTasks = antlrSourceDirectorySet.getFiles().stream()
.filter(GroowtAntlrUtil::isAntlrFile)
.map(file -> {
final var taskProvider = project.getTasks().register(
@ -64,10 +65,21 @@ public final class GroowtAntlrPlugin implements Plugin<Project> {
})
.toList();
if (!sourceSetTasks.isEmpty()) {
if (!generateTasks.isEmpty()) {
final var cleanAntlr = project.getTasks().register(
sourceSet.getTaskName("clean", "Antlr"),
Delete.class,
deleteTask -> {
deleteTask.setGroup(GROOWT_ANTLR);
deleteTask.delete(baseOutputDir);
}
);
project.getTasks().register(sourceSet.getTaskName("generate", "AllAntlr"), task -> {
task.dependsOn(sourceSetTasks);
task.setGroup(GROOWT_ANTLR);
//noinspection ResultOfMethodCallIgnored
task.doFirst(first -> baseOutputDir.get().getAsFile().delete());
task.dependsOn(generateTasks);
});
}
});

11
web-view-components-compiler/build.gradle
View File

@ -152,14 +152,15 @@ def toolSpec = { String name, String mainClass ->
}
final List<ToolSpec> toolSpecs = [
toolSpec('astFileMaker', 'AstFileMakerCli'),
toolSpec('astFileMaker', 'AstFileMakerCli'), // deprecated
toolSpec('convertToGroovy', 'ConvertToGroovy'),
toolSpec('groovyWvc', 'GroovyWvcCompiler'),
toolSpec('lexer', 'LexerTool'),
toolSpec('parser', 'ParserTool'),
toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'),
toolSpec('lexer', 'LexerTool'), // deprecated
toolSpec('parser', 'ParserTool'), // deprecated
toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'), // deprecated
toolSpec('parseWvc', 'ParseWvc'),
toolSpec('tokenizeWvc', 'TokenizeWvc'),
toolSpec('tokensFileMaker', 'TokensFileMakerCli')
toolSpec('tokensFileMaker', 'TokensFileMakerCli') // deprecated
]
toolSpecs.each { spec ->

10
web-view-components-compiler/makeLexerTest Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
ARGS="-v -d src/test/lexer/tokens-files -s _tokens -e .txt"
if [ "$1" == "--debug" ]; then
shift
bin/tokenizeWvc --debug $ARGS "$@"
else
bin/tokenizeWvc $ARGS "$@"
fi

10
web-view-components-compiler/makeParserTest Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
ARGS="-v -d src/test/parser/parse-tree-files -s _parseTree -e .txt"
if [ "$1" == "--debug" ]; then
shift
bin/parseWvc --debug $ARGS "$@"
else
bin/parseWvc $ARGS "$@"
fi

110
web-view-components-compiler/src/main/antlr/WebViewComponentsLexerBase.g4
View File

@ -8,12 +8,13 @@ options {
tokens {
PreambleBreak,
ComponentNlws,
GroovyCode,
GStringAttrValueEnd,
JStringAttrValueEnd,
ClosureAttrValueEnd,
DollarScriptletClose
DollarScriptletClose,
Nlws,
ErrorChar
}
channels {
@ -88,14 +89,12 @@ channels {
DollarSlashyStringClosureStart
);
public WebViewComponentsLexerBase() {
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
}
public WebViewComponentsLexerBase() {}
private void onPreambleClose() {
this.setType(PreambleBreak);
this.exitPreamble();
this.popMode();
this.mode(MAIN);
}
private void onGStringClosure() {
@ -116,10 +115,21 @@ channels {
// DEFAULT_MODE
PreambleOpen
: THREE_DASH ( NL | WS+ )? { this.canPreamble() }? { this.enterPreamble(); }
: THREE_DASH ( NL | WS+ )? { this.enterPreamble(); }
-> type(PreambleBreak), pushMode(GROOVY_CODE)
;
NotPreambleOpen
: .
{
this.rollbackOne();
this.setCanPreamble(false);
} -> skip, mode(MAIN)
;
// ----------------------------------------
mode MAIN;
ComponentOpen
: LT { !isAnyOf(this.getNextChar(), '/', '>') }? -> pushMode(TAG_START)
;
@ -156,25 +166,39 @@ DollarScriptletOpen
;
DollarReferenceStart
: DOLLAR { isIdentifierStartChar(this.getNextChar()) }? -> pushMode(IN_G_STRING_PATH)
: DOLLAR { isGStringIdentifierStartChar(this.getNextChar()) }? -> pushMode(IN_G_STRING_PATH)
;
QuestionTag
: LT QUESTION .*? QUESTION GT
QuestionTagOpen
: LT QUESTION
;
HtmlComment
: LT BANG TWO_DASH .*? TWO_DASH GT
QuestionTagClose
: QUESTION GT
;
HtmlCommentOpen
: LT BANG TWO_DASH
;
HtmlCommentClose
: TWO_DASH GT
;
RawText
: ( ~[-<$] // n.b.: LT cannot be escaped, only via &lt;
| MINUS { !this.isNext("--") }?
| DOLLAR { !this.isNext('{') && !isIdentifierStartChar(this.getNextChar()) }?
: ( ~[-<$?]
| MINUS { !this.isNext("->") }?
| LT { canFollowLessThan(this.getNextCharAsString()) }?
| LT BANG { !this.isNext("--") }?
| DOLLAR { !(this.isNext('{') || isIdentifierStartChar(this.getNextChar())) }?
| QUESTION { !this.isNext('>') }?
)+
;
MainError
: . -> type(ErrorChar), channel(ERROR)
;
// ----------------------------------------
mode TAG_START;
@ -227,11 +251,11 @@ StringIdentifierChar
;
TagStartNlws
: NLWS+ -> type(ComponentNlws), channel(HIDDEN)
: NLWS+ -> type(Nlws), channel(HIDDEN)
;
TagStartError
: . -> channel(ERROR)
: . -> type(ErrorChar), channel(ERROR)
;
// ----------------------------------------
@ -276,7 +300,7 @@ JStringAttrValueStart
;
ClosureAttrValueStart
: LEFT_CURLY InTagNlws? { !this.isNext('<') }?
: LEFT_CURLY { !this.isNextIgnoreNlws('<') }?
{
this.curlies.push(() -> {
this.setType(ClosureAttrValueEnd);
@ -296,7 +320,7 @@ ComponentAttrValueEnd
;
InTagNlws
: NLWS+ -> type(ComponentNlws), channel(HIDDEN)
: NLWS+ -> type(Nlws), channel(HIDDEN)
;
TagError
@ -307,10 +331,7 @@ TagError
mode GROOVY_CODE;
PreambleClose
: NL? THREE_DASH ( NL | WS+ )? { this.inPreamble() }?
{
this.onPreambleClose();
}
: THREE_DASH { this.inPreamble() && this.getCharPositionInLine() == 3 }? { this.onPreambleClose(); }
;
ScriptletClose
@ -318,10 +339,9 @@ ScriptletClose
;
GroovyCodeChars
: ( ~[/\n\r\-$%(){}'"]
| FS { !isAnyOf(this.getNextChar(), '/', '*') }?
| NL { !this.inPreamble() || !this.isNext("---") }?
: ( ~[-/$%(){}'"]
| MINUS { !(this.getCharPositionInLine() == 1 && this.isNext("--")) }?
| FS { !isAnyOf(this.getNextChar(), '/', '*') }?
| DOLLAR { !this.isNext('/') }?
| PERCENT { !this.isNext('>') }?
)+
@ -418,7 +438,7 @@ LineCommentEnd
mode IN_STAR_COMMENT;
StarCommentChars
: ~'*'+
: ~'*' | ( '*' { !this.isNext('/') }? )
;
StarCommentEnd
@ -449,11 +469,12 @@ GStringText
: ( ~[\n\r"$]
| BS DQ
| BS DOLLAR
| DOLLAR { !isGStringIdentifierStartChar(this.getNextChar()) }?
)+
;
GStringDollarValueStart
: DOLLAR { !this.isNext('{') }? -> pushMode(IN_G_STRING_PATH)
: DOLLAR { !this.isNext('{') && isGStringIdentifierStartChar(this.getNextChar()) }? -> pushMode(IN_G_STRING_PATH)
;
GStringClosureStart
@ -521,13 +542,14 @@ GStringPathEnd
yield new StringContinueEndSpec();
}
}
case DEFAULT_MODE -> new StringContinueEndSpec();
case MAIN -> new StringContinueEndSpec();
default -> throw new IllegalStateException("not a valid gString context: " + this.getModeName(this.peekMode(1)));
};
switch (endSpec) {
case StringContinueEndSpec ignored -> {
this.popMode();
this.rollbackOne();
this.rollbackOne(true);
this.skip();
}
case StringClosingEndSpec closingEndSpec -> {
this.setType(closingEndSpec.type());
@ -550,21 +572,21 @@ GStringIdentifierChar
// ----------------------------------------
mode IN_TRIPLE_J_STRING;
// TODO: check for unescaped SQ, I think groovy allows them
TripleJStringContent
: ( ~['] | BS SQ )+
: ( ~[']
| SQ { !(this.isNext("''") && this._input.LA(3) != '\'') }?
)+
;
TripleJStringEnd
: SQ SQ SQ -> popMode
: SQ SQ SQ { !this.isNext('\'') }? -> popMode
;
// ----------------------------------------
mode IN_TRIPLE_G_STRING;
// TODO: check for unescaped DQ, I think groovy allows them
TripleGStringDollarValueStart
: DOLLAR { !this.isNext('{') }? -> pushMode(IN_G_STRING_PATH)
: DOLLAR { isGStringIdentifierStartChar(this.getNextChar()) }? -> pushMode(IN_G_STRING_PATH)
;
TripleGStringClosureStart
@ -572,11 +594,15 @@ TripleGStringClosureStart
;
TripleGStringText
: ( ~["$] | BS DQ | BS DOLLAR )+
: ( ~["$]
| DQ { !(this.isNext("\"\"") && this._input.LA(3) != '"') }?
| BS DOLLAR
| DOLLAR { !isGStringIdentifierStartChar(this.getNextChar()) }?
)+
;
TripleGStringEnd
: DQ DQ DQ -> popMode
: DQ DQ DQ { !this.isNext('"') }? -> popMode
;
// ----------------------------------------
@ -587,8 +613,9 @@ ParenthesesSlashyStringText
;
ParenthesesSlashyStringDollarValueStart
: DOLLAR { !this.isNext('{') }? -> pushMode(IN_G_STRING_PATH)
: DOLLAR { isGStringIdentifierStartChar(this.getNextChar()) }? -> pushMode(IN_G_STRING_PATH)
;
ParenthesesSlashyStringClosureStart
: DOLLAR LEFT_CURLY { this.onGStringClosure(); }
;
@ -603,13 +630,14 @@ mode IN_DOLLAR_SLASHY_STRING;
DollarSlashyStringText
: ( ~[$/]
| DOLLAR DOLLAR
| DOLLAR FS
| FS DOLLAR DOLLAR
| DOLLAR { !isGStringIdentifierStartChar(this.getNextChar()) }?
| FS { !this.isNext('$') }?
)+
;
DollarSlashyStringDollarValueStart
: DOLLAR { !isAnyOf(this.getNextChar(), '/', '$', '{') }? -> pushMode(IN_G_STRING_PATH)
: DOLLAR { isGStringIdentifierStartChar(this.getNextChar()) }?
-> pushMode(IN_G_STRING_PATH)
;
DollarSlashyStringClosureStart

16
web-view-components-compiler/src/main/antlr/WebViewComponentsParser.g4
View File

@ -17,18 +17,22 @@ body
;
bodyText
: gStringBodyText | jStringBodyText
: ( questionTag | htmlComment | text | bodyTextGroovyElement )+
;
gStringBodyText
: jStringBodyText? ( gStringBodyTextGroovyElement jStringBodyText? )+
questionTag
: QuestionTagOpen ( text | bodyTextGroovyElement )* QuestionTagClose
;
jStringBodyText
: ( QuestionTag | HtmlComment | RawText )+
htmlComment
: HtmlCommentOpen ( text | bodyTextGroovyElement )* HtmlCommentClose
;
gStringBodyTextGroovyElement
text
: RawText
;
bodyTextGroovyElement
: plainScriptlet | equalsScriptlet | dollarScriptlet | dollarReference
;

49
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/AbstractWebViewComponentsLexer.java
View File

@ -1,5 +1,6 @@
package groowt.view.component.web.antlr;
import groovyjarjarantlr4.runtime.Token;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.atn.ATN;
@ -15,6 +16,7 @@ import java.util.LinkedList;
import java.util.function.Function;
import java.util.stream.Collectors;
import static groowt.view.component.web.antlr.LexerSemanticPredicates.isAnyOf;
import static groowt.view.component.web.antlr.TokenUtil.escapeChars;
public abstract class AbstractWebViewComponentsLexer extends Lexer {
@ -30,11 +32,13 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
super(recognizer, atn, decisionToDFA, sharedContextCache);
}
public void resetAcceptPosition(CharStream input, int index, int line, int charPositionInLine) {
public void resetAcceptPosition(CharStream input, int index, int line, int charPositionInLine, boolean consume) {
input.seek(index);
this.line = line;
this.charPositionInLine = charPositionInLine;
this.consume(input);
if (consume) {
this.consume(input);
}
}
}
@ -87,7 +91,7 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
return this.logger;
}
public boolean isCanPreamble() {
public boolean canPreamble() {
return this.canPreamble;
}
@ -95,7 +99,7 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
this.canPreamble = canPreamble;
}
public boolean isInPreamble() {
public boolean inPreamble() {
return this.inPreamble;
}
@ -103,7 +107,7 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
this.inPreamble = inPreamble;
}
public boolean isInConstructor() {
public boolean inConstructor() {
return this.inConstructor;
}
@ -176,19 +180,11 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
}
}
protected boolean canPreamble() {
return this.canPreamble;
}
protected void enterPreamble() {
this.inPreamble = true;
this.canPreamble = false;
}
protected boolean inPreamble() {
return this.inPreamble;
}
protected void exitPreamble() {
this.inPreamble = false;
}
@ -199,10 +195,6 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
this.inConstructor = true;
}
protected boolean inConstructor() {
return this.inConstructor;
}
protected boolean canExitConstructor() {
return this.inConstructor && this.parentheses.getStackSize() == 1 && this.parentheses.isLast();
}
@ -220,6 +212,10 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
return b.toString();
}
protected String getNextCharAsString() {
return Character.toString((char) this.getNextChar());
}
protected int getCurrentChar() {
return this._input.LA(-1);
}
@ -236,6 +232,16 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
return this.getNextCharsAsString(test.length()).equals(test);
}
protected boolean isNextIgnoreNlws(char test) {
for (int i = 1; this._input.LA(i) != Token.EOF; i++) {
final char subject = (char) this._input.LA(i);
if (!isAnyOf(subject, ' ', '\t', '\n', '\r')) {
return subject == test;
}
}
return false;
}
@Override
public final LexerATNSimulator getInterpreter() {
return this._interp;
@ -244,11 +250,16 @@ public abstract class AbstractWebViewComponentsLexer extends Lexer {
protected abstract PositionAdjustingLexerATNSimulator getPositionAdjustingInterpreter();
protected void rollbackOne() {
this.rollbackOne(false);
}
protected void rollbackOne(boolean consume) {
this.getPositionAdjustingInterpreter().resetAcceptPosition(
this._input,
this._tokenStartCharIndex - 1,
Math.max(this._tokenStartCharIndex - 1, 0),
this._tokenStartLine,
this._tokenStartCharIndex - 1
Math.max(this._tokenStartCharIndex - 1, 0),
consume
);
}

8
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/LexerSemanticPredicates.kt
View File

@ -4,6 +4,7 @@ package groowt.view.component.web.antlr
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.lang.invoke.MethodHandles
import java.util.regex.Pattern
private val logger: Logger = LoggerFactory.getLogger(
MethodHandles.lookup().lookupClass()
@ -66,6 +67,13 @@ fun canFollowGStringOpening(nextTwo: String): Boolean {
return result
}
private val notAllowedAfterLessThan = Pattern.compile("[\\p{L}%/>]")
fun canFollowLessThan(subject: String): Boolean {
val m = notAllowedAfterLessThan.matcher(subject)
return !m.matches()
}
fun isIdentifierStartChar(c: Char): Boolean = Character.isJavaIdentifierStart(c)
fun isIdentifierStartChar(subject: Int) = isIdentifierStartChar(subject.toChar())

66
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/ParserError.kt
View File

@ -5,25 +5,73 @@ import groowt.view.component.web.util.excerpt
import org.antlr.v4.runtime.ParserRuleContext
import org.antlr.v4.runtime.Token
open class ParserError(val type: ParserErrorType, val offending: Token, val context: ParserRuleContext)
open class ParserError {
val type: ParserErrorType
val offendingToken: Token?
val offendingSymbol: Any?
val msg: String?
val sourcePosition: SourcePosition
val context: ParserRuleContext
constructor(
type: ParserErrorType,
offendingToken: Token?,
sourcePosition: SourcePosition,
context: ParserRuleContext
) {
this.type = type
this.offendingToken = offendingToken
this.offendingSymbol = offendingToken
this.msg = null;
this.sourcePosition = sourcePosition
this.context = context
}
constructor(
type: ParserErrorType,
offendingSymbol: Any?,
msg: String,
sourcePosition: SourcePosition,
context: ParserRuleContext
) {
this.type = type
this.offendingToken = null
this.offendingSymbol = offendingSymbol
this.msg = msg
this.sourcePosition = sourcePosition
this.context = context
}
}
class MismatchedInputParserError(
type: ParserErrorType,
offending: Token,
offendingToken: Token,
sourcePosition: SourcePosition,
context: ParserRuleContext,
val expectedTokenTypes: Set<Int>
) : ParserError(type, offending, context)
) : ParserError(type, offendingToken, sourcePosition, context)
fun formatParserError(error: ParserError): String {
val sb = StringBuilder()
val sourcePosition = SourcePosition.fromStartOfToken(error.offending)
sb.append("At ")
.append(sourcePosition.toStringLong())
.append(error.sourcePosition.toStringLong())
.append(": ")
.append(error.type.message)
.append(" Offending token: ")
.append(formatTokenForError(error.offending))
.append(". ")
if (error.type != ParserErrorType.UNKNOWN) {
sb.append(error.type.message)
} else if (error.msg != null) {
sb.append(error.msg + ".")
} else {
sb.append("Parser unknown error.")
}
if (error.offendingToken != null) {
sb.append(" Offending token: ")
.append(formatTokenForError(error.offendingToken))
.append(". ")
} else if (error.offendingSymbol != null) {
sb.append(" Offending symbol: ${error.offendingSymbol}. ")
}
if (error is MismatchedInputParserError) {
sb.append("Expected any of: ")
.append(formatExpected(error.expectedTokenTypes))

27
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/ParserErrorListener.kt
View File

@ -18,7 +18,7 @@ class ParserErrorListener : BaseErrorListener() {
line: Int,
charPositionInLine: Int,
msg: String,
e: RecognitionException
e: RecognitionException?
) {
val parser = recognizer as WebViewComponentsParser
when (e) {
@ -30,20 +30,41 @@ class ParserErrorListener : BaseErrorListener() {
this.lexerErrors.add(error)
}
is NoViableAltException -> {
val error = ParserError(ParserErrorType.NO_VIABLE_ALTERNATIVE, e.offendingToken, parser.context)
val error = ParserError(
ParserErrorType.NO_VIABLE_ALTERNATIVE,
e.offendingToken,
SourcePosition.fromStartOfToken(e.offendingToken),
parser.context
)
parserErrors.add(error)
}
is InputMismatchException -> {
val error = MismatchedInputParserError(
ParserErrorType.INPUT_MISMATCH,
e.offendingToken,
SourcePosition.fromStartOfToken(e.offendingToken),
parser.context,
e.expectedTokens.toSet()
)
parserErrors.add(error)
}
is FailedPredicateException -> {
val error = ParserError(ParserErrorType.FAILED_PREDICATE, e.offendingToken, parser.context)
val error = ParserError(
ParserErrorType.FAILED_PREDICATE,
e.offendingToken,
SourcePosition.fromStartOfToken(e.offendingToken),
parser.context
)
parserErrors.add(error)
}
else -> {
val error = ParserError(
ParserErrorType.UNKNOWN,
offendingSymbol,
msg,
SourcePosition(line, charPositionInLine + 1),
parser.context
)
parserErrors.add(error)
}
}

3
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/ParserErrorType.kt
View File

@ -3,5 +3,6 @@ package groowt.view.component.web.antlr
enum class ParserErrorType(val message: String) {
NO_VIABLE_ALTERNATIVE("Parser has no viable alternative."),
INPUT_MISMATCH("Parser input mismatch."),
FAILED_PREDICATE("Parser input failed predicate.")
FAILED_PREDICATE("Parser input failed predicate."),
UNKNOWN("Parser unknown error.")
}

15
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/ParserUtil.kt
View File

@ -2,6 +2,7 @@
package groowt.view.component.web.antlr
import groowt.view.component.web.antlr.WebViewComponentsParser.CompilationUnitContext
import groowt.view.component.web.util.SourcePosition
import org.antlr.v4.runtime.*
import org.antlr.v4.runtime.tree.ErrorNode
import org.antlr.v4.runtime.tree.ParseTree
@ -85,7 +86,7 @@ private fun formatForError(parser: Parser, tree: ParseTree): String {
}
fun formatTree(parser: Parser, tree: Tree, colors: Boolean, consumer: Consumer<String>): Unit =
consumer.accept(formatTree(parser, tree, colors).toString())
consumer.accept(formatTree(parser, tree, colors))
fun formatTree(parser: Parser, tree: Tree, colors: Boolean = true): String =
doFormatTree(parser, tree, colors, 0, " ", StringBuilder()).toString()
@ -109,12 +110,10 @@ private fun formatBasicInfo(parser: Parser, tree: Tree, sb: StringBuilder) {
when (tree) {
is ParserRuleContext -> {
sb.append(parser.ruleNames[tree.ruleIndex])
.append(
"[${tree.start.line},${tree.start.charPositionInLine + 1}.."
+ "${tree.stop.line},${tree.stop.charPositionInLine + 1}]"
)
val start = SourcePosition.fromStartOfToken(tree.start)
val end = SourcePosition.fromEndOfToken(tree.stop)
sb.append("[${start.line},${start.column}..${end.line},${end.column}]")
}
is TerminalNode -> {
sb.append(parser.vocabulary.getDisplayName(tree.symbol.type))
.append("[${tree.symbol.line},${tree.symbol.charPositionInLine + 1}]")
@ -139,14 +138,14 @@ private fun doFormatTree(
sb.append(ansi().fgRed())
}
}
formatBasicInfo (parser, tree, sb)
formatBasicInfo(parser, tree, sb)
if (e != null) {
sb.append(": Exception: ${e.javaClass.simpleName}(${escapeChars(tree.text)})")
if (colors) {
sb.append(ansi().reset())
}
}
sb . append ("\n")
sb.append ("\n")
var i = 0
while (i < tree.childCount) {
doFormatTree(parser, tree.getChild(i), colors, indentTimes + 1, indentText, sb)

1
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsLexer.java
View File

@ -11,6 +11,7 @@ public class WebViewComponentsLexer extends WebViewComponentsLexerBase {
public WebViewComponentsLexer() {
super();
this._interp = new PositionAdjustingLexerATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache);
}
@Override

2
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt
View File

@ -14,7 +14,7 @@ class WebViewComponentsTokenStream(
constructor(lexer: WebViewComponentsLexer) : this(
lexer,
setOf(WebViewComponentsLexer.HIDDEN, WebViewComponentsLexer.ERROR)
setOf(WebViewComponentsLexer.HIDDEN)
)
private val tokens: MutableList<Token> = ArrayList()

81
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/DefaultAstBuilderVisitor.java
View File

@ -4,6 +4,7 @@ import groowt.view.component.web.WebViewComponentBugError;
import groowt.view.component.web.antlr.MergedGroovyCodeToken;
import groowt.view.component.web.antlr.TokenUtil;
import groowt.view.component.web.antlr.WebViewComponentsParser;
import groowt.view.component.web.antlr.WebViewComponentsParser.BodyTextContext;
import groowt.view.component.web.antlr.WebViewComponentsParserBaseVisitor;
import groowt.view.component.web.ast.node.*;
import groowt.view.component.web.util.TokenRange;
@ -18,7 +19,6 @@ import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.regex.Pattern;
public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor<Node> {
@ -73,18 +73,10 @@ public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor
return ctx.getChild(0).accept(this);
}
protected <R extends Node> R getSingleChildAs(ParserRuleContext ctx, Class<R> type) {
return type.cast(this.getSingleChild(ctx));
}
protected TokenRange getTokenRange(ParserRuleContext ctx) {
return TokenRange.of(ctx.start, ctx.stop);
}
protected TerminalNode getSingleChildTerminalNode(WebViewComponentsParser.JStringBodyTextContext ctx) {
return ctx.getChild(TerminalNode.class, 0);
}
@Override
public Node visitCompilationUnit(WebViewComponentsParser.CompilationUnitContext ctx) {
final PreambleNode preamble = this.getSingleAs(ctx.preamble(), PreambleNode.class);
@ -119,37 +111,57 @@ public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor
}
@Override
public Node visitBodyText(WebViewComponentsParser.BodyTextContext ctx) {
return this.getSingleChild(ctx);
}
@Override
public Node visitGStringBodyText(WebViewComponentsParser.GStringBodyTextContext ctx) {
final List<Node> children = new ArrayList<>();
public @Nullable Node visitBodyText(BodyTextContext ctx) {
final List<BodyTextChild> children = new ArrayList<>();
for (final var child : ctx.children) {
final @Nullable Node childResult = child.accept(this);
if (childResult != null) {
children.add(childResult);
children.add((BodyTextChild) childResult);
}
}
return this.nodeFactory.gStringBodyTextNode(this.getTokenRange(ctx), children);
if (children.isEmpty()) {
return null;
} else {
return this.nodeFactory.bodyTextNode(this.getTokenRange(ctx), children);
}
}
@Override
public @Nullable Node visitJStringBodyText(WebViewComponentsParser.JStringBodyTextContext ctx) {
final String text = ctx.getText();
if (isNotBlankNotEmpty(text)) {
return this.nodeFactory.jStringBodyTextNode(
this.getTokenRange(ctx),
text
);
public @Nullable Node visitQuestionTag(WebViewComponentsParser.QuestionTagContext ctx) {
final List<QuestionTagChild> children = new ArrayList<>();
for (final var child : ctx.children) {
final @Nullable Node childResult = child.accept(this);
if (childResult != null) {
children.add((QuestionTagChild) childResult);
}
}
return this.nodeFactory.questionTagNode(this.getTokenRange(ctx), children);
}
@Override
public Node visitHtmlComment(WebViewComponentsParser.HtmlCommentContext ctx) {
final List<HtmlCommentChild> children = new ArrayList<>();
for (final var child : ctx.children) {
final @Nullable Node childResult = child.accept(this);
if (childResult != null) {
children.add((HtmlCommentChild) childResult);
}
}
return this.nodeFactory.htmlCommentNode(this.getTokenRange(ctx), children);
}
@Override
public @Nullable Node visitText(WebViewComponentsParser.TextContext ctx) {
final String content = ctx.getText();
if (isNotBlankNotEmpty(content)) {
return this.nodeFactory.textNode(this.getTokenRange(ctx), content);
} else {
return null;
}
}
@Override
public Node visitGStringBodyTextGroovyElement(WebViewComponentsParser.GStringBodyTextGroovyElementContext ctx) {
public Node visitBodyTextGroovyElement(WebViewComponentsParser.BodyTextGroovyElementContext ctx) {
return this.getSingleChild(ctx);
}
@ -208,16 +220,6 @@ public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor
return this.nodeFactory.componentArgsNode(this.getTokenRange(ctx), typeNode, constructorNode, attrNodes);
}
private static final Pattern lowercaseLetterPattern = Pattern.compile("\\p{Ll}");
protected boolean startsWithLowercaseLetter(String subject) {
if (subject.isEmpty()) {
throw new IllegalArgumentException(
"Cannot test for starting lowercase letter when the subject length is 0; given: " + subject);
}
return lowercaseLetterPattern.matcher(subject.substring(0, 1)).matches();
}
@Override
public Node visitComponentType(WebViewComponentsParser.ComponentTypeContext ctx) {
final var typedIdentifier = ctx.TypedIdentifier();
@ -291,7 +293,6 @@ public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor
if (groovyCode != null) {
final MergedGroovyCodeToken groovyCodeToken = (MergedGroovyCodeToken) groovyCode.getSymbol();
if (canBeGString(groovyCodeToken.getOriginals())) {
// TODO: we need to set the appropriate type: slashy, dollar slashy, etc.
return this.nodeFactory.gStringValueNode(ctxTokenRange, groovyCodeToken.getTokenIndex());
} else {
return this.nodeFactory.jStringValueNode(ctxTokenRange, groovyCode.getText());
@ -337,7 +338,7 @@ public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor
public @Nullable Node visitEqualsScriptlet(WebViewComponentsParser.EqualsScriptletContext ctx) {
final TerminalNode groovyCode = ctx.GroovyCode();
if (groovyCode != null) {
return this.nodeFactory.dollarScriptletNode(
return this.nodeFactory.equalsScriptletNode(
this.getTokenRange(ctx),
ctx.GroovyCode().getSymbol().getTokenIndex()
);
@ -384,12 +385,12 @@ public class DefaultAstBuilderVisitor extends WebViewComponentsParserBaseVisitor
@Override
public Node visitTerminal(TerminalNode node) {
throw new UnsupportedOperationException();
throw new WebViewComponentBugError("Should not be visiting terminal nodes.");
}
@Override
public Node visitErrorNode(ErrorNode node) {
throw new IllegalStateException("Found an ErrorNode: " + node);
throw new WebViewComponentBugError("Should not have found an ErrorNode by this point: " + node);
}
}

29
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/DefaultNodeFactory.java
View File

@ -22,8 +22,10 @@ public class DefaultNodeFactory implements NodeFactory {
CompilationUnitNode.class,
PreambleNode.class,
BodyNode.class,
GStringBodyTextNode.class,
JStringBodyTextNode.class,
BodyTextNode.class,
QuestionNode.class,
HtmlCommentNode.class,
TextNode.class,
TypedComponentNode.class,
FragmentComponentNode.class,
ComponentArgsNode.class,
@ -102,13 +104,23 @@ public class DefaultNodeFactory implements NodeFactory {
}
@Override
public GStringBodyTextNode gStringBodyTextNode(TokenRange tokenRange, List<? extends Node> children) {
return this.objectFactory.get(GStringBodyTextNode.class, tokenRange, children);
public BodyTextNode bodyTextNode(TokenRange tokenRange, List<? extends BodyTextChild> children) {
return this.objectFactory.get(BodyTextNode.class, tokenRange, children);
}
@Override
public JStringBodyTextNode jStringBodyTextNode(TokenRange tokenRange, String content) {
return this.objectFactory.get(JStringBodyTextNode.class, tokenRange, content);
public QuestionNode questionTagNode(TokenRange tokenRange, List<? extends QuestionTagChild> children) {
return this.objectFactory.get(QuestionNode.class, tokenRange, children);
}
@Override
public HtmlCommentNode htmlCommentNode(TokenRange tokenRange, List<? extends HtmlCommentChild> children) {
return this.objectFactory.get(HtmlCommentNode.class, tokenRange, children);
}
@Override
public TextNode textNode(TokenRange tokenRange, String content) {
return this.objectFactory.get(TextNode.class, tokenRange, content);
}
@Override
@ -196,6 +208,11 @@ public class DefaultNodeFactory implements NodeFactory {
return this.objectFactory.get(ComponentValueNode.class, tokenRange, componentNode);
}
@Override
public EqualsScriptletNode equalsScriptletNode(TokenRange tokenRange, int groovyIndex) {
return this.objectFactory.get(EqualsScriptletNode.class, tokenRange, groovyIndex);
}
@Override
public PlainScriptletNode plainScriptletNode(TokenRange tokenRange, int groovyIndex) {
return this.objectFactory.get(PlainScriptletNode.class, tokenRange, groovyIndex);

10
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/NodeFactory.java
View File

@ -18,9 +18,13 @@ public interface NodeFactory {
BodyNode bodyNode(TokenRange tokenRange, List<? extends BodyChildNode> children);
GStringBodyTextNode gStringBodyTextNode(TokenRange tokenRange, List<? extends Node> children);
BodyTextNode bodyTextNode(TokenRange tokenRange, List<? extends BodyTextChild> children);
JStringBodyTextNode jStringBodyTextNode(TokenRange tokenRange, String content);
QuestionNode questionTagNode(TokenRange tokenRange, List<? extends QuestionTagChild> children);
HtmlCommentNode htmlCommentNode(TokenRange tokenRange, List<? extends HtmlCommentChild> children);
TextNode textNode(TokenRange tokenRange, String content);
TypedComponentNode typedComponentNode(
TokenRange tokenRange,
@ -59,6 +63,8 @@ public interface NodeFactory {
ComponentValueNode componentValueNode(TokenRange tokenRange, ComponentNode componentNode);
EqualsScriptletNode equalsScriptletNode(TokenRange tokenRange, int groovyIndex);
PlainScriptletNode plainScriptletNode(TokenRange tokenRange, int groovyIndex);
DollarScriptletNode dollarScriptletNode(TokenRange tokenRange, int groovyIndex);

6
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/BodyNode.java
View File

@ -21,7 +21,11 @@ public class BodyNode extends AbstractTreeNode {
}
@Inject
public BodyNode(NodeExtensionContainer extensionContainer, @Given TokenRange tokenRange, @Given List<? extends BodyChildNode> children) {
public BodyNode(
NodeExtensionContainer extensionContainer,
@Given TokenRange tokenRange,
@Given List<? extends BodyChildNode> children
) {
super(tokenRange, extensionContainer, childrenAsNodes(checkChildren(children)));
}

9
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/BodyTextChild.java Normal file
View File

@ -0,0 +1,9 @@
package groowt.view.component.web.ast.node;
public interface BodyTextChild {
default Node asNode() {
return (Node) this;
}
}

32
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/BodyTextNode.java Normal file
View File

@ -0,0 +1,32 @@
package groowt.view.component.web.ast.node;
import groowt.util.di.annotation.Given;
import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
import java.util.List;
public class BodyTextNode extends AbstractTreeNode implements BodyChildNode {
protected static List<? extends BodyTextChild> checkChildren(List<? extends BodyTextChild> children) {
if (children.isEmpty()) {
throw new IllegalArgumentException("A valid BodyTextNode must have at least one child BodyTextChildNode.");
}
return children;
}
protected static List<Node> childrenAsNodes(List<? extends BodyTextChild> children) {
return children.stream().map(BodyTextChild::asNode).toList();
}
@Inject
public BodyTextNode(
NodeExtensionContainer extensionContainer,
@Given TokenRange tokenRange,
@Given List<? extends BodyTextChild> children
) {
super(tokenRange, extensionContainer, childrenAsNodes(checkChildren(children)));
}
}

5
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/ComponentNode.java
View File

@ -6,9 +6,8 @@ import org.jetbrains.annotations.Nullable;
import java.util.List;
public sealed abstract class ComponentNode extends AbstractTreeNode implements BodyChildNode permits
FragmentComponentNode,
TypedComponentNode {
public sealed abstract class ComponentNode extends AbstractTreeNode implements BodyChildNode
permits FragmentComponentNode, TypedComponentNode {
private final BodyNode bodyNode;

2
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/DollarReferenceNode.java
View File

@ -7,7 +7,7 @@ import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
public class DollarReferenceNode extends AbstractLeafNode {
public class DollarReferenceNode extends AbstractLeafNode implements GroovyBodyNode {
private final int groovyTokenIndex;

2
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/DollarScriptletNode.java
View File

@ -7,7 +7,7 @@ import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
public class DollarScriptletNode extends AbstractLeafNode {
public class DollarScriptletNode extends AbstractLeafNode implements GroovyBodyNode {
private final GStringScriptletExtension gStringScriptlet;

50
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/EqualsScriptletNode.java Normal file
View File

@ -0,0 +1,50 @@
package groowt.view.component.web.ast.node;
import groowt.util.di.annotation.Given;
import groowt.view.component.web.antlr.TokenList;
import groowt.view.component.web.ast.extension.GroovyCodeNodeExtension;
import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import org.antlr.v4.runtime.Token;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
public class EqualsScriptletNode extends AbstractLeafNode implements GroovyBodyNode {
private final int groovyIndex;
private final GroovyCodeNodeExtension groovyCode;
public EqualsScriptletNode(
TokenList tokenList,
NodeExtensionContainer extensionContainer,
@Given TokenRange tokenRange,
@Given int groovyIndex
) {
super(tokenRange, extensionContainer);
this.groovyIndex = groovyIndex;
this.groovyCode = this.createGroovyCode(tokenList);
}
protected GroovyCodeNodeExtension createGroovyCode(TokenList tokenList) {
return this.createExtension(
GroovyCodeNodeExtension.class,
TokenRange.fromIndex(tokenList, this.groovyIndex),
(Function<? super List<Token>, String>) this::toValidGroovyCode
);
}
protected String toValidGroovyCode(List<Token> groovyTokens) {
return "{ -> " + groovyTokens.stream().map(Token::getText).collect(Collectors.joining()) + " }";
}
public int getGroovyIndex() {
return this.groovyIndex;
}
public GroovyCodeNodeExtension getGroovyCode() {
return this.groovyCode;
}
}

1
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/GStringBodyTextNode.java
View File

@ -8,6 +8,7 @@ import jakarta.inject.Inject;
import java.util.List;
@Deprecated
public class GStringBodyTextNode extends AbstractTreeNode implements BodyChildNode {
protected static List<? extends Node> checkChildren(List<? extends Node> children) {

10
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/GroovyBodyNode.java Normal file
View File

@ -0,0 +1,10 @@
package groowt.view.component.web.ast.node;
public interface GroovyBodyNode extends BodyTextChild, HtmlCommentChild, QuestionTagChild {
@Override
default Node asNode() {
return (Node) this;
}
}

9
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/HtmlCommentChild.java Normal file
View File

@ -0,0 +1,9 @@
package groowt.view.component.web.ast.node;
public interface HtmlCommentChild {
default Node asNode() {
return (Node) this;
}
}

21
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/HtmlCommentNode.java Normal file
View File

@ -0,0 +1,21 @@
package groowt.view.component.web.ast.node;
import groowt.util.di.annotation.Given;
import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
import java.util.List;
public class HtmlCommentNode extends AbstractTreeNode implements BodyTextChild {
@Inject
public HtmlCommentNode(
NodeExtensionContainer extensionContainer,
@Given TokenRange tokenRange,
@Given List<? extends HtmlCommentChild> children
) {
super(tokenRange, extensionContainer, children.stream().map(HtmlCommentChild::asNode).toList());
}
}

1
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/JStringBodyTextNode.java
View File

@ -5,6 +5,7 @@ import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
@Deprecated
public class JStringBodyTextNode extends AbstractLeafNode implements BodyChildNode {
private final String content;

4
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/PlainScriptletNode.java
View File

@ -12,7 +12,7 @@ import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
public class PlainScriptletNode extends AbstractLeafNode implements BodyChildNode {
public class PlainScriptletNode extends AbstractLeafNode implements GroovyBodyNode {
private final int groovyIndex;
private final GroovyCodeNodeExtension groovyCode;
@ -38,7 +38,7 @@ public class PlainScriptletNode extends AbstractLeafNode implements BodyChildNod
}
protected String toValidGroovyCode(List<Token> groovyTokens) {
return "{ Writer out ->\n" + groovyTokens.stream().map(Token::getText).collect(Collectors.joining()) + "\n}";
return "{ Writer out -> " + groovyTokens.stream().map(Token::getText).collect(Collectors.joining()) + " }";
}
public int getGroovyIndex() {

21
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/QuestionNode.java Normal file
View File

@ -0,0 +1,21 @@
package groowt.view.component.web.ast.node;
import groowt.util.di.annotation.Given;
import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
import java.util.List;
public class QuestionNode extends AbstractTreeNode implements BodyTextChild {
@Inject
public QuestionNode(
NodeExtensionContainer extensionContainer,
@Given TokenRange tokenRange,
@Given List<? extends QuestionTagChild> children
) {
super(tokenRange, extensionContainer, children.stream().map(QuestionTagChild::asNode).toList());
}
}

9
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/QuestionTagChild.java Normal file
View File

@ -0,0 +1,9 @@
package groowt.view.component.web.ast.node;
public interface QuestionTagChild {
default Node asNode() {
return (Node) this;
}
}

31
web-view-components-compiler/src/main/java/groowt/view/component/web/ast/node/TextNode.java Normal file
View File

@ -0,0 +1,31 @@
package groowt.view.component.web.ast.node;
import groowt.util.di.annotation.Given;
import groowt.view.component.web.ast.extension.NodeExtensionContainer;
import groowt.view.component.web.util.TokenRange;
import jakarta.inject.Inject;
public class TextNode extends AbstractLeafNode implements BodyTextChild, HtmlCommentChild, QuestionTagChild {
private final String content;
@Inject
public TextNode(
NodeExtensionContainer extensionContainer,
@Given TokenRange tokenRange,
@Given String content
) {
super(tokenRange, extensionContainer);
this.content = content;
}
@Override
public Node asNode() {
return this;
}
public String getContent() {
return this.content;
}
}

42
web-view-components-compiler/src/main/java/groowt/view/component/web/util/SourcePosition.java
View File

@ -20,30 +20,34 @@ public record SourcePosition(int line, int column) {
public static SourcePosition fromEndOfToken(Token token) {
final var text = token.getText();
int line = token.getLine();
int col = token.getCharPositionInLine() + 1;
int i = 0;
while (i < text.length()) {
final char c0 = text.charAt(i);
if (c0 == '\r') {
line++;
col = 1;
final char c1 = text.charAt(i + 1);
if (c1 == '\n') {
i += 2;
if (token.getType() == Token.EOF || text.length() == 1) {
return new SourcePosition(token.getLine(), token.getCharPositionInLine() + 1);
} else {
int line = token.getLine();
int col = token.getCharPositionInLine() + 1;
int i = 0;
while (i < text.length()) {
final char c0 = text.charAt(i);
if (c0 == '\r') {
line++;
col = 1;
final char c1 = text.charAt(i + 1);
if (c1 == '\n') {
i += 2;
} else {
i++;
}
} else if (c0 == '\n') {
line++;
col = 1;
i++;
} else {
col++;
i++;
}
} else if (c0 == '\n') {
line++;
col = 1;
i++;
} else {
col++;
i++;
}
return new SourcePosition(line, col);
}
return new SourcePosition(line, col);
}
public String toStringShort() {

69
web-view-components-compiler/src/test/ast/trees/complicated_ast.txt
View File

@ -1,69 +0,0 @@
CompilationUnitNode(1,1..21,6)
PreambleNode(1,1..6,1)
PreambleBreak[1,1](---\n)
GroovyCode[2,1](import some.Thing // a comment...World!')
PreambleBreak[4,31](\n---\n)
BodyNode(6,1..21,1)
JStringBodyTextNode(6,1..7,1)
RawText[6,1](<!DOCTYPE html>\n)
TypedComponentNode(7,1..20,8)
ComponentArgsNode(7,2..7,6)
StringComponentTypeNode(7,2..7,6)
StringIdentifier[7,2](html)
BodyNode(7,7..20,1)
TypedComponentNode(8,5..8,18)
ComponentArgsNode(8,6..8,10)
StringComponentTypeNode(8,6..8,10)
StringIdentifier[8,6](head)
TypedComponentNode(9,5..19,12)
ComponentArgsNode(9,6..9,10)
StringComponentTypeNode(9,6..9,10)
StringIdentifier[9,6](body)
BodyNode(9,11..19,5)
TypedComponentNode(10,9..10,29)
ComponentArgsNode(10,10..10,12)
StringComponentTypeNode(10,10..10,12)
StringIdentifier[10,10](h1)
BodyNode(10,13..10,24)
GStringBodyTextNode(10,13..10,24)
DollarScriptletNode(10,13..10,24)
DollarScriptletOpen[10,13](${)
GroovyCode[10,15](greeting)
DollarScriptletClose[10,23](})
TypedComponentNode(11,9..18,34)
ComponentArgsNode(11,10..11,32)
ClassComponentTypeNode(11,10..11,32)
TypedIdentifier[11,10](groowt.view.web.Select)
BodyNode(11,33..18,9)
TypedComponentNode(12,13..14,20)
ComponentArgsNode(12,14..12,36)
ClassComponentTypeNode(12,14..12,18)
TypedIdentifier[12,14](Case)
KeyValueAttrNode(12,19..12,36)
KeyNode(12,19..12,24)
AttributeIdentifier[12,19](cond)
Equals[12,23](=)
ClosureValueNode(12,24..12,36)
ClosureAttrValueStart[12,24]({)
GroovyCode[12,25](isItTrue())
ClosureAttrValueEnd[12,35](})
BodyNode(12,37..14,13)
TypedComponentNode(13,17..13,37)
ComponentArgsNode(13,18..13,19)
StringComponentTypeNode(13,18..13,19)
StringIdentifier[13,18](p)
BodyNode(13,20..13,33)
JStringBodyTextNode(13,20..13,33)
RawText[13,20](It's true! :))
TypedComponentNode(15,13..17,23)
ComponentArgsNode(15,14..15,21)
ClassComponentTypeNode(15,14..15,21)
TypedIdentifier[15,14](Default)
BodyNode(15,22..17,13)
TypedComponentNode(16,17..16,40)
ComponentArgsNode(16,18..16,19)
StringComponentTypeNode(16,18..16,19)
StringIdentifier[16,18](p)
BodyNode(16,20..16,36)
JStringBodyTextNode(16,20..16,36)
RawText[16,20](It's false... :()

10
web-view-components-compiler/src/test/ast/trees/helloTarget_ast.txt
View File

@ -1,10 +0,0 @@
CompilationUnitNode(1,1..2,6)
BodyNode(1,1..2,1)
GStringBodyTextNode(1,1..2,1)
JStringBodyTextNode(1,1..1,8)
RawText[1,1](Hello, )
DollarReferenceNode(1,8..1,15)
DollarReferenceStart[1,8]($)
GroovyCode[1,9](target)
JStringBodyTextNode(1,15..2,1)
RawText[1,15](!\n)

9
web-view-components-compiler/src/test/ast/trees/simpleComponentWithBody_ast.txt
View File

@ -1,9 +0,0 @@
CompilationUnitNode(1,1..2,6)
BodyNode(1,1..2,1)
TypedComponentNode(1,1..1,35)
ComponentArgsNode(1,2..1,10)
ClassComponentTypeNode(1,2..1,10)
TypedIdentifier[1,2](Greeting)
BodyNode(1,11..1,24)
JStringBodyTextNode(1,11..1,24)
RawText[1,11](to the world!)

5
web-view-components-compiler/src/test/groovy/groowt/view/component/web/antlr/WebViewComponentsTokenStreamTests.groovy
View File

@ -10,6 +10,7 @@ import org.junit.jupiter.api.Test
import static groowt.view.component.web.antlr.TokenUtil.getTokenName
import static groowt.view.component.web.antlr.WebViewComponentsLexer.GroovyCode
import static groowt.view.component.web.antlr.WebViewComponentsLexer.PreambleBreak
import static groowt.view.component.web.antlr.WebViewComponentsLexerBase.RawText
import static org.antlr.v4.runtime.Recognizer.EOF
import static org.junit.jupiter.api.Assertions.*
@ -50,11 +51,11 @@ class WebViewComponentsTokenStreamTests {
def lexer = new WebViewComponentsLexer(input)
def tokenStream = new WebViewComponentsTokenStream(lexer)
def tokens = tokenStream.allTokens
assertTypes([PreambleBreak, GroovyCode, PreambleBreak, EOF], tokens)
assertTypes([PreambleBreak, GroovyCode, PreambleBreak, RawText, EOF], tokens)
assertMergedGroovyCodeToken(tokens[1]) {
assertEquals('println \'Hello, World!\' // comment\n', it.text)
}
assertIterableEquals(0..3, tokens*.tokenIndex)
assertIterableEquals(0..4, tokens*.tokenIndex)
}
}

48
web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java
View File

@ -7,57 +7,15 @@ import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.Token;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static groowt.view.component.web.antlr.WebViewComponentsLexer.*;
import static org.junit.jupiter.api.Assertions.*;
public class WebViewComponentsLexerTests {
private static void assertTokenType(int type, Token token) {
assertEquals(
type,
token.getType(),
() -> "Expected " + VOCABULARY.getDisplayName(type)
+ " but got " + VOCABULARY.getDisplayName(token.getType())
);
}
@Test
public void helloTarget() {
final var input = CharStreams.fromString("Hello, $target!");
final var lexer = new WebViewComponentsLexer(input);
final var tokenStream = new WebViewComponentsTokenStream(lexer);
final var allTokens = tokenStream.getAllTokens();
assertEquals(5, allTokens.size(), () -> {
return "Wrong number of tokens; tokens: " + allTokens.stream()
.map(Token::toString)
.collect(Collectors.joining(", "));
});
final var t0 = allTokens.get(0);
final var t1 = allTokens.get(1);
final var t2 = allTokens.get(2);
final var t3 = allTokens.get(3);
final var t4 = allTokens.get(4);
assertEquals("Hello, ", t0.getText());
assertTokenType(RawText, t0);
assertEquals("$", t1.getText());
assertTokenType(DollarReferenceStart, t1);
assertEquals("target", t2.getText());
assertTokenType(GroovyCode, t2);
assertInstanceOf(MergedGroovyCodeToken.class, t2);
assertEquals("!", t3.getText());
assertTokenType(RawText, t3);
assertTokenType(EOF, t4);
}
@TestFactory
public Collection<DynamicTest> lexerFileTests() {
return FileComparisonTestUtil.getTestsFor(
@ -71,10 +29,8 @@ public class WebViewComponentsLexerTests {
sourceFile -> {
final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile));
final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input);
final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(
lexer,
Set.of(WebViewComponentsLexer.HIDDEN) // include bad tokens for testing
);
// include all (!) (non-skipped) tokens for testing via Set.of()
final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer, Set.of());
final List<Token> allTokens = tokenStream.getAllTokensSkipEOF();
final var sb = new StringBuilder();
for (int i = 0; i < allTokens.size(); i++) {

2
web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsParserTests.java
View File

@ -27,7 +27,7 @@ public final class WebViewComponentsParserTests {
private static final Logger logger = LoggerFactory.getLogger(WebViewComponentsParserTests.class);
private static final String parserFileBase = String.join(File.separator, "src", "test", "parser");
private static final String parserTreeFileBase = String.join(File.separator, parserFileBase, "trees");
private static final String parserTreeFileBase = String.join(File.separator, parserFileBase, "parse-tree-files");
private static final String parseTreeFileSuffix = "_parseTree";
private static final String parseTreeFileExtension = ".txt";
private static final Set<String> parserFileGlobs = Set.of(

1
web-view-components-compiler/src/test/lexer/doctypeHtml.wvc Normal file
View File

@ -0,0 +1 @@
<!DOCTYPE html>

1
web-view-components-compiler/src/test/lexer/emptyFragment.wvc Normal file
View File

@ -0,0 +1 @@
<></>

3
web-view-components-compiler/src/test/lexer/multiLineTag.wvc Normal file
View File

@ -0,0 +1,3 @@
<Test
someAttr
/>

1
web-view-components-compiler/src/test/lexer/noPreamble.wvc Normal file
View File

@ -0,0 +1 @@
Hello!

2
web-view-components-compiler/src/test/lexer/preambleFourDash.wvc Normal file
View File

@ -0,0 +1,2 @@
---
----

2
web-view-components-compiler/src/test/lexer/preambleStartFourDash.wvc Normal file
View File

@ -0,0 +1,2 @@
----
---

2
web-view-components-compiler/src/test/lexer/preambleTooManyThreeDash.wvc Normal file
View File

@ -0,0 +1,2 @@
---
------

6
web-view-components-compiler/src/test/parser/blankPreambleWithExtraLines.wvc → web-view-components-compiler/src/test/lexer/preambleWithComment.wvc
View File

@ -1,7 +1,3 @@
---
// ---
---

3
web-view-components-compiler/src/test/lexer/preambleWithDollarSlashyString.wvc Normal file
View File

@ -0,0 +1,3 @@
---
def regex = $/match --- me $ $test ${test('---')} $$ $/ / /$
---

3
web-view-components-compiler/src/test/lexer/preambleWithGString.wvc Normal file
View File

@ -0,0 +1,3 @@
---
def test = "---$test${test('---')}"
---

3
web-view-components-compiler/src/test/lexer/preambleWithJString.wvc Normal file
View File

@ -0,0 +1,3 @@
---
def test = '---'
---

3
web-view-components-compiler/src/test/lexer/preambleWithParenthesesSlashyString.wvc Normal file
View File

@ -0,0 +1,3 @@
---
def regex = (/match --- \/ me/)
---

5
web-view-components-compiler/src/test/lexer/preambleWithTripleGString.wvc Normal file
View File

@ -0,0 +1,5 @@
---
def test = """
--- $test ${test("---")} " \$
"""
---

5
web-view-components-compiler/src/test/lexer/preambleWithTripleJString.wvc Normal file
View File

@ -0,0 +1,5 @@
---
def test = '''
---
'''
---

1
web-view-components-compiler/src/test/lexer/selfCloseComponent.wvc Normal file
View File

@ -0,0 +1 @@
<Test />

1
web-view-components-compiler/src/test/lexer/spaceThenFakePreamble.wvc Normal file
View File

@ -0,0 +1 @@
---

5
web-view-components-compiler/src/test/lexer/tokens-files/badTagStart_tokens.txt
View File

@ -1,5 +1,6 @@
0: ComponentOpen[1,1](<)
1: StringIdentifier[1,2](bad)
2: TagError[1,5](!)
3: ComponentSelfClose[1,7](/>)
4: RawText[1,9](\n)
3: Nlws[1,6]( )
4: ComponentSelfClose[1,7](/>)
5: RawText[1,9](\n)

101
web-view-components-compiler/src/test/lexer/tokens-files/complicated_tokens.txt
View File

@ -1,7 +1,7 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](import some.Thing // a comment...World!')
2: PreambleBreak[4,31](\n---\n)
3: RawText[6,1](<!DOCTYPE html>\n)
1: GroovyCode[2,1](import some.Thing // a comment...rld!'\n)
2: PreambleBreak[5,1](---)
3: RawText[5,4](\n<!DOCTYPE html>\n)
4: ComponentOpen[7,1](<)
5: StringIdentifier[7,2](html)
6: ComponentClose[7,6](>)
@ -33,50 +33,51 @@
32: RawText[11,33](\n )
33: ComponentOpen[12,13](<)
34: TypedIdentifier[12,14](Case)
35: AttributeIdentifier[12,19](cond)
36: Equals[12,23](=)
37: ClosureAttrValueStart[12,24]({)
38: GroovyCode[12,25](isItTrue())
39: ClosureAttrValueEnd[12,35](})
40: ComponentClose[12,36](>)
41: RawText[12,37](\n )
42: ComponentOpen[13,17](<)
43: StringIdentifier[13,18](p)
44: ComponentClose[13,19](>)
45: RawText[13,20](It's true! :))
46: ClosingComponentOpen[13,33](</)
47: StringIdentifier[13,35](p)
48: ComponentClose[13,36](>)
49: RawText[13,37](\n )
50: ClosingComponentOpen[14,13](</)
51: TypedIdentifier[14,15](Case)
52: ComponentClose[14,19](>)
53: RawText[14,20](\n )
54: ComponentOpen[15,13](<)
55: TypedIdentifier[15,14](Default)
56: ComponentClose[15,21](>)
57: RawText[15,22](\n )
58: ComponentOpen[16,17](<)
59: StringIdentifier[16,18](p)
60: ComponentClose[16,19](>)
61: RawText[16,20](It's false... :()
62: ClosingComponentOpen[16,36](</)
63: StringIdentifier[16,38](p)
64: ComponentClose[16,39](>)
65: RawText[16,40](\n )
66: ClosingComponentOpen[17,13](</)
67: TypedIdentifier[17,15](Default)
68: ComponentClose[17,22](>)
69: RawText[17,23](\n )
70: ClosingComponentOpen[18,9](</)
71: TypedIdentifier[18,11](groowt.view.web.Select)
72: ComponentClose[18,33](>)
73: RawText[18,34](\n )
74: ClosingComponentOpen[19,5](</)
75: StringIdentifier[19,7](body)
76: ComponentClose[19,11](>)
77: RawText[19,12](\n)
78: ClosingComponentOpen[20,1](</)
79: StringIdentifier[20,3](html)
80: ComponentClose[20,7](>)
81: RawText[20,8](\n)
35: Nlws[12,18]( )
36: AttributeIdentifier[12,19](cond)
37: Equals[12,23](=)
38: ClosureAttrValueStart[12,24]({)
39: GroovyCode[12,25](isItTrue())
40: ClosureAttrValueEnd[12,35](})
41: ComponentClose[12,36](>)
42: RawText[12,37](\n )
43: ComponentOpen[13,17](<)
44: StringIdentifier[13,18](p)
45: ComponentClose[13,19](>)
46: RawText[13,20](It's true! :))
47: ClosingComponentOpen[13,33](</)
48: StringIdentifier[13,35](p)
49: ComponentClose[13,36](>)
50: RawText[13,37](\n )
51: ClosingComponentOpen[14,13](</)
52: TypedIdentifier[14,15](Case)
53: ComponentClose[14,19](>)
54: RawText[14,20](\n )
55: ComponentOpen[15,13](<)
56: TypedIdentifier[15,14](Default)
57: ComponentClose[15,21](>)
58: RawText[15,22](\n )
59: ComponentOpen[16,17](<)
60: StringIdentifier[16,18](p)
61: ComponentClose[16,19](>)
62: RawText[16,20](It's false... :()
63: ClosingComponentOpen[16,36](</)
64: StringIdentifier[16,38](p)
65: ComponentClose[16,39](>)
66: RawText[16,40](\n )
67: ClosingComponentOpen[17,13](</)
68: TypedIdentifier[17,15](Default)
69: ComponentClose[17,22](>)
70: RawText[17,23](\n )
71: ClosingComponentOpen[18,9](</)
72: TypedIdentifier[18,11](groowt.view.web.Select)
73: ComponentClose[18,33](>)
74: RawText[18,34](\n )
75: ClosingComponentOpen[19,5](</)
76: StringIdentifier[19,7](body)
77: ComponentClose[19,11](>)
78: RawText[19,12](\n)
79: ClosingComponentOpen[20,1](</)
80: StringIdentifier[20,3](html)
81: ComponentClose[20,7](>)
82: RawText[20,8](\n)

5
web-view-components-compiler/src/test/lexer/tokens-files/dataComponent_tokens.txt
View File

@ -1,4 +1,5 @@
0: ComponentOpen[1,1](<)
1: StringIdentifier[1,2](data-test)
2: ComponentSelfClose[1,12](/>)
3: RawText[1,14](\n)
2: Nlws[1,11]( )
3: ComponentSelfClose[1,12](/>)
4: RawText[1,14](\n)

1
web-view-components-compiler/src/test/lexer/tokens-files/doctypeHtml_tokens.txt Normal file
View File

@ -0,0 +1 @@
0: RawText[1,1](<!DOCTYPE html>\n)

3
web-view-components-compiler/src/test/lexer/tokens-files/emptyFragment_tokens.txt Normal file
View File

@ -0,0 +1,3 @@
0: FragmentOpen[1,1](<>)
1: FragmentClose[1,3](</>)
2: RawText[1,6](\n)

7
web-view-components-compiler/src/test/lexer/tokens-files/multiLineTag_tokens.txt Normal file
View File

@ -0,0 +1,7 @@
0: ComponentOpen[1,1](<)
1: TypedIdentifier[1,2](Test)
2: Nlws[1,6](\n )
3: AttributeIdentifier[2,5](someAttr)
4: Nlws[2,13](\n)
5: ComponentSelfClose[3,1](/>)
6: RawText[3,3](\n)

1
web-view-components-compiler/src/test/lexer/tokens-files/noPreamble_tokens.txt Normal file
View File

@ -0,0 +1 @@
0: RawText[1,1](Hello!\n)

3
web-view-components-compiler/src/test/lexer/tokens-files/preambleFourDash_tokens.txt Normal file
View File

@ -0,0 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: PreambleBreak[2,1](---)
2: RawText[2,4](-\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleStartFourDash_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---)
1: GroovyCode[1,4](-\n)
2: PreambleBreak[2,1](---)
3: RawText[2,4](\n)

3
web-view-components-compiler/src/test/lexer/tokens-files/preambleTooManyThreeDash_tokens.txt Normal file
View File

@ -0,0 +1,3 @@
0: PreambleBreak[1,1](---\n)
1: PreambleBreak[2,1](---)
2: RawText[2,4](---\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithComment_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](// ---\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithDollarSlashyString_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def regex = $/match --- me $ $... / /$\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithGString_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = "---$test${test('---')}"\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithJString_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = '---'\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithParenthesesSlashyString_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def regex = (/match --- \/ me/)\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithTripleGString_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = """\n--- $test ${te...\n"""\n)
2: PreambleBreak[5,1](---)
3: RawText[5,4](\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/preambleWithTripleJString_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](def test = '''\n---\n'''\n)
2: PreambleBreak[5,1](---)
3: RawText[5,4](\n)

5
web-view-components-compiler/src/test/lexer/tokens-files/selfCloseComponent_tokens.txt Normal file
View File

@ -0,0 +1,5 @@
0: ComponentOpen[1,1](<)
1: TypedIdentifier[1,2](Test)
2: Nlws[1,6]( )
3: ComponentSelfClose[1,7](/>)
4: RawText[1,9](\n)

1
web-view-components-compiler/src/test/lexer/tokens-files/spaceAfterSlash_tokens.txt
View File

@ -1 +1,2 @@
0: ClosingComponentOpen[1,1](</)
1: Nlws[1,3](\n)

1
web-view-components-compiler/src/test/lexer/tokens-files/spaceThenFakePreamble_tokens.txt Normal file
View File

@ -0,0 +1 @@
0: RawText[1,1]( ---\n)

4
web-view-components-compiler/src/test/lexer/tokens-files/withPreamble_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
0: PreambleBreak[1,1](---\n)
1: GroovyCode[2,1](package test\n)
2: PreambleBreak[3,1](---)
3: RawText[3,4](\n)

3
web-view-components-compiler/src/test/lexer/withPreamble.wvc Normal file
View File

@ -0,0 +1,3 @@
---
package test
---

2
web-view-components-compiler/src/test/parser/blankPreambleOnly.wvc
View File

@ -1,2 +0,0 @@
---
---

2
web-view-components-compiler/src/test/parser/fragment.wvc
View File

@ -1 +1 @@
<><p>Hello, World!</p></>
<>Hello, World!</>

1
web-view-components-compiler/src/test/parser/helloWorld.wvc Normal file
View File

@ -0,0 +1 @@
Hello, World!

137
web-view-components-compiler/src/test/parser/trees/complicated_parseTree.txt → web-view-components-compiler/src/test/parser/parse-tree-files/complicated_parseTree.txt
View File

@ -1,94 +1,93 @@
compilationUnit[1,1..21,1]
preamble[1,1..4,31]
preamble[1,1..5,4]
PreambleBreak[1,1](---\n)
GroovyCode[2,1](import some.Thing // a comment\n\ndef greeting = 'Hello, World!')
PreambleBreak[4,31](\n---\n)
body[6,1..20,8]
bodyText[6,1..6,1]
jStringBodyText[6,1..6,1]
RawText[6,1](<!DOCTYPE html>\n)
GroovyCode[2,1](import some.Thing // a comment\n\ndef greeting = 'Hello, World!'\n)
PreambleBreak[5,1](---)
body[5,4..20,8]
bodyText[5,4..7,1]
text[5,4..7,1]
RawText[5,4](\n<!DOCTYPE html>\n)
component[7,1..20,7]
componentWithChildren[7,1..20,7]
openComponent[7,1..7,6]
ComponentOpen[7,1](<)
componentArgs[7,2..7,2]
componentType[7,2..7,2]
componentArgs[7,2..7,6]
componentType[7,2..7,6]
StringIdentifier[7,2](html)
ComponentClose[7,6](>)
body[7,7..19,12]
bodyText[7,7..7,7]
jStringBodyText[7,7..7,7]
bodyText[7,7..8,5]
text[7,7..8,5]
RawText[7,7](\n )
component[8,5..8,17]
componentWithChildren[8,5..8,17]
openComponent[8,5..8,10]
ComponentOpen[8,5](<)
componentArgs[8,6..8,6]
componentType[8,6..8,6]
componentArgs[8,6..8,10]
componentType[8,6..8,10]
StringIdentifier[8,6](head)
ComponentClose[8,10](>)
closingComponent[8,11..8,17]
ClosingComponentOpen[8,11](</)
componentType[8,13..8,13]
componentType[8,13..8,17]
StringIdentifier[8,13](head)
ComponentClose[8,17](>)
bodyText[8,18..8,18]
jStringBodyText[8,18..8,18]
bodyText[8,18..9,5]
text[8,18..9,5]
RawText[8,18](\n )
component[9,5..19,11]
componentWithChildren[9,5..19,11]
openComponent[9,5..9,10]
ComponentOpen[9,5](<)
componentArgs[9,6..9,6]
componentType[9,6..9,6]
componentArgs[9,6..9,10]
componentType[9,6..9,10]
StringIdentifier[9,6](body)
ComponentClose[9,10](>)
body[9,11..18,34]
bodyText[9,11..9,11]
jStringBodyText[9,11..9,11]
body[9,11..19,5]
bodyText[9,11..10,9]
text[9,11..10,9]
RawText[9,11](\n )
component[10,9..10,28]
componentWithChildren[10,9..10,28]
openComponent[10,9..10,12]
ComponentOpen[10,9](<)
componentArgs[10,10..10,10]
componentType[10,10..10,10]
componentArgs[10,10..10,12]
componentType[10,10..10,12]
StringIdentifier[10,10](h1)
ComponentClose[10,12](>)
body[10,13..10,23]
bodyText[10,13..10,23]
gStringBodyText[10,13..10,23]
gStringBodyTextGroovyElement[10,13..10,23]
dollarScriptlet[10,13..10,23]
DollarScriptletOpen[10,13](${)
GroovyCode[10,15](greeting)
DollarScriptletClose[10,23](})
bodyTextGroovyElement[10,13..10,23]
dollarScriptlet[10,13..10,23]
DollarScriptletOpen[10,13](${)
GroovyCode[10,15](greeting)
DollarScriptletClose[10,23](})
closingComponent[10,24..10,28]
ClosingComponentOpen[10,24](</)
componentType[10,26..10,26]
componentType[10,26..10,28]
StringIdentifier[10,26](h1)
ComponentClose[10,28](>)
bodyText[10,29..10,29]
jStringBodyText[10,29..10,29]
bodyText[10,29..11,9]
text[10,29..11,9]
RawText[10,29](\n )
component[11,9..18,33]
componentWithChildren[11,9..18,33]
openComponent[11,9..11,32]
ComponentOpen[11,9](<)
componentArgs[11,10..11,10]
componentType[11,10..11,10]
componentArgs[11,10..11,32]
componentType[11,10..11,32]
TypedIdentifier[11,10](groowt.view.web.Select)
ComponentClose[11,32](>)
body[11,33..17,23]
bodyText[11,33..11,33]
jStringBodyText[11,33..11,33]
body[11,33..18,9]
bodyText[11,33..12,13]
text[11,33..12,13]
RawText[11,33](\n )
component[12,13..14,19]
componentWithChildren[12,13..14,19]
openComponent[12,13..12,36]
ComponentOpen[12,13](<)
componentArgs[12,14..12,35]
componentType[12,14..12,14]
componentType[12,14..12,18]
TypedIdentifier[12,14](Case)
attr[12,19..12,35]
keyValueAttr[12,19..12,35]
@ -100,9 +99,9 @@ compilationUnit[1,1..21,1]
GroovyCode[12,25](isItTrue())
ClosureAttrValueEnd[12,35](})
ComponentClose[12,36](>)
body[12,37..13,37]
bodyText[12,37..12,37]
jStringBodyText[12,37..12,37]
body[12,37..14,13]
bodyText[12,37..13,17]
text[12,37..13,17]
RawText[12,37](\n )
component[13,17..13,36]
componentWithChildren[13,17..13,36]
@ -112,37 +111,37 @@ compilationUnit[1,1..21,1]
componentType[13,18..13,18]
StringIdentifier[13,18](p)
ComponentClose[13,19](>)
body[13,20..13,20]
bodyText[13,20..13,20]
jStringBodyText[13,20..13,20]
body[13,20..13,33]
bodyText[13,20..13,33]
text[13,20..13,33]
RawText[13,20](It's true! :))
closingComponent[13,33..13,36]
ClosingComponentOpen[13,33](</)
componentType[13,35..13,35]
StringIdentifier[13,35](p)
ComponentClose[13,36](>)
bodyText[13,37..13,37]
jStringBodyText[13,37..13,37]
bodyText[13,37..14,13]
text[13,37..14,13]
RawText[13,37](\n )
closingComponent[14,13..14,19]
ClosingComponentOpen[14,13](</)
componentType[14,15..14,15]
componentType[14,15..14,19]
TypedIdentifier[14,15](Case)
ComponentClose[14,19](>)
bodyText[14,20..14,20]
jStringBodyText[14,20..14,20]
bodyText[14,20..15,13]
text[14,20..15,13]
RawText[14,20](\n )
component[15,13..17,22]
componentWithChildren[15,13..17,22]
openComponent[15,13..15,21]
ComponentOpen[15,13](<)
componentArgs[15,14..15,14]
componentType[15,14..15,14]
componentArgs[15,14..15,21]
componentType[15,14..15,21]
TypedIdentifier[15,14](Default)
ComponentClose[15,21](>)
body[15,22..16,40]
bodyText[15,22..15,22]
jStringBodyText[15,22..15,22]
body[15,22..17,13]
bodyText[15,22..16,17]
text[15,22..16,17]
RawText[15,22](\n )
component[16,17..16,39]
componentWithChildren[16,17..16,39]
@ -152,48 +151,48 @@ compilationUnit[1,1..21,1]
componentType[16,18..16,18]
StringIdentifier[16,18](p)
ComponentClose[16,19](>)
body[16,20..16,20]
bodyText[16,20..16,20]
jStringBodyText[16,20..16,20]
body[16,20..16,36]
bodyText[16,20..16,36]
text[16,20..16,36]
RawText[16,20](It's false... :()
closingComponent[16,36..16,39]
ClosingComponentOpen[16,36](</)
componentType[16,38..16,38]
StringIdentifier[16,38](p)
ComponentClose[16,39](>)
bodyText[16,40..16,40]
jStringBodyText[16,40..16,40]
bodyText[16,40..17,13]
text[16,40..17,13]
RawText[16,40](\n )
closingComponent[17,13..17,22]
ClosingComponentOpen[17,13](</)
componentType[17,15..17,15]
componentType[17,15..17,22]
TypedIdentifier[17,15](Default)
ComponentClose[17,22](>)
bodyText[17,23..17,23]
jStringBodyText[17,23..17,23]
bodyText[17,23..18,9]
text[17,23..18,9]
RawText[17,23](\n )
closingComponent[18,9..18,33]
ClosingComponentOpen[18,9](</)
componentType[18,11..18,11]
componentType[18,11..18,33]
TypedIdentifier[18,11](groowt.view.web.Select)
ComponentClose[18,33](>)
bodyText[18,34..18,34]
jStringBodyText[18,34..18,34]
bodyText[18,34..19,5]
text[18,34..19,5]
RawText[18,34](\n )
closingComponent[19,5..19,11]
ClosingComponentOpen[19,5](</)
componentType[19,7..19,7]
componentType[19,7..19,11]
StringIdentifier[19,7](body)
ComponentClose[19,11](>)
bodyText[19,12..19,12]
jStringBodyText[19,12..19,12]
text[19,12..19,12]
RawText[19,12](\n)
closingComponent[20,1..20,7]
ClosingComponentOpen[20,1](</)
componentType[20,3..20,3]
componentType[20,3..20,7]
StringIdentifier[20,3](html)
ComponentClose[20,7](>)
bodyText[20,8..20,8]
jStringBodyText[20,8..20,8]
text[20,8..20,8]
RawText[20,8](\n)
EOF[21,1](<EOF>)

8
web-view-components-compiler/src/test/parser/trees/emptyHtml_parseTree.txt → web-view-components-compiler/src/test/parser/parse-tree-files/emptyHtml_parseTree.txt
View File

@ -4,16 +4,16 @@ compilationUnit[1,1..2,1]
componentWithChildren[1,1..1,13]
openComponent[1,1..1,6]
ComponentOpen[1,1](<)
componentArgs[1,2..1,2]
componentType[1,2..1,2]
componentArgs[1,2..1,6]
componentType[1,2..1,6]
StringIdentifier[1,2](html)
ComponentClose[1,6](>)
closingComponent[1,7..1,13]
ClosingComponentOpen[1,7](</)
componentType[1,9..1,9]
componentType[1,9..1,13]
StringIdentifier[1,9](html)
ComponentClose[1,13](>)
bodyText[1,14..1,14]
jStringBodyText[1,14..1,14]
text[1,14..1,14]
RawText[1,14](\n)
EOF[2,1](<EOF>)

14
web-view-components-compiler/src/test/parser/parse-tree-files/fragment_parseTree.txt Normal file
View File

@ -0,0 +1,14 @@
compilationUnit[1,1..2,1]
body[1,1..1,19]
component[1,1..1,19]
fragmentComponent[1,1..1,19]
FragmentOpen[1,1](<>)
body[1,3..1,16]
bodyText[1,3..1,16]
text[1,3..1,16]
RawText[1,3](Hello, World!)
FragmentClose[1,16](</>)
bodyText[1,19..1,19]
text[1,19..1,19]
RawText[1,19](\n)
EOF[2,1](<EOF>)

12
web-view-components-compiler/src/test/parser/parse-tree-files/helloTarget_parseTree.txt Normal file
View File

@ -0,0 +1,12 @@
compilationUnit[1,1..1,16]
body[1,1..1,15]
bodyText[1,1..1,15]
text[1,1..1,8]
RawText[1,1](Hello, )
bodyTextGroovyElement[1,8..1,15]
dollarReference[1,8..1,15]
DollarReferenceStart[1,8]($)
GroovyCode[1,9](target)
text[1,15..1,15]
RawText[1,15](!)
EOF[1,16](<EOF>)

6
web-view-components-compiler/src/test/parser/parse-tree-files/helloWorld_parseTree.txt Normal file
View File

@ -0,0 +1,6 @@
compilationUnit[1,1..2,1]
body[1,1..2,1]
bodyText[1,1..2,1]
text[1,1..2,1]
RawText[1,1](Hello, World!\n)
EOF[2,1](<EOF>)

5
web-view-components-compiler/src/test/parser/preambleWithClass.wvc
View File

@ -1,5 +0,0 @@
---
class Greeting {
String myGreeting
}
---

5
web-view-components-compiler/src/test/parser/trees/blankPreambleOnly_parseTree.txt
View File

@ -1,5 +0,0 @@
compilationUnit[1,1..3,1]
preamble[1,1..2,1]
PreambleBreak[1,1](---\n)
PreambleBreak[2,1](---\n)
EOF[3,1](<EOF>)

6
web-view-components-compiler/src/test/parser/trees/blankPreambleWithExtraLines_parseTree.txt
View File

@ -1,6 +0,0 @@
compilationUnit[1,1..8,1]
preamble[1,1..6,1]
PreambleBreak[1,1](---\n)
GroovyCode[2,1](\n\n\n\n)
PreambleBreak[6,1](\n---\n)
EOF[8,1](<EOF>)

28
web-view-components-compiler/src/test/parser/trees/fragment_parseTree.txt
View File

@ -1,28 +0,0 @@
compilationUnit[1,1..2,1]
body[1,1..1,26]
component[1,1..1,23]
fragmentComponent[1,1..1,23]
FragmentOpen[1,1](<>)
body[1,3..1,22]
component[1,3..1,22]
componentWithChildren[1,3..1,22]
openComponent[1,3..1,5]
ComponentOpen[1,3](<)
componentArgs[1,4..1,4]
componentType[1,4..1,4]
StringIdentifier[1,4](p)
ComponentClose[1,5](>)
body[1,6..1,6]
bodyText[1,6..1,6]
jStringBodyText[1,6..1,6]
RawText[1,6](Hello, World!)
closingComponent[1,19..1,22]
ClosingComponentOpen[1,19](</)
componentType[1,21..1,21]
StringIdentifier[1,21](p)
ComponentClose[1,22](>)
FragmentClose[1,23](</>)
bodyText[1,26..1,26]
jStringBodyText[1,26..1,26]
RawText[1,26](\n)
EOF[2,1](<EOF>)

13
web-view-components-compiler/src/test/parser/trees/helloTarget_parseTree.txt
View File

@ -1,13 +0,0 @@
compilationUnit[1,1..1,16]
body[1,1..1,15]
bodyText[1,1..1,15]
gStringBodyText[1,1..1,15]
jStringBodyText[1,1..1,1]
RawText[1,1](Hello, )
gStringBodyTextGroovyElement[1,8..1,9]
dollarReference[1,8..1,9]
DollarReferenceStart[1,8]($)
GroovyCode[1,9](target)
jStringBodyText[1,15..1,15]
RawText[1,15](!)
EOF[1,16](<EOF>)

6
web-view-components-compiler/src/test/parser/trees/preambleWithClass_parseTree.txt
View File

@ -1,6 +0,0 @@
compilationUnit[1,1..6,1]
preamble[1,1..4,2]
PreambleBreak[1,1](---\n)
GroovyCode[2,1](class Greeting {\n String myGreeting\n})
PreambleBreak[4,2](\n---\n)
EOF[6,1](<EOF>)

23
web-view-components-compiler/src/testFixtures/java/groowt/view/component/web/ast/NodeFactoryTests.java
View File

@ -1,6 +1,5 @@
package groowt.view.component.web.ast;
import groowt.view.component.web.ast.extension.GStringNodeExtension;
import groowt.view.component.web.ast.node.*;
import groowt.view.component.web.util.TokenRange;
import org.antlr.v4.runtime.Token;
@ -75,14 +74,26 @@ public abstract class NodeFactoryTests {
}
@Test
public void gStringBodyTextNode(@Mock LeafNode child) {
when(child.hasExtension(GStringNodeExtension.class)).thenReturn(true);
assertNotNull(this.nodeFactory.gStringBodyTextNode(this.getTokenRange(), List.of(child)));
public void bodyTextNode(@Mock BodyTextChild child, @Mock TreeNode childAsNode) {
when(child.asNode()).thenReturn(childAsNode);
assertNotNull(this.nodeFactory.bodyTextNode(this.getTokenRange(), List.of(child)));
}
@Test
public void jStringBodyTextNode() {
assertNotNull(this.nodeFactory.jStringBodyTextNode(this.getTokenRange(), "Hello!"));
public void questionTagNode(@Mock QuestionTagChild child, @Mock TreeNode childAsNode) {
when(child.asNode()).thenReturn(childAsNode);
assertNotNull(this.nodeFactory.questionTagNode(this.getTokenRange(), List.of(child)));
}
@Test
public void htmlCommentNode(@Mock HtmlCommentChild child, @Mock TreeNode childAsNode) {
when(child.asNode()).thenReturn(childAsNode);
assertNotNull(this.nodeFactory.htmlCommentNode(this.getTokenRange(), List.of(child)));
}
@Test
public void textNode() {
assertNotNull(this.nodeFactory.textNode(this.getTokenRange(), "Hello, World!"));
}
@Test

37
web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/AbstractSourceTransformerCli.kt
View File

@ -16,36 +16,36 @@ abstract class AbstractSourceTransformerCli : Callable<Int> {
protected lateinit var targets: List<Path>
@Option(
names = ["--n", "--dry-run"],
description = ["Do a dry run; do not output files to disk."]
names = ["-i", "--interactive"],
description = ["Allow interactive recovery from errors. If false, implies -W."]
)
protected var dryRun: Boolean = false
protected var interactive = false
@Option(
names = ["-y", "--yes"],
description = ["Automatically write output files if there are no processing errors."]
names = ["-P", "--print-only"],
description = ["Only print result(s) to stdout; do not write files to disk."]
)
protected var autoYes: Boolean = false
protected var printOnly = false
@Option(
names = ["-W", "--write-over"],
description = ["If an output file already exists, write over it without asking."]
)
protected var autoWriteOver: Boolean = false
protected var autoWriteOver = false
@Option(
names = ["-v", "--verbose"],
description = ["Log verbosely to the console."]
description = ["Log exceptions and errors verbosely to stderr."]
)
protected var verbose: Boolean = false
protected var verbose = false
private val scanner = Scanner(System.`in`)
protected fun getYesNo(prompt: String, allowAuto: Boolean = true): Boolean {
if (this.autoYes && allowAuto) {
return true
protected open fun getYesNo(prompt: String, fallback: Boolean): Boolean {
if (!interactive) {
return fallback
} else {
print("$prompt (y/n) ")
print("$prompt (y/n): ")
while (true) {
if (this.scanner.hasNextLine()) {
val input = this.scanner.nextLine()
@ -60,24 +60,21 @@ abstract class AbstractSourceTransformerCli : Callable<Int> {
}
private fun doWrite(resolvedTarget: Path, text: String) {
if (this.dryRun) {
println("Dry-run: would write to $resolvedTarget")
} else {
println("Writing to $resolvedTarget...")
if (!this.printOnly) {
Files.writeString(resolvedTarget, text)
}
}
protected fun writeToDisk(target: Path, text: String) {
protected open fun writeToDisk(target: Path, text: String) {
val outputDir = getOutputDir()
if (outputDir != null) {
Files.createDirectories(outputDir)
}
val resolvedTarget = outputDir?.resolve(target) ?: target
if (Files.exists(resolvedTarget) && !autoWriteOver) {
if (getYesNo("$resolvedTarget already exists. Write over?")) {
if (getYesNo("$resolvedTarget already exists. Write over?", true)) {
doWrite(resolvedTarget, text)
} else {
} else if (interactive) {
println("Skipping writing to $resolvedTarget")
}
} else {

161
web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/ParseWvc.kt Normal file
View File

@ -0,0 +1,161 @@
package groowt.view.component.web.tools
import groowt.view.component.web.antlr.*
import groowt.view.component.web.antlr.WebViewComponentsLexerBase.ERROR
import groowt.view.component.web.antlr.WebViewComponentsLexerBase.HIDDEN
import groowt.view.component.web.antlr.WebViewComponentsParser.CompilationUnitContext
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.ConsoleErrorListener
import picocli.CommandLine
import picocli.CommandLine.Option
import java.nio.file.Path
import kotlin.io.path.nameWithoutExtension
import kotlin.system.exitProcess
open class ParseWvc : AbstractSourceTransformerCli() {
companion object {
@JvmStatic
fun main(args: Array<String>) {
exitProcess(CommandLine(ParseWvc()).execute(*args))
}
}
@Option(
names = ["-s", "--suffix"],
description = ["The suffix (not extension!) to append to the output file."]
)
protected var suffix: String? = null
@Option(
names = ["-e", "--extension"],
description = ["The extension for output files."],
defaultValue = ".txt"
)
protected lateinit var extension: String
@Option(
names = ["-d", "--output-dir"],
description = ["The output directory."],
defaultValue = ".",
paramLabel = "outputDir"
)
protected lateinit var myOutputDir: Path
@Option(
names = ["--strict"],
description = ["If true, do not recover from syntax errors during parsing."],
negatable = true
)
protected var strict = true
protected open fun onLexerErrors(errors: List<LexerError>): Boolean {
System.err.println("There were lexer errors.")
errors.forEach { System.err.println(formatLexerError(it)) }
return this.getYesNo("Do you wish to try again?", false)
}
protected open fun onParserErrors(errors: List<ParserError>): Boolean {
System.err.println("There were parser errors.")
errors.forEach { System.err.println(formatParserError(it)) }
return this.getYesNo("Do you wish to try again?", false)
}
protected open fun getOutputPath(target: Path): Path =
Path.of(target.nameWithoutExtension + (suffix ?: "") + extension)
protected open fun onSuccess(
target: Path,
parser: WebViewComponentsParser,
cuContext: CompilationUnitContext
): Boolean {
val formatted = formatTree(
parser = parser,
tree = cuContext,
colors = false
)
if (this.interactive) {
println("Please review the following parse tree:\n$formatted")
} else {
println(formatted)
}
if (this.getYesNo("Write to disk?", true)) {
this.writeToDisk(getOutputPath(target), formatted)
return false
} else {
return this.getYesNo("Do you wish to redo this file?", false)
}
}
protected open fun onException(e: Exception): Boolean {
System.err.println("There was an exception during parsing: $e")
if (this.verbose) {
e.printStackTrace(System.err)
}
return this.getYesNo("Do you wish to try again?", false)
}
override fun getOutputDir() = myOutputDir
override fun transform(target: Path): Int {
if (interactive) {
println("Parsing $target")
}
var code = 0
while (true) {
try {
val input = CharStreams.fromPath(target)
val lexer = WebViewComponentsLexer(input)
lexer.removeErrorListener(ConsoleErrorListener.INSTANCE)
val lexerErrorListener = LexerErrorListener()
lexer.addErrorListener(lexerErrorListener)
val tokenStream = if (strict) {
WebViewComponentsTokenStream(lexer) // only ignore hidden (default)
} else {
WebViewComponentsTokenStream(lexer, setOf(HIDDEN, ERROR)) // ignore hidden and error
}
val parser = WebViewComponentsParser(tokenStream)
parser.removeErrorListener(ConsoleErrorListener.INSTANCE)
val parserErrorListener = ParserErrorListener()
parser.addErrorListener(parserErrorListener)
val cuContext = parser.compilationUnit()
val lexerErrors = lexerErrorListener.getErrors() + parserErrorListener.getLexerErrors()
val parserErrors = parserErrorListener.getParserErrors()
if (lexerErrors.isNotEmpty()) {
val recover = this.onLexerErrors(lexerErrors)
if (!recover) {
code = 1
break
}
} else if (parserErrors.isNotEmpty()) {
val recover = this.onParserErrors(parserErrors)
if (!recover) {
code = 1
break
}
} else {
val redo = this.onSuccess(target, parser, cuContext)
if (!redo) {
break
}
}
} catch (e: Exception) {
val recover = this.onException(e)
if (!recover) {
code = 1
break
}
}
}
return code
}
}

40
web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/TokenizeWvc.kt
View File

@ -8,7 +8,6 @@ import picocli.CommandLine
import picocli.CommandLine.Command
import picocli.CommandLine.Option
import java.nio.file.Path
import java.util.*
import kotlin.io.path.nameWithoutExtension
import kotlin.system.exitProcess
@ -31,9 +30,9 @@ open class TokenizeWvc : AbstractSourceTransformerCli() {
@Option(
names = ["-s", "--suffix"],
description = ["The suffix (not extension!) to append to the output file names."]
description = ["The suffix (not extension!) to append to output file names."]
)
protected lateinit var suffix: Optional<String>
protected var suffix: String? = null
@Option(
names = ["-e", "--extension"],
@ -48,47 +47,46 @@ open class TokenizeWvc : AbstractSourceTransformerCli() {
defaultValue = ".",
paramLabel = "outputDir"
)
private var myOutputDir: Path? = null
protected lateinit var myOutputDir: Path
protected fun onErrors(errors: List<LexerError>): Boolean {
println("There were errors during tokenization.")
errors.forEach { println(formatLexerError(it)) }
System.err.println("There were errors during tokenization.")
errors.forEach { System.err.println(formatLexerError(it)) }
return this.getYesNo("Do you wish to try again?", false)
}
private fun getOutputPath(target: Path): Path =
Path.of(target.nameWithoutExtension + suffix.orElse("") + extension)
protected fun getOutputPath(target: Path): Path =
Path.of(target.nameWithoutExtension + (suffix ?: "") + extension)
protected fun onSuccess(target: Path, allTokens: List<Token>): Boolean {
val formatted = allTokens.mapIndexed { index, token ->
"$index: ${formatToken(token)}"
}.joinToString(separator = "\n")
if (!this.autoYes) {
if (interactive) {
println("Please review the following tokens:\n$formatted")
if (this.getYesNo("Write to disk?")) {
this.writeToDisk(getOutputPath(target), formatted)
return false
} else {
return this.getYesNo("Do you wish to redo this file?", false)
}
} else {
}
if (this.getYesNo("Write to disk?", true)) {
this.writeToDisk(getOutputPath(target), formatted)
return false
} else {
return this.getYesNo("Do you wish to redo this file?", false)
}
}
protected fun onException(e: Exception): Boolean {
println("There was an exception during processing: $e")
System.err.println("There was an exception during processing: $e")
if (this.verbose) {
e.printStackTrace()
e.printStackTrace(System.err)
}
return this.getYesNo("Do you wish to try again?", false)
}
override fun getOutputDir(): Path? = myOutputDir
override fun getOutputDir() = myOutputDir
override fun transform(target: Path): Int {
println("Processing $target")
if (interactive) {
println("Tokenizing $target")
}
var code = 0
while (true) {
try {
@ -99,7 +97,7 @@ open class TokenizeWvc : AbstractSourceTransformerCli() {
val errorListener = LexerErrorListener()
lexer.addErrorListener(errorListener)
val tokenStream = WebViewComponentsTokenStream(lexer, setOf(WebViewComponentsLexer.HIDDEN))
val tokenStream = WebViewComponentsTokenStream(lexer, setOf()) // include everything (ignore nothing)
val allTokens = tokenStream.getAllTokensSkipEOF()
val errors = errorListener.getErrors()