diff --git a/web-view-components-compiler/build.gradle b/web-view-components-compiler/build.gradle index 7609c62..7d6111b 100644 --- a/web-view-components-compiler/build.gradle +++ b/web-view-components-compiler/build.gradle @@ -151,7 +151,8 @@ final List toolSpecs = [ toolSpec('groovyWvc', 'GroovyWvcCompiler'), toolSpec('lexer', 'LexerTool'), toolSpec('parser', 'ParserTool'), - toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli') + toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'), + toolSpec('tokensFileMaker', 'TokensFileMakerCli') ] toolSpecs.each { spec -> diff --git a/web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt b/web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt index c219735..d5dfa96 100644 --- a/web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt +++ b/web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt @@ -190,6 +190,11 @@ class WebViewComponentsTokenStream (private val tokenSource: TokenSource) : Toke return this.tokens } + fun getAllTokensSkipEOF(): List { + this.fill() + return this.tokens.filter { it.type != Token.EOF } + } + private fun fill() { val oldIndex = this.currentIndex this.initialize() diff --git a/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java b/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java index b9842ac..4b104c3 100644 --- a/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java +++ b/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java @@ -1,9 +1,17 @@ package groowt.view.component.web.antlr; +import groowt.view.component.web.testutil.FileComparisonTestUtil; +import groowt.view.component.web.util.ExtensionUtil; +import groowt.view.component.web.util.FileUtil; +import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.Token; +import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import java.nio.file.Path; +import java.util.Collection; import java.util.stream.Collectors; import static groowt.view.component.web.antlr.WebViewComponentsLexer.*; @@ -48,4 +56,25 @@ public class WebViewComponentsLexerTests { assertTokenType(EOF, t4); } + @TestFactory + public Collection lexerFileTests() { + return FileComparisonTestUtil.getTestsFor( + Path.of("src", "test", "lexer"), + "*.wvc", + Path.of("src", "test", "lexer", "tokens-files"), + sourcePath -> { + final String nameWithoutExtension = ExtensionUtil.getNameWithoutExtension(sourcePath); + return Path.of(nameWithoutExtension + "_tokens.txt"); + }, + sourceFile -> { + final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile)); + final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input); + final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer); + return tokenStream.getAllTokensSkipEOF().stream() + .map(TokenUtil::formatToken) + .collect(Collectors.joining("\n")); + } + ); + } + } diff --git a/web-view-components-compiler/src/test/lexer/helloTarget.wvc b/web-view-components-compiler/src/test/lexer/helloTarget.wvc new file mode 100644 index 0000000..a9c4d42 --- /dev/null +++ b/web-view-components-compiler/src/test/lexer/helloTarget.wvc @@ -0,0 +1 @@ +Hello, $target! diff --git a/web-view-components-compiler/src/test/lexer/helloWorld.wvc b/web-view-components-compiler/src/test/lexer/helloWorld.wvc new file mode 100644 index 0000000..8ab686e --- /dev/null +++ b/web-view-components-compiler/src/test/lexer/helloWorld.wvc @@ -0,0 +1 @@ +Hello, World! diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt new file mode 100644 index 0000000..3128441 --- /dev/null +++ b/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt @@ -0,0 +1,4 @@ +RawText[1,0](Hello, ) +DollarReferenceStart[1,7]($) +GroovyCode[1,8](target) +RawText[1,14](!\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt new file mode 100644 index 0000000..d28150d --- /dev/null +++ b/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt @@ -0,0 +1 @@ +RawText[1,0](Hello, World!\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractTreeFileMaker.groovy b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractOutputFileMaker.groovy similarity index 54% rename from web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractTreeFileMaker.groovy rename to web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractOutputFileMaker.groovy index 8fc79c1..47acae1 100644 --- a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractTreeFileMaker.groovy +++ b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractOutputFileMaker.groovy @@ -1,10 +1,6 @@ package groowt.view.component.web.tools -import java.util.regex.Pattern - -abstract class AbstractTreeFileMaker implements SourceFileProcessor { - - private static final Pattern withoutExtension = ~/(?.*)\.(?.+)/ +abstract class AbstractOutputFileMaker implements SourceFileProcessor { private final Scanner scanner = new Scanner(System.in) @@ -15,15 +11,6 @@ abstract class AbstractTreeFileMaker implements SourceFileProcessor { boolean autoYes boolean verbose - protected String getNameWithoutExtension(File file) { - def m = withoutExtension.matcher(file.name) - if (m.matches()) { - return m.group('name') - } else { - throw new IllegalArgumentException("Could not determine file name without extension for ${file}") - } - } - protected boolean getYesNoInput(String prompt, boolean force = false) { if (this.autoYes && !force) { return true @@ -40,4 +27,20 @@ abstract class AbstractTreeFileMaker implements SourceFileProcessor { } } + protected void writeToDisk(String name, String formatted) { + this.outputDirectory.mkdirs() + def out = new File(this.outputDirectory, name + this.suffix + this.extension) + if (out.exists()) { + if (this.getYesNoInput("${out} already exists. Write over? (y/n)")) { + println "Writing to $out..." + out.write(formatted) + } else { + println "Skipping writing to $out." + } + } else { + println "Writing to $out..." + out.write(formatted) + } + } + } diff --git a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AstFileMaker.groovy b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AstFileMaker.groovy index a62793d..6704f1b 100644 --- a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AstFileMaker.groovy +++ b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AstFileMaker.groovy @@ -11,10 +11,11 @@ import groowt.view.component.web.ast.DefaultAstBuilder import groowt.view.component.web.ast.DefaultNodeFactory import groowt.view.component.web.ast.NodeUtil import groowt.view.component.web.ast.node.Node +import groowt.view.component.web.util.ExtensionUtil import org.jetbrains.annotations.Nullable @InheritConstructors -final class AstFileMaker extends AbstractTreeFileMaker { +final class AstFileMaker extends AbstractOutputFileMaker { protected sealed interface BuildResult permits BuildSuccess, BuildFailure {} @@ -31,22 +32,6 @@ final class AstFileMaker extends AbstractTreeFileMaker { @Nullable String message } - private void writeFormatted(String name, String formatted) { - this.outputDirectory.mkdirs() - def outFile = new File(this.outputDirectory, name + this.suffix + this.extension) - if (outFile.exists()) { - if (this.getYesNoInput("$outFile already exists. Write over? (y/n)")) { - println "Writing to $outFile..." - outFile.write(formatted) - } else { - println "Skipping writing to $outFile." - } - } else { - println "Writing to $outFile..." - outFile.write(formatted) - } - } - private boolean onSuccess(String name, BuildSuccess buildSuccess) { def formatted = NodeUtil.formatAst(buildSuccess.node, buildSuccess.tokenList) if (!this.autoYes) { @@ -54,7 +39,7 @@ final class AstFileMaker extends AbstractTreeFileMaker { println formatted } if (this.getYesNoInput('Would you like to write to disk? (y/n)')) { - this.writeFormatted(name, formatted) + this.writeToDisk(name, formatted) return true } else { return !this.getYesNoInput('Do you wish to redo this file? (y/n)') @@ -117,7 +102,7 @@ final class AstFileMaker extends AbstractTreeFileMaker { @Override void process(File sourceFile) { - def name = this.getNameWithoutExtension(sourceFile) + def name = ExtensionUtil.getNameWithoutExtension(sourceFile) println "Processing $name" boolean doneYet = false while (!doneYet) { diff --git a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/ParseTreeFileMaker.groovy b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/ParseTreeFileMaker.groovy index c0b3be4..7e81b34 100644 --- a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/ParseTreeFileMaker.groovy +++ b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/ParseTreeFileMaker.groovy @@ -3,31 +3,11 @@ package groowt.view.component.web.tools import groovy.transform.InheritConstructors import groowt.view.component.web.antlr.* import groowt.view.component.web.antlr.AntlrUtil.ParseErrorCollector +import groowt.view.component.web.util.ExtensionUtil import org.antlr.v4.runtime.CharStreams @InheritConstructors -final class ParseTreeFileMaker extends AbstractTreeFileMaker { - - private void writeFormatted( - String name, - WebViewComponentsParser parser, - WebViewComponentsParser.CompilationUnitContext cu - ) { - this.outputDirectory.mkdirs() - def formatted = ParserUtil.formatTree(parser, cu, false) - def out = new File(this.outputDirectory, name + this.suffix + this.extension) - if (out.exists()) { - if (this.getYesNoInput("${out} already exists. Write over? (y/n)")) { - println "Writing to $out..." - out.write(formatted) - } else { - println "Skipping writing to $out." - } - } else { - println "Writing to $out..." - out.write(formatted) - } - } +final class ParseTreeFileMaker extends AbstractOutputFileMaker { /** * @return true if done now, false if not done yet @@ -42,7 +22,8 @@ final class ParseTreeFileMaker extends AbstractTreeFileMaker { println ParserUtil.formatTree(parser, cu, true) } if (this.getYesNoInput('Write to disk? (y/n)')) { - this.writeFormatted(name, parser, cu) + def formatted = ParserUtil.formatTree(parser, cu, false) + this.writeToDisk(name, formatted) return true } else { return !this.getYesNoInput('Do you wish to redo this file? (y/n)') @@ -86,8 +67,8 @@ final class ParseTreeFileMaker extends AbstractTreeFileMaker { @Override void process(File sourceFile) { - def name = this.getNameWithoutExtension(sourceFile) - println "processing: $name" + def name = ExtensionUtil.getNameWithoutExtension(sourceFile) + println "Processing: $name" boolean doneYet = false while (!doneYet) { def (parser, cu, errors) = this.parse(sourceFile) diff --git a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMaker.groovy b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMaker.groovy new file mode 100644 index 0000000..fc7fe93 --- /dev/null +++ b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMaker.groovy @@ -0,0 +1,57 @@ +package groowt.view.component.web.tools + +import groovy.transform.InheritConstructors +import groowt.view.component.web.antlr.TokenUtil +import groowt.view.component.web.antlr.WebViewComponentsLexer +import groowt.view.component.web.antlr.WebViewComponentsTokenStream +import groowt.view.component.web.util.ExtensionUtil +import org.antlr.v4.runtime.CharStreams +import org.antlr.v4.runtime.Token + +@InheritConstructors +class TokensFileMaker extends AbstractOutputFileMaker { + + protected boolean onSuccess(String name, List allTokens) { + def formatted = allTokens.collect(TokenUtil.&formatToken).join('\n') + if (!this.autoYes) { + println 'Please review the following tokens:' + println formatted + } + if (this.getYesNoInput('Write to disk? (y/n)')) { + this.writeToDisk(name, formatted) + return true + } else { + return !this.getYesNoInput('Do you wish to redo this file? (y/n)') + } + } + + protected boolean onException(String name, Exception e) { + println "There was an exception while tokenizing $name: $e.message" + e.printStackTrace() + if (this.getYesNoInput('Do you wish to try again? (y/n)', true)) { + println "Trying $name again..." + return false + } else { + println "Skipping $name." + return true + } + } + + @Override + void process(File sourceFile) { + def name = ExtensionUtil.getNameWithoutExtension(sourceFile) + println "Processing: $name" + boolean doneYet = false + while (!doneYet) { + try { + def input = CharStreams.fromString(sourceFile.getText()) + def lexer = new WebViewComponentsLexer(input) + def tokenStream = new WebViewComponentsTokenStream(lexer) + doneYet = this.onSuccess(name, tokenStream.getAllTokensSkipEOF()) + } catch (Exception e) { + doneYet = this.onException(name, e) + } + } + } + +} diff --git a/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMakerCli.groovy b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMakerCli.groovy new file mode 100644 index 0000000..5fd3a12 --- /dev/null +++ b/web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMakerCli.groovy @@ -0,0 +1,29 @@ +package groowt.view.component.web.tools + +import picocli.CommandLine + +@CommandLine.Command( + name = 'tokensFileMaker', + description = 'Tokenize given input files and output files containing their tokens.', + mixinStandardHelpOptions = true +) +class TokensFileMakerCli extends SourceFileProcessorSpec { + + static void main(String[] args) { + System.exit(new CommandLine(new TokensFileMakerCli()).execute(args)) + } + + TokensFileMakerCli() { + super({ SourceFileProcessorSpec spec -> + new TokensFileMaker( + dryRun: spec.dryRun, + suffix: spec.suffix.orElse('_tokens'), + extension: spec.extension, + outputDirectory: spec.outputDirectory.orElse(new File('src/test/lexer/tokens-files')), + autoYes: spec.autoYes, + verbose: spec.verbose + ) + }, 'src/test/lexer') + } + +}