Basic lexer input-output comparison tests.

web-view-components-compiler/build.gradle

@@ -151,7 +151,8 @@ final List<ToolSpec> toolSpecs = [
         toolSpec('groovyWvc', 'GroovyWvcCompiler'),
         toolSpec('lexer', 'LexerTool'),
         toolSpec('parser', 'ParserTool'),
-        toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli')
+        toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'),
+        toolSpec('tokensFileMaker', 'TokensFileMakerCli')
 ]
 
 toolSpecs.each { spec ->

web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt

@@ -190,6 +190,11 @@ class WebViewComponentsTokenStream (private val tokenSource: TokenSource) : Toke
         return this.tokens
     }
 
+    fun getAllTokensSkipEOF(): List<Token> {
+        this.fill()
+        return this.tokens.filter { it.type != Token.EOF }
+    }
+
     private fun fill() {
         val oldIndex = this.currentIndex
         this.initialize()

web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java

@@ -1,9 +1,17 @@
 package groowt.view.component.web.antlr;
 
+import groowt.view.component.web.testutil.FileComparisonTestUtil;
+import groowt.view.component.web.util.ExtensionUtil;
+import groowt.view.component.web.util.FileUtil;
+import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.Token;
+import org.junit.jupiter.api.DynamicTest;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestFactory;
 
+import java.nio.file.Path;
+import java.util.Collection;
 import java.util.stream.Collectors;
 
 import static groowt.view.component.web.antlr.WebViewComponentsLexer.*;
@@ -48,4 +56,25 @@ public class WebViewComponentsLexerTests {
         assertTokenType(EOF, t4);
     }
 
+    @TestFactory
+    public Collection<DynamicTest> lexerFileTests() {
+        return FileComparisonTestUtil.getTestsFor(
+                Path.of("src", "test", "lexer"),
+                "*.wvc",
+                Path.of("src", "test", "lexer", "tokens-files"),
+                sourcePath -> {
+                    final String nameWithoutExtension = ExtensionUtil.getNameWithoutExtension(sourcePath);
+                    return Path.of(nameWithoutExtension + "_tokens.txt");
+                },
+                sourceFile -> {
+                    final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile));
+                    final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input);
+                    final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer);
+                    return tokenStream.getAllTokensSkipEOF().stream()
+                            .map(TokenUtil::formatToken)
+                            .collect(Collectors.joining("\n"));
+                }
+        );
+    }
+
 }

web-view-components-compiler/src/test/lexer/helloTarget.wvc

@@ -0,0 +1 @@
+Hello, $target!

web-view-components-compiler/src/test/lexer/helloWorld.wvc

@@ -0,0 +1 @@
+Hello, World!

web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt

@@ -0,0 +1,4 @@
+RawText[1,0](Hello, )
+DollarReferenceStart[1,7]($)
+GroovyCode[1,8](target)
+RawText[1,14](!\n)

web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt

@@ -0,0 +1 @@
+RawText[1,0](Hello, World!\n)

web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractOutputFileMaker.groovy

@@ -1,10 +1,6 @@
 package groowt.view.component.web.tools
 
-import java.util.regex.Pattern
+abstract class AbstractOutputFileMaker implements SourceFileProcessor {
-
-abstract class AbstractTreeFileMaker implements SourceFileProcessor {
-
-    private static final Pattern withoutExtension = ~/(?<name>.*)\.(?<ext>.+)/
 
     private final Scanner scanner = new Scanner(System.in)
 
@@ -15,15 +11,6 @@ abstract class AbstractTreeFileMaker implements SourceFileProcessor {
     boolean autoYes
     boolean verbose
 
-    protected String getNameWithoutExtension(File file) {
-        def m = withoutExtension.matcher(file.name)
-        if (m.matches()) {
-            return m.group('name')
-        } else {
-            throw new IllegalArgumentException("Could not determine file name without extension for ${file}")
-        }
-    }
-
     protected boolean getYesNoInput(String prompt, boolean force = false) {
         if (this.autoYes && !force) {
             return true
@@ -40,4 +27,20 @@ abstract class AbstractTreeFileMaker implements SourceFileProcessor {
         }
     }
 
+    protected void writeToDisk(String name, String formatted) {
+        this.outputDirectory.mkdirs()
+        def out = new File(this.outputDirectory, name + this.suffix + this.extension)
+        if (out.exists()) {
+            if (this.getYesNoInput("${out} already exists. Write over? (y/n)")) {
+                println "Writing to $out..."
+                out.write(formatted)
+            } else {
+                println "Skipping writing to $out."
+            }
+        } else {
+            println "Writing to $out..."
+            out.write(formatted)
+        }
+    }
+
 }

web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AstFileMaker.groovy

@@ -11,10 +11,11 @@ import groowt.view.component.web.ast.DefaultAstBuilder
 import groowt.view.component.web.ast.DefaultNodeFactory
 import groowt.view.component.web.ast.NodeUtil
 import groowt.view.component.web.ast.node.Node
+import groowt.view.component.web.util.ExtensionUtil
 import org.jetbrains.annotations.Nullable
 
 @InheritConstructors
-final class AstFileMaker extends AbstractTreeFileMaker {
+final class AstFileMaker extends AbstractOutputFileMaker {
 
     protected sealed interface BuildResult permits BuildSuccess, BuildFailure {}
 
@@ -31,22 +32,6 @@ final class AstFileMaker extends AbstractTreeFileMaker {
         @Nullable String message
     }
 
-    private void writeFormatted(String name, String formatted) {
-        this.outputDirectory.mkdirs()
-        def outFile = new File(this.outputDirectory, name + this.suffix + this.extension)
-        if (outFile.exists()) {
-            if (this.getYesNoInput("$outFile already exists. Write over? (y/n)")) {
-                println "Writing to $outFile..."
-                outFile.write(formatted)
-            } else {
-                println "Skipping writing to $outFile."
-            }
-        } else {
-            println "Writing to $outFile..."
-            outFile.write(formatted)
-        }
-    }
-
     private boolean onSuccess(String name, BuildSuccess buildSuccess) {
         def formatted = NodeUtil.formatAst(buildSuccess.node, buildSuccess.tokenList)
         if (!this.autoYes) {
@@ -54,7 +39,7 @@ final class AstFileMaker extends AbstractTreeFileMaker {
             println formatted
         }
         if (this.getYesNoInput('Would you like to write to disk? (y/n)')) {
-            this.writeFormatted(name, formatted)
+            this.writeToDisk(name, formatted)
             return true
         } else {
             return !this.getYesNoInput('Do you wish to redo this file? (y/n)')
@@ -117,7 +102,7 @@ final class AstFileMaker extends AbstractTreeFileMaker {
 
     @Override
     void process(File sourceFile) {
-        def name = this.getNameWithoutExtension(sourceFile)
+        def name = ExtensionUtil.getNameWithoutExtension(sourceFile)
         println "Processing $name"
         boolean doneYet = false
         while (!doneYet) {

web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/ParseTreeFileMaker.groovy

@@ -3,31 +3,11 @@ package groowt.view.component.web.tools
 import groovy.transform.InheritConstructors
 import groowt.view.component.web.antlr.*
 import groowt.view.component.web.antlr.AntlrUtil.ParseErrorCollector
+import groowt.view.component.web.util.ExtensionUtil
 import org.antlr.v4.runtime.CharStreams
 
 @InheritConstructors
-final class ParseTreeFileMaker extends AbstractTreeFileMaker {
+final class ParseTreeFileMaker extends AbstractOutputFileMaker {
-
-    private void writeFormatted(
-            String name,
-            WebViewComponentsParser parser,
-            WebViewComponentsParser.CompilationUnitContext cu
-    ) {
-        this.outputDirectory.mkdirs()
-        def formatted = ParserUtil.formatTree(parser, cu, false)
-        def out = new File(this.outputDirectory, name + this.suffix + this.extension)
-        if (out.exists()) {
-            if (this.getYesNoInput("${out} already exists. Write over? (y/n)")) {
-                println "Writing to $out..."
-                out.write(formatted)
-            } else {
-                println "Skipping writing to $out."
-            }
-        } else {
-            println "Writing to $out..."
-            out.write(formatted)
-        }
-    }
 
     /**
      * @return true if done now, false if not done yet
@@ -42,7 +22,8 @@ final class ParseTreeFileMaker extends AbstractTreeFileMaker {
             println ParserUtil.formatTree(parser, cu, true)
         }
         if (this.getYesNoInput('Write to disk? (y/n)')) {
-            this.writeFormatted(name, parser, cu)
+            def formatted = ParserUtil.formatTree(parser, cu, false)
+            this.writeToDisk(name, formatted)
             return true
         } else {
             return !this.getYesNoInput('Do you wish to redo this file? (y/n)')
@@ -86,8 +67,8 @@ final class ParseTreeFileMaker extends AbstractTreeFileMaker {
 
     @Override
     void process(File sourceFile) {
-        def name = this.getNameWithoutExtension(sourceFile)
+        def name = ExtensionUtil.getNameWithoutExtension(sourceFile)
-        println "processing: $name"
+        println "Processing: $name"
         boolean doneYet = false
         while (!doneYet) {
             def (parser, cu, errors) = this.parse(sourceFile)

web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMaker.groovy

@@ -0,0 +1,57 @@
+package groowt.view.component.web.tools
+
+import groovy.transform.InheritConstructors
+import groowt.view.component.web.antlr.TokenUtil
+import groowt.view.component.web.antlr.WebViewComponentsLexer
+import groowt.view.component.web.antlr.WebViewComponentsTokenStream
+import groowt.view.component.web.util.ExtensionUtil
+import org.antlr.v4.runtime.CharStreams
+import org.antlr.v4.runtime.Token
+
+@InheritConstructors
+class TokensFileMaker extends AbstractOutputFileMaker {
+
+    protected boolean onSuccess(String name, List<Token> allTokens) {
+        def formatted = allTokens.collect(TokenUtil.&formatToken).join('\n')
+        if (!this.autoYes) {
+            println 'Please review the following tokens:'
+            println formatted
+        }
+        if (this.getYesNoInput('Write to disk? (y/n)')) {
+            this.writeToDisk(name, formatted)
+            return true
+        } else {
+            return !this.getYesNoInput('Do you wish to redo this file? (y/n)')
+        }
+    }
+
+    protected boolean onException(String name, Exception e) {
+        println "There was an exception while tokenizing $name: $e.message"
+        e.printStackTrace()
+        if (this.getYesNoInput('Do you wish to try again? (y/n)', true)) {
+            println "Trying $name again..."
+            return false
+        } else {
+            println "Skipping $name."
+            return true
+        }
+    }
+
+    @Override
+    void process(File sourceFile) {
+        def name = ExtensionUtil.getNameWithoutExtension(sourceFile)
+        println "Processing: $name"
+        boolean doneYet = false
+        while (!doneYet) {
+            try {
+                def input = CharStreams.fromString(sourceFile.getText())
+                def lexer = new WebViewComponentsLexer(input)
+                def tokenStream = new WebViewComponentsTokenStream(lexer)
+                doneYet = this.onSuccess(name, tokenStream.getAllTokensSkipEOF())
+            } catch (Exception e) {
+                doneYet = this.onException(name, e)
+            }
+        }
+    }
+
+}

web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMakerCli.groovy

@@ -0,0 +1,29 @@
+package groowt.view.component.web.tools
+
+import picocli.CommandLine
+
+@CommandLine.Command(
+        name = 'tokensFileMaker',
+        description = 'Tokenize given input files and output files containing their tokens.',
+        mixinStandardHelpOptions = true
+)
+class TokensFileMakerCli extends SourceFileProcessorSpec {
+
+    static void main(String[] args) {
+        System.exit(new CommandLine(new TokensFileMakerCli()).execute(args))
+    }
+
+    TokensFileMakerCli() {
+        super({ SourceFileProcessorSpec spec ->
+            new TokensFileMaker(
+                    dryRun: spec.dryRun,
+                    suffix: spec.suffix.orElse('_tokens'),
+                    extension: spec.extension,
+                    outputDirectory: spec.outputDirectory.orElse(new File('src/test/lexer/tokens-files')),
+                    autoYes: spec.autoYes,
+                    verbose: spec.verbose
+            )
+        }, 'src/test/lexer')
+    }
+
+}