jessebrault/groowt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Basic lexer input-output comparison tests.

Browse Source
This commit is contained in:
JesseBrault0709 2024-05-20 20:02:48 +02:00
parent d099f9514d
commit e747ac3c32
12 changed files with 156 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
web-view-components-compiler/build.gradle
View File

@ -151,7 +151,8 @@ final List<ToolSpec> toolSpecs = [
toolSpec('groovyWvc', 'GroovyWvcCompiler'),
toolSpec('lexer', 'LexerTool'),
toolSpec('parser', 'ParserTool'),
toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli')
toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'),
toolSpec('tokensFileMaker', 'TokensFileMakerCli')
]
toolSpecs.each { spec ->

5
web-view-components-compiler/src/main/java/groowt/view/component/web/antlr/WebViewComponentsTokenStream.kt
View File

@ -190,6 +190,11 @@ class WebViewComponentsTokenStream (private val tokenSource: TokenSource) : Toke
return this.tokens
}
fun getAllTokensSkipEOF(): List<Token> {
this.fill()
return this.tokens.filter { it.type != Token.EOF }
}
private fun fill() {
val oldIndex = this.currentIndex
this.initialize()

29
web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java
View File

@ -1,9 +1,17 @@
package groowt.view.component.web.antlr;
import groowt.view.component.web.testutil.FileComparisonTestUtil;
import groowt.view.component.web.util.ExtensionUtil;
import groowt.view.component.web.util.FileUtil;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.Token;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import java.nio.file.Path;
import java.util.Collection;
import java.util.stream.Collectors;
import static groowt.view.component.web.antlr.WebViewComponentsLexer.*;
@ -48,4 +56,25 @@ public class WebViewComponentsLexerTests {
assertTokenType(EOF, t4);
}
@TestFactory
public Collection<DynamicTest> lexerFileTests() {
return FileComparisonTestUtil.getTestsFor(
Path.of("src", "test", "lexer"),
"*.wvc",
Path.of("src", "test", "lexer", "tokens-files"),
sourcePath -> {
final String nameWithoutExtension = ExtensionUtil.getNameWithoutExtension(sourcePath);
return Path.of(nameWithoutExtension + "_tokens.txt");
},
sourceFile -> {
final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile));
final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input);
final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer);
return tokenStream.getAllTokensSkipEOF().stream()
.map(TokenUtil::formatToken)
.collect(Collectors.joining("\n"));
}
);
}
}

1
web-view-components-compiler/src/test/lexer/helloTarget.wvc Normal file
View File

@ -0,0 +1 @@
Hello, $target!

1
web-view-components-compiler/src/test/lexer/helloWorld.wvc Normal file
View File

@ -0,0 +1 @@
Hello, World!

4
web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt Normal file
View File

@ -0,0 +1,4 @@
RawText[1,0](Hello, )
DollarReferenceStart[1,7]($)
GroovyCode[1,8](target)
RawText[1,14](!\n)

1
web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt Normal file
View File

@ -0,0 +1 @@
RawText[1,0](Hello, World!\n)

31
web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractTreeFileMaker.groovy → web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AbstractOutputFileMaker.groovy
View File

@ -1,10 +1,6 @@
package groowt.view.component.web.tools
import java.util.regex.Pattern
abstract class AbstractTreeFileMaker implements SourceFileProcessor {
private static final Pattern withoutExtension = ~/(?<name>.*)\.(?<ext>.+)/
abstract class AbstractOutputFileMaker implements SourceFileProcessor {
private final Scanner scanner = new Scanner(System.in)
@ -15,15 +11,6 @@ abstract class AbstractTreeFileMaker implements SourceFileProcessor {
boolean autoYes
boolean verbose
protected String getNameWithoutExtension(File file) {
def m = withoutExtension.matcher(file.name)
if (m.matches()) {
return m.group('name')
} else {
throw new IllegalArgumentException("Could not determine file name without extension for ${file}")
}
}
protected boolean getYesNoInput(String prompt, boolean force = false) {
if (this.autoYes && !force) {
return true
@ -40,4 +27,20 @@ abstract class AbstractTreeFileMaker implements SourceFileProcessor {
}
}
protected void writeToDisk(String name, String formatted) {
this.outputDirectory.mkdirs()
def out = new File(this.outputDirectory, name + this.suffix + this.extension)
if (out.exists()) {
if (this.getYesNoInput("${out} already exists. Write over? (y/n)")) {
println "Writing to $out..."
out.write(formatted)
} else {
println "Skipping writing to $out."
}
} else {
println "Writing to $out..."
out.write(formatted)
}
}
}

23
web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/AstFileMaker.groovy
View File

@ -11,10 +11,11 @@ import groowt.view.component.web.ast.DefaultAstBuilder
import groowt.view.component.web.ast.DefaultNodeFactory
import groowt.view.component.web.ast.NodeUtil
import groowt.view.component.web.ast.node.Node
import groowt.view.component.web.util.ExtensionUtil
import org.jetbrains.annotations.Nullable
@InheritConstructors
final class AstFileMaker extends AbstractTreeFileMaker {
final class AstFileMaker extends AbstractOutputFileMaker {
protected sealed interface BuildResult permits BuildSuccess, BuildFailure {}
@ -31,22 +32,6 @@ final class AstFileMaker extends AbstractTreeFileMaker {
@Nullable String message
}
private void writeFormatted(String name, String formatted) {
this.outputDirectory.mkdirs()
def outFile = new File(this.outputDirectory, name + this.suffix + this.extension)
if (outFile.exists()) {
if (this.getYesNoInput("$outFile already exists. Write over? (y/n)")) {
println "Writing to $outFile..."
outFile.write(formatted)
} else {
println "Skipping writing to $outFile."
}
} else {
println "Writing to $outFile..."
outFile.write(formatted)
}
}
private boolean onSuccess(String name, BuildSuccess buildSuccess) {
def formatted = NodeUtil.formatAst(buildSuccess.node, buildSuccess.tokenList)
if (!this.autoYes) {
@ -54,7 +39,7 @@ final class AstFileMaker extends AbstractTreeFileMaker {
println formatted
}
if (this.getYesNoInput('Would you like to write to disk? (y/n)')) {
this.writeFormatted(name, formatted)
this.writeToDisk(name, formatted)
return true
} else {
return !this.getYesNoInput('Do you wish to redo this file? (y/n)')
@ -117,7 +102,7 @@ final class AstFileMaker extends AbstractTreeFileMaker {
@Override
void process(File sourceFile) {
def name = this.getNameWithoutExtension(sourceFile)
def name = ExtensionUtil.getNameWithoutExtension(sourceFile)
println "Processing $name"
boolean doneYet = false
while (!doneYet) {

31
web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/ParseTreeFileMaker.groovy
View File

@ -3,31 +3,11 @@ package groowt.view.component.web.tools
import groovy.transform.InheritConstructors
import groowt.view.component.web.antlr.*
import groowt.view.component.web.antlr.AntlrUtil.ParseErrorCollector
import groowt.view.component.web.util.ExtensionUtil
import org.antlr.v4.runtime.CharStreams
@InheritConstructors
final class ParseTreeFileMaker extends AbstractTreeFileMaker {
private void writeFormatted(
String name,
WebViewComponentsParser parser,
WebViewComponentsParser.CompilationUnitContext cu
) {
this.outputDirectory.mkdirs()
def formatted = ParserUtil.formatTree(parser, cu, false)
def out = new File(this.outputDirectory, name + this.suffix + this.extension)
if (out.exists()) {
if (this.getYesNoInput("${out} already exists. Write over? (y/n)")) {
println "Writing to $out..."
out.write(formatted)
} else {
println "Skipping writing to $out."
}
} else {
println "Writing to $out..."
out.write(formatted)
}
}
final class ParseTreeFileMaker extends AbstractOutputFileMaker {
/**
* @return true if done now, false if not done yet
@ -42,7 +22,8 @@ final class ParseTreeFileMaker extends AbstractTreeFileMaker {
println ParserUtil.formatTree(parser, cu, true)
}
if (this.getYesNoInput('Write to disk? (y/n)')) {
this.writeFormatted(name, parser, cu)
def formatted = ParserUtil.formatTree(parser, cu, false)
this.writeToDisk(name, formatted)
return true
} else {
return !this.getYesNoInput('Do you wish to redo this file? (y/n)')
@ -86,8 +67,8 @@ final class ParseTreeFileMaker extends AbstractTreeFileMaker {
@Override
void process(File sourceFile) {
def name = this.getNameWithoutExtension(sourceFile)
println "processing: $name"
def name = ExtensionUtil.getNameWithoutExtension(sourceFile)
println "Processing: $name"
boolean doneYet = false
while (!doneYet) {
def (parser, cu, errors) = this.parse(sourceFile)

57
web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMaker.groovy Normal file
View File

@ -0,0 +1,57 @@
package groowt.view.component.web.tools
import groovy.transform.InheritConstructors
import groowt.view.component.web.antlr.TokenUtil
import groowt.view.component.web.antlr.WebViewComponentsLexer
import groowt.view.component.web.antlr.WebViewComponentsTokenStream
import groowt.view.component.web.util.ExtensionUtil
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.Token
@InheritConstructors
class TokensFileMaker extends AbstractOutputFileMaker {
protected boolean onSuccess(String name, List<Token> allTokens) {
def formatted = allTokens.collect(TokenUtil.&formatToken).join('\n')
if (!this.autoYes) {
println 'Please review the following tokens:'
println formatted
}
if (this.getYesNoInput('Write to disk? (y/n)')) {
this.writeToDisk(name, formatted)
return true
} else {
return !this.getYesNoInput('Do you wish to redo this file? (y/n)')
}
}
protected boolean onException(String name, Exception e) {
println "There was an exception while tokenizing $name: $e.message"
e.printStackTrace()
if (this.getYesNoInput('Do you wish to try again? (y/n)', true)) {
println "Trying $name again..."
return false
} else {
println "Skipping $name."
return true
}
}
@Override
void process(File sourceFile) {
def name = ExtensionUtil.getNameWithoutExtension(sourceFile)
println "Processing: $name"
boolean doneYet = false
while (!doneYet) {
try {
def input = CharStreams.fromString(sourceFile.getText())
def lexer = new WebViewComponentsLexer(input)
def tokenStream = new WebViewComponentsTokenStream(lexer)
doneYet = this.onSuccess(name, tokenStream.getAllTokensSkipEOF())
} catch (Exception e) {
doneYet = this.onException(name, e)
}
}
}
}

29
web-view-components-compiler/src/tools/groovy/groowt/view/component/web/tools/TokensFileMakerCli.groovy Normal file
View File

@ -0,0 +1,29 @@
package groowt.view.component.web.tools
import picocli.CommandLine
@CommandLine.Command(
name = 'tokensFileMaker',
description = 'Tokenize given input files and output files containing their tokens.',
mixinStandardHelpOptions = true
)
class TokensFileMakerCli extends SourceFileProcessorSpec {
static void main(String[] args) {
System.exit(new CommandLine(new TokensFileMakerCli()).execute(args))
}
TokensFileMakerCli() {
super({ SourceFileProcessorSpec spec ->
new TokensFileMaker(
dryRun: spec.dryRun,
suffix: spec.suffix.orElse('_tokens'),
extension: spec.extension,
outputDirectory: spec.outputDirectory.orElse(new File('src/test/lexer/tokens-files')),
autoYes: spec.autoYes,
verbose: spec.verbose
)
}, 'src/test/lexer')
}
}