From 1b6344dbf2c371ad08ba728d4d7d4ba6cd843562 Mon Sep 17 00:00:00 2001 From: JesseBrault0709 <62299747+JesseBrault0709@users.noreply.github.com> Date: Tue, 21 May 2024 12:31:26 +0200 Subject: [PATCH] Rewrote tokenize tool in kotlin. --- web-view-components-compiler/build.gradle | 1 + .../antlr/WebViewComponentsLexerTests.java | 13 +- .../lexer/tokens-files/complicated_tokens.txt | 166 +++++++++--------- .../tokens-files/dataComponent_tokens.txt | 10 +- .../lexer/tokens-files/emptyHtml_tokens.txt | 14 +- .../emptyTypedWithPackage_tokens.txt | 14 +- .../lexer/tokens-files/helloTarget_tokens.txt | 8 +- .../lexer/tokens-files/helloWorld_tokens.txt | 2 +- .../web/tools/AbstractSourceTransformerCli.kt | 98 +++++++++++ .../view/component/web/tools/TokenizeWvc.kt | 129 ++++++++++++++ 10 files changed, 345 insertions(+), 110 deletions(-) create mode 100644 web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/AbstractSourceTransformerCli.kt create mode 100644 web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/TokenizeWvc.kt diff --git a/web-view-components-compiler/build.gradle b/web-view-components-compiler/build.gradle index 7d6111b..ee538f5 100644 --- a/web-view-components-compiler/build.gradle +++ b/web-view-components-compiler/build.gradle @@ -152,6 +152,7 @@ final List toolSpecs = [ toolSpec('lexer', 'LexerTool'), toolSpec('parser', 'ParserTool'), toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'), + toolSpec('tokenizeWvc', 'TokenizeWvc'), toolSpec('tokensFileMaker', 'TokensFileMakerCli') ] diff --git a/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java b/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java index 4b104c3..94a1db2 100644 --- a/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java +++ b/web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.TestFactory; import java.nio.file.Path; import java.util.Collection; +import java.util.List; import java.util.stream.Collectors; import static groowt.view.component.web.antlr.WebViewComponentsLexer.*; @@ -70,9 +71,15 @@ public class WebViewComponentsLexerTests { final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile)); final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input); final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer); - return tokenStream.getAllTokensSkipEOF().stream() - .map(TokenUtil::formatToken) - .collect(Collectors.joining("\n")); + final List allTokens = tokenStream.getAllTokensSkipEOF(); + final var sb = new StringBuilder(); + for (int i = 0; i < allTokens.size(); i++) { + sb.append(i).append(": ").append(TokenUtil.formatToken(allTokens.get(i))); + if (i < allTokens.size() - 1) { + sb.append("\n"); + } + } + return sb.toString(); } ); } diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/complicated_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/complicated_tokens.txt index 07e1c23..06ccc0f 100644 --- a/web-view-components-compiler/src/test/lexer/tokens-files/complicated_tokens.txt +++ b/web-view-components-compiler/src/test/lexer/tokens-files/complicated_tokens.txt @@ -1,83 +1,83 @@ -PreambleBreak[1,1](---\n) -GroovyCode[2,1](import some.Thing // a comment...World!') -PreambleBreak[4,31](\n---\n) -RawText[6,1](\n) -ComponentOpen[7,1](<) -StringIdentifier[7,2](html) -ComponentClose[7,6](>) -RawText[7,7](\n ) -ComponentOpen[8,5](<) -StringIdentifier[8,6](head) -ComponentClose[8,10](>) -ClosingComponentOpen[8,11]() -RawText[8,18](\n ) -ComponentOpen[9,5](<) -StringIdentifier[9,6](body) -ComponentClose[9,10](>) -RawText[9,11](\n ) -ComponentOpen[10,9](<) -StringIdentifier[10,10](h1) -ComponentClose[10,12](>) -DollarScriptletOpen[10,13](${) -GroovyCode[10,15](greeting) -DollarScriptletClose[10,23](}) -ClosingComponentOpen[10,24]() -RawText[10,29](\n ) -ComponentOpen[11,9](<) -TypedIdentifier[11,10](groowt.view.web.Select) -ComponentClose[11,32](>) -RawText[11,33](\n ) -ComponentOpen[12,13](<) -TypedIdentifier[12,14](Case) -ComponentNlws[12,18]( ) -AttributeIdentifier[12,19](cond) -Equals[12,23](=) -ClosureAttrValueStart[12,24]({) -GroovyCode[12,25](isItTrue()) -ClosureAttrValueEnd[12,35](}) -ComponentClose[12,36](>) -RawText[12,37](\n ) -ComponentOpen[13,17](<) -StringIdentifier[13,18](p) -ComponentClose[13,19](>) -RawText[13,20](It's true! :)) -ClosingComponentOpen[13,33]() -RawText[13,37](\n ) -ClosingComponentOpen[14,13]() -RawText[14,20](\n ) -ComponentOpen[15,13](<) -TypedIdentifier[15,14](Default) -ComponentClose[15,21](>) -RawText[15,22](\n ) -ComponentOpen[16,17](<) -StringIdentifier[16,18](p) -ComponentClose[16,19](>) -RawText[16,20](It's false... :() -ClosingComponentOpen[16,36]() -RawText[16,40](\n ) -ClosingComponentOpen[17,13]() -RawText[17,23](\n ) -ClosingComponentOpen[18,9]() -RawText[18,34](\n ) -ClosingComponentOpen[19,5]() -RawText[19,12](\n) -ClosingComponentOpen[20,1]() -RawText[20,8](\n) \ No newline at end of file +0: PreambleBreak[1,1](---\n) +1: GroovyCode[2,1](import some.Thing // a comment...World!') +2: PreambleBreak[4,31](\n---\n) +3: RawText[6,1](\n) +4: ComponentOpen[7,1](<) +5: StringIdentifier[7,2](html) +6: ComponentClose[7,6](>) +7: RawText[7,7](\n ) +8: ComponentOpen[8,5](<) +9: StringIdentifier[8,6](head) +10: ComponentClose[8,10](>) +11: ClosingComponentOpen[8,11]() +14: RawText[8,18](\n ) +15: ComponentOpen[9,5](<) +16: StringIdentifier[9,6](body) +17: ComponentClose[9,10](>) +18: RawText[9,11](\n ) +19: ComponentOpen[10,9](<) +20: StringIdentifier[10,10](h1) +21: ComponentClose[10,12](>) +22: DollarScriptletOpen[10,13](${) +23: GroovyCode[10,15](greeting) +24: DollarScriptletClose[10,23](}) +25: ClosingComponentOpen[10,24]() +28: RawText[10,29](\n ) +29: ComponentOpen[11,9](<) +30: TypedIdentifier[11,10](groowt.view.web.Select) +31: ComponentClose[11,32](>) +32: RawText[11,33](\n ) +33: ComponentOpen[12,13](<) +34: TypedIdentifier[12,14](Case) +35: ComponentNlws[12,18]( ) +36: AttributeIdentifier[12,19](cond) +37: Equals[12,23](=) +38: ClosureAttrValueStart[12,24]({) +39: GroovyCode[12,25](isItTrue()) +40: ClosureAttrValueEnd[12,35](}) +41: ComponentClose[12,36](>) +42: RawText[12,37](\n ) +43: ComponentOpen[13,17](<) +44: StringIdentifier[13,18](p) +45: ComponentClose[13,19](>) +46: RawText[13,20](It's true! :)) +47: ClosingComponentOpen[13,33]() +50: RawText[13,37](\n ) +51: ClosingComponentOpen[14,13]() +54: RawText[14,20](\n ) +55: ComponentOpen[15,13](<) +56: TypedIdentifier[15,14](Default) +57: ComponentClose[15,21](>) +58: RawText[15,22](\n ) +59: ComponentOpen[16,17](<) +60: StringIdentifier[16,18](p) +61: ComponentClose[16,19](>) +62: RawText[16,20](It's false... :() +63: ClosingComponentOpen[16,36]() +66: RawText[16,40](\n ) +67: ClosingComponentOpen[17,13]() +70: RawText[17,23](\n ) +71: ClosingComponentOpen[18,9]() +74: RawText[18,34](\n ) +75: ClosingComponentOpen[19,5]() +78: RawText[19,12](\n) +79: ClosingComponentOpen[20,1]() +82: RawText[20,8](\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/dataComponent_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/dataComponent_tokens.txt index 23711ba..ff17a63 100644 --- a/web-view-components-compiler/src/test/lexer/tokens-files/dataComponent_tokens.txt +++ b/web-view-components-compiler/src/test/lexer/tokens-files/dataComponent_tokens.txt @@ -1,5 +1,5 @@ -ComponentOpen[1,1](<) -StringIdentifier[1,2](data-test) -ComponentNlws[1,11]( ) -ComponentSelfClose[1,12](/>) -RawText[1,14](\n) \ No newline at end of file +0: ComponentOpen[1,1](<) +1: StringIdentifier[1,2](data-test) +2: ComponentNlws[1,11]( ) +3: ComponentSelfClose[1,12](/>) +4: RawText[1,14](\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/emptyHtml_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/emptyHtml_tokens.txt index e2a03f0..ba320ae 100644 --- a/web-view-components-compiler/src/test/lexer/tokens-files/emptyHtml_tokens.txt +++ b/web-view-components-compiler/src/test/lexer/tokens-files/emptyHtml_tokens.txt @@ -1,7 +1,7 @@ -ComponentOpen[1,1](<) -StringIdentifier[1,2](html) -ComponentClose[1,6](>) -ClosingComponentOpen[1,7]() -RawText[1,14](\n) \ No newline at end of file +0: ComponentOpen[1,1](<) +1: StringIdentifier[1,2](html) +2: ComponentClose[1,6](>) +3: ClosingComponentOpen[1,7]() +6: RawText[1,14](\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/emptyTypedWithPackage_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/emptyTypedWithPackage_tokens.txt index 21fb391..cc0ad03 100644 --- a/web-view-components-compiler/src/test/lexer/tokens-files/emptyTypedWithPackage_tokens.txt +++ b/web-view-components-compiler/src/test/lexer/tokens-files/emptyTypedWithPackage_tokens.txt @@ -1,7 +1,7 @@ -ComponentOpen[1,1](<) -TypedIdentifier[1,2](groowt.Test) -ComponentClose[1,13](>) -ClosingComponentOpen[1,14]() -RawText[1,28](\n) \ No newline at end of file +0: ComponentOpen[1,1](<) +1: TypedIdentifier[1,2](groowt.Test) +2: ComponentClose[1,13](>) +3: ClosingComponentOpen[1,14]() +6: RawText[1,28](\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt index f4ec876..847304f 100644 --- a/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt +++ b/web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt @@ -1,4 +1,4 @@ -RawText[1,1](Hello, ) -DollarReferenceStart[1,8]($) -GroovyCode[1,9](target) -RawText[1,15](!\n) \ No newline at end of file +0: RawText[1,1](Hello, ) +1: DollarReferenceStart[1,8]($) +2: GroovyCode[1,9](target) +3: RawText[1,15](!\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt b/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt index 3a7aeeb..ea14f0f 100644 --- a/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt +++ b/web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt @@ -1 +1 @@ -RawText[1,1](Hello, World!\n) \ No newline at end of file +0: RawText[1,1](Hello, World!\n) \ No newline at end of file diff --git a/web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/AbstractSourceTransformerCli.kt b/web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/AbstractSourceTransformerCli.kt new file mode 100644 index 0000000..9abf5fb --- /dev/null +++ b/web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/AbstractSourceTransformerCli.kt @@ -0,0 +1,98 @@ +package groowt.view.component.web.tools + +import picocli.CommandLine.Option +import picocli.CommandLine.Parameters +import java.nio.file.Files +import java.nio.file.Path +import java.util.* +import java.util.concurrent.Callable + +abstract class AbstractSourceTransformerCli : Callable { + + @Parameters( + arity = "1..*", + description = ["The wvc source file(s) to process."] + ) + protected lateinit var targets: List + + @Option( + names = ["--n", "--dry-run"], + description = ["Do a dry run; do not output files to disk."] + ) + protected var dryRun: Boolean = false + + @Option( + names = ["-y", "--yes"], + description = ["Automatically write output files if there are no processing errors."] + ) + protected var autoYes: Boolean = false + + @Option( + names = ["-W", "--write-over"], + description = ["If an output file already exists, write over it without asking."] + ) + protected var autoWriteOver: Boolean = false + + @Option( + names = ["-v", "--verbose"], + description = ["Log verbosely to the console."] + ) + protected var verbose: Boolean = false + + private val scanner = Scanner(System.`in`) + + protected fun getYesNo(prompt: String, allowAuto: Boolean = true): Boolean { + if (this.autoYes && allowAuto) { + return true + } else { + print("$prompt (y/n) ") + while (true) { + if (this.scanner.hasNextLine()) { + val input = this.scanner.nextLine() + if (input.contains("n")) { + return false + } else if (input.contains("y")) { + return true + } + } + } + } + } + + private fun doWrite(resolvedTarget: Path, text: String) { + if (this.dryRun) { + println("Dry-run: would write to $resolvedTarget") + } else { + println("Writing to $resolvedTarget...") + Files.writeString(resolvedTarget, text) + } + } + + protected fun writeToDisk(target: Path, text: String) { + val outputDir = getOutputDir() + if (outputDir != null) { + Files.createDirectories(outputDir) + } + val resolvedTarget = outputDir?.resolve(target) ?: target + if (Files.exists(resolvedTarget) && !autoWriteOver) { + if (getYesNo("$resolvedTarget already exists. Write over?")) { + doWrite(resolvedTarget, text) + } else { + println("Skipping writing to $resolvedTarget") + } + } else { + doWrite(resolvedTarget, text) + } + } + + protected abstract fun transform(target: Path): Int + + protected abstract fun getOutputDir(): Path? + + override fun call(): Int { + return this.targets.fold(0) { acc, target -> + acc.or(this.transform(target)) + } + } + +} diff --git a/web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/TokenizeWvc.kt b/web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/TokenizeWvc.kt new file mode 100644 index 0000000..6615aa1 --- /dev/null +++ b/web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/TokenizeWvc.kt @@ -0,0 +1,129 @@ +package groowt.view.component.web.tools + +import groowt.view.component.web.antlr.* +import org.antlr.v4.runtime.CharStreams +import org.antlr.v4.runtime.ConsoleErrorListener +import org.antlr.v4.runtime.Token +import picocli.CommandLine +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.nio.file.Path +import java.util.* +import kotlin.io.path.nameWithoutExtension +import kotlin.system.exitProcess + +@Command( + name = "tokenizeWvc", + description = ["Tokenizes a Web View Component source file."], + mixinStandardHelpOptions = true, + version = ["0.1.0"] +) +open class TokenizeWvc : AbstractSourceTransformerCli() { + + companion object { + + @JvmStatic + fun main(args: Array) { + exitProcess(CommandLine(TokenizeWvc()).execute(*args)) + } + + } + + @Option( + names = ["-s", "--suffix"], + description = ["The suffix (not extension!) to append to the output file names."] + ) + protected lateinit var suffix: Optional + + @Option( + names = ["-e", "--extension"], + description = ["The extension for output files."], + defaultValue = ".txt" + ) + protected lateinit var extension: String + + @Option( + names = ["-d", "--output-dir"], + description = ["The output directory."], + defaultValue = ".", + paramLabel = "outputDir" + ) + private var myOutputDir: Path? = null + + protected fun onErrors(errors: List): Boolean { + println("There were errors during tokenization.") + errors.forEach { println(format(it)) } + return this.getYesNo("Do you wish to try again?", false) + } + + private fun getOutputPath(target: Path): Path = + Path.of(target.nameWithoutExtension + suffix.orElse("") + extension) + + protected fun onSuccess(target: Path, allTokens: List): Boolean { + val formatted = allTokens.mapIndexed { index, token -> + "$index: ${formatToken(token)}" + }.joinToString(separator = "\n") + if (!this.autoYes) { + println("Please review the following tokens:\n$formatted") + if (this.getYesNo("Write to disk?")) { + this.writeToDisk(getOutputPath(target), formatted) + return false + } else { + return this.getYesNo("Do you wish to redo this file?", false) + } + } else { + this.writeToDisk(getOutputPath(target), formatted) + return false + } + } + + protected fun onException(e: Exception): Boolean { + println("There was an exception during processing: $e") + if (this.verbose) { + e.printStackTrace() + } + return this.getYesNo("Do you wish to try again?", false) + } + + override fun getOutputDir(): Path? = myOutputDir + + override fun transform(target: Path): Int { + println("Processing $target") + var code = 0 + while (true) { + try { + val input = CharStreams.fromPath(target) + + val lexer = WebViewComponentsLexer(input) + lexer.removeErrorListener(ConsoleErrorListener.INSTANCE) + val errorListener = LexerErrorListener() + lexer.addErrorListener(errorListener) + + val tokenStream = WebViewComponentsTokenStream(lexer) + val allTokens = tokenStream.getAllTokensSkipEOF() + + val errors = errorListener.getErrors() + if (errors.isNotEmpty()) { + val recover = this.onErrors(errors) + if (!recover) { + code = 1 + break + } + } else { + val redo = this.onSuccess(target, allTokens) + if (!redo) { + break + } + } + } catch (e: Exception) { + val recover = this.onException(e) + if (!recover) { + code = 1 + break + } + } + } + return code + } + +}