jessebrault/groowt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrote tokenize tool in kotlin.

Browse Source
This commit is contained in:
JesseBrault0709 2024-05-21 12:31:26 +02:00
parent 752d58c40a
commit 1b6344dbf2
10 changed files with 345 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web-view-components-compiler/build.gradle
View File

@ -152,6 +152,7 @@ final List<ToolSpec> toolSpecs = [
toolSpec('lexer', 'LexerTool'), toolSpec('lexer', 'LexerTool'),
toolSpec('parser', 'ParserTool'), toolSpec('parser', 'ParserTool'),
toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'), toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'),
toolSpec('tokenizeWvc', 'TokenizeWvc'),
toolSpec('tokensFileMaker', 'TokensFileMakerCli') toolSpec('tokensFileMaker', 'TokensFileMakerCli')
] ]

13
web-view-components-compiler/src/test/java/groowt/view/component/web/antlr/WebViewComponentsLexerTests.java
View File

@ -12,6 +12,7 @@ import org.junit.jupiter.api.TestFactory;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Collection; import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static groowt.view.component.web.antlr.WebViewComponentsLexer.*; import static groowt.view.component.web.antlr.WebViewComponentsLexer.*;
@ -70,9 +71,15 @@ public class WebViewComponentsLexerTests {
final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile)); final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile));
final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input); final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input);
final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer); final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer);
return tokenStream.getAllTokensSkipEOF().stream() final List<Token> allTokens = tokenStream.getAllTokensSkipEOF();
.map(TokenUtil::formatToken) final var sb = new StringBuilder();
.collect(Collectors.joining("\n")); for (int i = 0; i < allTokens.size(); i++) {
sb.append(i).append(": ").append(TokenUtil.formatToken(allTokens.get(i)));
if (i < allTokens.size() - 1) {
sb.append("\n");
}
}
return sb.toString();
} }
); );
} }

166
web-view-components-compiler/src/test/lexer/tokens-files/complicated_tokens.txt
View File

@ -1,83 +1,83 @@
PreambleBreak[1,1](---\n) 0: PreambleBreak[1,1](---\n)
GroovyCode[2,1](import some.Thing // a comment...World!') 1: GroovyCode[2,1](import some.Thing // a comment...World!')
PreambleBreak[4,31](\n---\n) 2: PreambleBreak[4,31](\n---\n)
RawText[6,1](<!DOCTYPE html>\n) 3: RawText[6,1](<!DOCTYPE html>\n)
ComponentOpen[7,1](<) 4: ComponentOpen[7,1](<)
StringIdentifier[7,2](html) 5: StringIdentifier[7,2](html)
ComponentClose[7,6](>) 6: ComponentClose[7,6](>)
RawText[7,7](\n ) 7: RawText[7,7](\n )
ComponentOpen[8,5](<) 8: ComponentOpen[8,5](<)
StringIdentifier[8,6](head) 9: StringIdentifier[8,6](head)
ComponentClose[8,10](>) 10: ComponentClose[8,10](>)
ClosingComponentOpen[8,11](</) 11: ClosingComponentOpen[8,11](</)
StringIdentifier[8,13](head) 12: StringIdentifier[8,13](head)
ComponentClose[8,17](>) 13: ComponentClose[8,17](>)
RawText[8,18](\n ) 14: RawText[8,18](\n )
ComponentOpen[9,5](<) 15: ComponentOpen[9,5](<)
StringIdentifier[9,6](body) 16: StringIdentifier[9,6](body)
ComponentClose[9,10](>) 17: ComponentClose[9,10](>)
RawText[9,11](\n ) 18: RawText[9,11](\n )
ComponentOpen[10,9](<) 19: ComponentOpen[10,9](<)
StringIdentifier[10,10](h1) 20: StringIdentifier[10,10](h1)
ComponentClose[10,12](>) 21: ComponentClose[10,12](>)
DollarScriptletOpen[10,13](${) 22: DollarScriptletOpen[10,13](${)
GroovyCode[10,15](greeting) 23: GroovyCode[10,15](greeting)
DollarScriptletClose[10,23](}) 24: DollarScriptletClose[10,23](})
ClosingComponentOpen[10,24](</) 25: ClosingComponentOpen[10,24](</)
StringIdentifier[10,26](h1) 26: StringIdentifier[10,26](h1)
ComponentClose[10,28](>) 27: ComponentClose[10,28](>)
RawText[10,29](\n ) 28: RawText[10,29](\n )
ComponentOpen[11,9](<) 29: ComponentOpen[11,9](<)
TypedIdentifier[11,10](groowt.view.web.Select) 30: TypedIdentifier[11,10](groowt.view.web.Select)
ComponentClose[11,32](>) 31: ComponentClose[11,32](>)
RawText[11,33](\n ) 32: RawText[11,33](\n )
ComponentOpen[12,13](<) 33: ComponentOpen[12,13](<)
TypedIdentifier[12,14](Case) 34: TypedIdentifier[12,14](Case)
ComponentNlws[12,18]( ) 35: ComponentNlws[12,18]( )
AttributeIdentifier[12,19](cond) 36: AttributeIdentifier[12,19](cond)
Equals[12,23](=) 37: Equals[12,23](=)
ClosureAttrValueStart[12,24]({) 38: ClosureAttrValueStart[12,24]({)
GroovyCode[12,25](isItTrue()) 39: GroovyCode[12,25](isItTrue())
ClosureAttrValueEnd[12,35](}) 40: ClosureAttrValueEnd[12,35](})
ComponentClose[12,36](>) 41: ComponentClose[12,36](>)
RawText[12,37](\n ) 42: RawText[12,37](\n )
ComponentOpen[13,17](<) 43: ComponentOpen[13,17](<)
StringIdentifier[13,18](p) 44: StringIdentifier[13,18](p)
ComponentClose[13,19](>) 45: ComponentClose[13,19](>)
RawText[13,20](It's true! :)) 46: RawText[13,20](It's true! :))
ClosingComponentOpen[13,33](</) 47: ClosingComponentOpen[13,33](</)
StringIdentifier[13,35](p) 48: StringIdentifier[13,35](p)
ComponentClose[13,36](>) 49: ComponentClose[13,36](>)
RawText[13,37](\n ) 50: RawText[13,37](\n )
ClosingComponentOpen[14,13](</) 51: ClosingComponentOpen[14,13](</)
TypedIdentifier[14,15](Case) 52: TypedIdentifier[14,15](Case)
ComponentClose[14,19](>) 53: ComponentClose[14,19](>)
RawText[14,20](\n ) 54: RawText[14,20](\n )
ComponentOpen[15,13](<) 55: ComponentOpen[15,13](<)
TypedIdentifier[15,14](Default) 56: TypedIdentifier[15,14](Default)
ComponentClose[15,21](>) 57: ComponentClose[15,21](>)
RawText[15,22](\n ) 58: RawText[15,22](\n )
ComponentOpen[16,17](<) 59: ComponentOpen[16,17](<)
StringIdentifier[16,18](p) 60: StringIdentifier[16,18](p)
ComponentClose[16,19](>) 61: ComponentClose[16,19](>)
RawText[16,20](It's false... :() 62: RawText[16,20](It's false... :()
ClosingComponentOpen[16,36](</) 63: ClosingComponentOpen[16,36](</)
StringIdentifier[16,38](p) 64: StringIdentifier[16,38](p)
ComponentClose[16,39](>) 65: ComponentClose[16,39](>)
RawText[16,40](\n ) 66: RawText[16,40](\n )
ClosingComponentOpen[17,13](</) 67: ClosingComponentOpen[17,13](</)
TypedIdentifier[17,15](Default) 68: TypedIdentifier[17,15](Default)
ComponentClose[17,22](>) 69: ComponentClose[17,22](>)
RawText[17,23](\n ) 70: RawText[17,23](\n )
ClosingComponentOpen[18,9](</) 71: ClosingComponentOpen[18,9](</)
TypedIdentifier[18,11](groowt.view.web.Select) 72: TypedIdentifier[18,11](groowt.view.web.Select)
ComponentClose[18,33](>) 73: ComponentClose[18,33](>)
RawText[18,34](\n ) 74: RawText[18,34](\n )
ClosingComponentOpen[19,5](</) 75: ClosingComponentOpen[19,5](</)
StringIdentifier[19,7](body) 76: StringIdentifier[19,7](body)
ComponentClose[19,11](>) 77: ComponentClose[19,11](>)
RawText[19,12](\n) 78: RawText[19,12](\n)
ClosingComponentOpen[20,1](</) 79: ClosingComponentOpen[20,1](</)
StringIdentifier[20,3](html) 80: StringIdentifier[20,3](html)
ComponentClose[20,7](>) 81: ComponentClose[20,7](>)
RawText[20,8](\n) 82: RawText[20,8](\n)

10
web-view-components-compiler/src/test/lexer/tokens-files/dataComponent_tokens.txt
View File

@ -1,5 +1,5 @@
ComponentOpen[1,1](<) 0: ComponentOpen[1,1](<)
StringIdentifier[1,2](data-test) 1: StringIdentifier[1,2](data-test)
ComponentNlws[1,11]( ) 2: ComponentNlws[1,11]( )
ComponentSelfClose[1,12](/>) 3: ComponentSelfClose[1,12](/>)
RawText[1,14](\n) 4: RawText[1,14](\n)

14
web-view-components-compiler/src/test/lexer/tokens-files/emptyHtml_tokens.txt
View File

@ -1,7 +1,7 @@
ComponentOpen[1,1](<) 0: ComponentOpen[1,1](<)
StringIdentifier[1,2](html) 1: StringIdentifier[1,2](html)
ComponentClose[1,6](>) 2: ComponentClose[1,6](>)
ClosingComponentOpen[1,7](</) 3: ClosingComponentOpen[1,7](</)
StringIdentifier[1,9](html) 4: StringIdentifier[1,9](html)
ComponentClose[1,13](>) 5: ComponentClose[1,13](>)
RawText[1,14](\n) 6: RawText[1,14](\n)

14
web-view-components-compiler/src/test/lexer/tokens-files/emptyTypedWithPackage_tokens.txt
View File

@ -1,7 +1,7 @@
ComponentOpen[1,1](<) 0: ComponentOpen[1,1](<)
TypedIdentifier[1,2](groowt.Test) 1: TypedIdentifier[1,2](groowt.Test)
ComponentClose[1,13](>) 2: ComponentClose[1,13](>)
ClosingComponentOpen[1,14](</) 3: ClosingComponentOpen[1,14](</)
TypedIdentifier[1,16](groowt.Test) 4: TypedIdentifier[1,16](groowt.Test)
ComponentClose[1,27](>) 5: ComponentClose[1,27](>)
RawText[1,28](\n) 6: RawText[1,28](\n)

8
web-view-components-compiler/src/test/lexer/tokens-files/helloTarget_tokens.txt
View File

@ -1,4 +1,4 @@
RawText[1,1](Hello, ) 0: RawText[1,1](Hello, )
DollarReferenceStart[1,8]($) 1: DollarReferenceStart[1,8]($)
GroovyCode[1,9](target) 2: GroovyCode[1,9](target)
RawText[1,15](!\n) 3: RawText[1,15](!\n)

2
web-view-components-compiler/src/test/lexer/tokens-files/helloWorld_tokens.txt
View File

@ -1 +1 @@
RawText[1,1](Hello, World!\n) 0: RawText[1,1](Hello, World!\n)

98
web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/AbstractSourceTransformerCli.kt Normal file
View File

@ -0,0 +1,98 @@
package groowt.view.component.web.tools
import picocli.CommandLine.Option
import picocli.CommandLine.Parameters
import java.nio.file.Files
import java.nio.file.Path
import java.util.*
import java.util.concurrent.Callable
abstract class AbstractSourceTransformerCli : Callable<Int> {
@Parameters(
arity = "1..*",
description = ["The wvc source file(s) to process."]
)
protected lateinit var targets: List<Path>
@Option(
names = ["--n", "--dry-run"],
description = ["Do a dry run; do not output files to disk."]
)
protected var dryRun: Boolean = false
@Option(
names = ["-y", "--yes"],
description = ["Automatically write output files if there are no processing errors."]
)
protected var autoYes: Boolean = false
@Option(
names = ["-W", "--write-over"],
description = ["If an output file already exists, write over it without asking."]
)
protected var autoWriteOver: Boolean = false
@Option(
names = ["-v", "--verbose"],
description = ["Log verbosely to the console."]
)
protected var verbose: Boolean = false
private val scanner = Scanner(System.`in`)
protected fun getYesNo(prompt: String, allowAuto: Boolean = true): Boolean {
if (this.autoYes && allowAuto) {
return true
} else {
print("$prompt (y/n) ")
while (true) {
if (this.scanner.hasNextLine()) {
val input = this.scanner.nextLine()
if (input.contains("n")) {
return false
} else if (input.contains("y")) {
return true
}
}
}
}
}
private fun doWrite(resolvedTarget: Path, text: String) {
if (this.dryRun) {
println("Dry-run: would write to $resolvedTarget")
} else {
println("Writing to $resolvedTarget...")
Files.writeString(resolvedTarget, text)
}
}
protected fun writeToDisk(target: Path, text: String) {
val outputDir = getOutputDir()
if (outputDir != null) {
Files.createDirectories(outputDir)
}
val resolvedTarget = outputDir?.resolve(target) ?: target
if (Files.exists(resolvedTarget) && !autoWriteOver) {
if (getYesNo("$resolvedTarget already exists. Write over?")) {
doWrite(resolvedTarget, text)
} else {
println("Skipping writing to $resolvedTarget")
}
} else {
doWrite(resolvedTarget, text)
}
}
protected abstract fun transform(target: Path): Int
protected abstract fun getOutputDir(): Path?
override fun call(): Int {
return this.targets.fold(0) { acc, target ->
acc.or(this.transform(target))
}
}
}

129
web-view-components-compiler/src/tools/kotlin/groowt/view/component/web/tools/TokenizeWvc.kt Normal file
View File

@ -0,0 +1,129 @@
package groowt.view.component.web.tools
import groowt.view.component.web.antlr.*
import org.antlr.v4.runtime.CharStreams
import org.antlr.v4.runtime.ConsoleErrorListener
import org.antlr.v4.runtime.Token
import picocli.CommandLine
import picocli.CommandLine.Command
import picocli.CommandLine.Option
import java.nio.file.Path
import java.util.*
import kotlin.io.path.nameWithoutExtension
import kotlin.system.exitProcess
@Command(
name = "tokenizeWvc",
description = ["Tokenizes a Web View Component source file."],
mixinStandardHelpOptions = true,
version = ["0.1.0"]
)
open class TokenizeWvc : AbstractSourceTransformerCli() {
companion object {
@JvmStatic
fun main(args: Array<String>) {
exitProcess(CommandLine(TokenizeWvc()).execute(*args))
}
}
@Option(
names = ["-s", "--suffix"],
description = ["The suffix (not extension!) to append to the output file names."]
)
protected lateinit var suffix: Optional<String>
@Option(
names = ["-e", "--extension"],
description = ["The extension for output files."],
defaultValue = ".txt"
)
protected lateinit var extension: String
@Option(
names = ["-d", "--output-dir"],
description = ["The output directory."],
defaultValue = ".",
paramLabel = "outputDir"
)
private var myOutputDir: Path? = null
protected fun onErrors(errors: List<LexerError>): Boolean {
println("There were errors during tokenization.")
errors.forEach { println(format(it)) }
return this.getYesNo("Do you wish to try again?", false)
}
private fun getOutputPath(target: Path): Path =
Path.of(target.nameWithoutExtension + suffix.orElse("") + extension)
protected fun onSuccess(target: Path, allTokens: List<Token>): Boolean {
val formatted = allTokens.mapIndexed { index, token ->
"$index: ${formatToken(token)}"
}.joinToString(separator = "\n")
if (!this.autoYes) {
println("Please review the following tokens:\n$formatted")
if (this.getYesNo("Write to disk?")) {
this.writeToDisk(getOutputPath(target), formatted)
return false
} else {
return this.getYesNo("Do you wish to redo this file?", false)
}
} else {
this.writeToDisk(getOutputPath(target), formatted)
return false
}
}
protected fun onException(e: Exception): Boolean {
println("There was an exception during processing: $e")
if (this.verbose) {
e.printStackTrace()
}
return this.getYesNo("Do you wish to try again?", false)
}
override fun getOutputDir(): Path? = myOutputDir
override fun transform(target: Path): Int {
println("Processing $target")
var code = 0
while (true) {
try {
val input = CharStreams.fromPath(target)
val lexer = WebViewComponentsLexer(input)
lexer.removeErrorListener(ConsoleErrorListener.INSTANCE)
val errorListener = LexerErrorListener()
lexer.addErrorListener(errorListener)
val tokenStream = WebViewComponentsTokenStream(lexer)
val allTokens = tokenStream.getAllTokensSkipEOF()
val errors = errorListener.getErrors()
if (errors.isNotEmpty()) {
val recover = this.onErrors(errors)
if (!recover) {
code = 1
break
}
} else {
val redo = this.onSuccess(target, allTokens)
if (!redo) {
break
}
}
} catch (e: Exception) {
val recover = this.onException(e)
if (!recover) {
code = 1
break
}
}
}
return code
}
}