Rewrote tokenize tool in kotlin.
This commit is contained in:
JesseBrault0709
parent
752d58c40a
commit
1b6344dbf2
@ -152,6 +152,7 @@ final List<ToolSpec> toolSpecs = [
|
||||
toolSpec('lexer', 'LexerTool'),
|
||||
toolSpec('parser', 'ParserTool'),
|
||||
toolSpec('parseTreeFileMaker', 'ParseTreeFileMakerCli'),
|
||||
toolSpec('tokenizeWvc', 'TokenizeWvc'),
|
||||
toolSpec('tokensFileMaker', 'TokensFileMakerCli')
|
||||
]
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@ import org.junit.jupiter.api.TestFactory;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static groowt.view.component.web.antlr.WebViewComponentsLexer.*;
|
||||
@ -70,9 +71,15 @@ public class WebViewComponentsLexerTests {
|
||||
final CharStream input = CharStreams.fromString(FileUtil.readFile(sourceFile));
|
||||
final WebViewComponentsLexer lexer = new WebViewComponentsLexer(input);
|
||||
final WebViewComponentsTokenStream tokenStream = new WebViewComponentsTokenStream(lexer);
|
||||
return tokenStream.getAllTokensSkipEOF().stream()
|
||||
.map(TokenUtil::formatToken)
|
||||
.collect(Collectors.joining("\n"));
|
||||
final List<Token> allTokens = tokenStream.getAllTokensSkipEOF();
|
||||
final var sb = new StringBuilder();
|
||||
for (int i = 0; i < allTokens.size(); i++) {
|
||||
sb.append(i).append(": ").append(TokenUtil.formatToken(allTokens.get(i)));
|
||||
if (i < allTokens.size() - 1) {
|
||||
sb.append("\n");
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@ -1,83 +1,83 @@
|
||||
PreambleBreak[1,1](---\n)
|
||||
GroovyCode[2,1](import some.Thing // a comment...World!')
|
||||
PreambleBreak[4,31](\n---\n)
|
||||
RawText[6,1](<!DOCTYPE html>\n)
|
||||
ComponentOpen[7,1](<)
|
||||
StringIdentifier[7,2](html)
|
||||
ComponentClose[7,6](>)
|
||||
RawText[7,7](\n )
|
||||
ComponentOpen[8,5](<)
|
||||
StringIdentifier[8,6](head)
|
||||
ComponentClose[8,10](>)
|
||||
ClosingComponentOpen[8,11](</)
|
||||
StringIdentifier[8,13](head)
|
||||
ComponentClose[8,17](>)
|
||||
RawText[8,18](\n )
|
||||
ComponentOpen[9,5](<)
|
||||
StringIdentifier[9,6](body)
|
||||
ComponentClose[9,10](>)
|
||||
RawText[9,11](\n )
|
||||
ComponentOpen[10,9](<)
|
||||
StringIdentifier[10,10](h1)
|
||||
ComponentClose[10,12](>)
|
||||
DollarScriptletOpen[10,13](${)
|
||||
GroovyCode[10,15](greeting)
|
||||
DollarScriptletClose[10,23](})
|
||||
ClosingComponentOpen[10,24](</)
|
||||
StringIdentifier[10,26](h1)
|
||||
ComponentClose[10,28](>)
|
||||
RawText[10,29](\n )
|
||||
ComponentOpen[11,9](<)
|
||||
TypedIdentifier[11,10](groowt.view.web.Select)
|
||||
ComponentClose[11,32](>)
|
||||
RawText[11,33](\n )
|
||||
ComponentOpen[12,13](<)
|
||||
TypedIdentifier[12,14](Case)
|
||||
ComponentNlws[12,18]( )
|
||||
AttributeIdentifier[12,19](cond)
|
||||
Equals[12,23](=)
|
||||
ClosureAttrValueStart[12,24]({)
|
||||
GroovyCode[12,25](isItTrue())
|
||||
ClosureAttrValueEnd[12,35](})
|
||||
ComponentClose[12,36](>)
|
||||
RawText[12,37](\n )
|
||||
ComponentOpen[13,17](<)
|
||||
StringIdentifier[13,18](p)
|
||||
ComponentClose[13,19](>)
|
||||
RawText[13,20](It's true! :))
|
||||
ClosingComponentOpen[13,33](</)
|
||||
StringIdentifier[13,35](p)
|
||||
ComponentClose[13,36](>)
|
||||
RawText[13,37](\n )
|
||||
ClosingComponentOpen[14,13](</)
|
||||
TypedIdentifier[14,15](Case)
|
||||
ComponentClose[14,19](>)
|
||||
RawText[14,20](\n )
|
||||
ComponentOpen[15,13](<)
|
||||
TypedIdentifier[15,14](Default)
|
||||
ComponentClose[15,21](>)
|
||||
RawText[15,22](\n )
|
||||
ComponentOpen[16,17](<)
|
||||
StringIdentifier[16,18](p)
|
||||
ComponentClose[16,19](>)
|
||||
RawText[16,20](It's false... :()
|
||||
ClosingComponentOpen[16,36](</)
|
||||
StringIdentifier[16,38](p)
|
||||
ComponentClose[16,39](>)
|
||||
RawText[16,40](\n )
|
||||
ClosingComponentOpen[17,13](</)
|
||||
TypedIdentifier[17,15](Default)
|
||||
ComponentClose[17,22](>)
|
||||
RawText[17,23](\n )
|
||||
ClosingComponentOpen[18,9](</)
|
||||
TypedIdentifier[18,11](groowt.view.web.Select)
|
||||
ComponentClose[18,33](>)
|
||||
RawText[18,34](\n )
|
||||
ClosingComponentOpen[19,5](</)
|
||||
StringIdentifier[19,7](body)
|
||||
ComponentClose[19,11](>)
|
||||
RawText[19,12](\n)
|
||||
ClosingComponentOpen[20,1](</)
|
||||
StringIdentifier[20,3](html)
|
||||
ComponentClose[20,7](>)
|
||||
RawText[20,8](\n)
|
||||
0: PreambleBreak[1,1](---\n)
|
||||
1: GroovyCode[2,1](import some.Thing // a comment...World!')
|
||||
2: PreambleBreak[4,31](\n---\n)
|
||||
3: RawText[6,1](<!DOCTYPE html>\n)
|
||||
4: ComponentOpen[7,1](<)
|
||||
5: StringIdentifier[7,2](html)
|
||||
6: ComponentClose[7,6](>)
|
||||
7: RawText[7,7](\n )
|
||||
8: ComponentOpen[8,5](<)
|
||||
9: StringIdentifier[8,6](head)
|
||||
10: ComponentClose[8,10](>)
|
||||
11: ClosingComponentOpen[8,11](</)
|
||||
12: StringIdentifier[8,13](head)
|
||||
13: ComponentClose[8,17](>)
|
||||
14: RawText[8,18](\n )
|
||||
15: ComponentOpen[9,5](<)
|
||||
16: StringIdentifier[9,6](body)
|
||||
17: ComponentClose[9,10](>)
|
||||
18: RawText[9,11](\n )
|
||||
19: ComponentOpen[10,9](<)
|
||||
20: StringIdentifier[10,10](h1)
|
||||
21: ComponentClose[10,12](>)
|
||||
22: DollarScriptletOpen[10,13](${)
|
||||
23: GroovyCode[10,15](greeting)
|
||||
24: DollarScriptletClose[10,23](})
|
||||
25: ClosingComponentOpen[10,24](</)
|
||||
26: StringIdentifier[10,26](h1)
|
||||
27: ComponentClose[10,28](>)
|
||||
28: RawText[10,29](\n )
|
||||
29: ComponentOpen[11,9](<)
|
||||
30: TypedIdentifier[11,10](groowt.view.web.Select)
|
||||
31: ComponentClose[11,32](>)
|
||||
32: RawText[11,33](\n )
|
||||
33: ComponentOpen[12,13](<)
|
||||
34: TypedIdentifier[12,14](Case)
|
||||
35: ComponentNlws[12,18]( )
|
||||
36: AttributeIdentifier[12,19](cond)
|
||||
37: Equals[12,23](=)
|
||||
38: ClosureAttrValueStart[12,24]({)
|
||||
39: GroovyCode[12,25](isItTrue())
|
||||
40: ClosureAttrValueEnd[12,35](})
|
||||
41: ComponentClose[12,36](>)
|
||||
42: RawText[12,37](\n )
|
||||
43: ComponentOpen[13,17](<)
|
||||
44: StringIdentifier[13,18](p)
|
||||
45: ComponentClose[13,19](>)
|
||||
46: RawText[13,20](It's true! :))
|
||||
47: ClosingComponentOpen[13,33](</)
|
||||
48: StringIdentifier[13,35](p)
|
||||
49: ComponentClose[13,36](>)
|
||||
50: RawText[13,37](\n )
|
||||
51: ClosingComponentOpen[14,13](</)
|
||||
52: TypedIdentifier[14,15](Case)
|
||||
53: ComponentClose[14,19](>)
|
||||
54: RawText[14,20](\n )
|
||||
55: ComponentOpen[15,13](<)
|
||||
56: TypedIdentifier[15,14](Default)
|
||||
57: ComponentClose[15,21](>)
|
||||
58: RawText[15,22](\n )
|
||||
59: ComponentOpen[16,17](<)
|
||||
60: StringIdentifier[16,18](p)
|
||||
61: ComponentClose[16,19](>)
|
||||
62: RawText[16,20](It's false... :()
|
||||
63: ClosingComponentOpen[16,36](</)
|
||||
64: StringIdentifier[16,38](p)
|
||||
65: ComponentClose[16,39](>)
|
||||
66: RawText[16,40](\n )
|
||||
67: ClosingComponentOpen[17,13](</)
|
||||
68: TypedIdentifier[17,15](Default)
|
||||
69: ComponentClose[17,22](>)
|
||||
70: RawText[17,23](\n )
|
||||
71: ClosingComponentOpen[18,9](</)
|
||||
72: TypedIdentifier[18,11](groowt.view.web.Select)
|
||||
73: ComponentClose[18,33](>)
|
||||
74: RawText[18,34](\n )
|
||||
75: ClosingComponentOpen[19,5](</)
|
||||
76: StringIdentifier[19,7](body)
|
||||
77: ComponentClose[19,11](>)
|
||||
78: RawText[19,12](\n)
|
||||
79: ClosingComponentOpen[20,1](</)
|
||||
80: StringIdentifier[20,3](html)
|
||||
81: ComponentClose[20,7](>)
|
||||
82: RawText[20,8](\n)
|
||||
@ -1,5 +1,5 @@
|
||||
ComponentOpen[1,1](<)
|
||||
StringIdentifier[1,2](data-test)
|
||||
ComponentNlws[1,11]( )
|
||||
ComponentSelfClose[1,12](/>)
|
||||
RawText[1,14](\n)
|
||||
0: ComponentOpen[1,1](<)
|
||||
1: StringIdentifier[1,2](data-test)
|
||||
2: ComponentNlws[1,11]( )
|
||||
3: ComponentSelfClose[1,12](/>)
|
||||
4: RawText[1,14](\n)
|
||||
@ -1,7 +1,7 @@
|
||||
ComponentOpen[1,1](<)
|
||||
StringIdentifier[1,2](html)
|
||||
ComponentClose[1,6](>)
|
||||
ClosingComponentOpen[1,7](</)
|
||||
StringIdentifier[1,9](html)
|
||||
ComponentClose[1,13](>)
|
||||
RawText[1,14](\n)
|
||||
0: ComponentOpen[1,1](<)
|
||||
1: StringIdentifier[1,2](html)
|
||||
2: ComponentClose[1,6](>)
|
||||
3: ClosingComponentOpen[1,7](</)
|
||||
4: StringIdentifier[1,9](html)
|
||||
5: ComponentClose[1,13](>)
|
||||
6: RawText[1,14](\n)
|
||||
@ -1,7 +1,7 @@
|
||||
ComponentOpen[1,1](<)
|
||||
TypedIdentifier[1,2](groowt.Test)
|
||||
ComponentClose[1,13](>)
|
||||
ClosingComponentOpen[1,14](</)
|
||||
TypedIdentifier[1,16](groowt.Test)
|
||||
ComponentClose[1,27](>)
|
||||
RawText[1,28](\n)
|
||||
0: ComponentOpen[1,1](<)
|
||||
1: TypedIdentifier[1,2](groowt.Test)
|
||||
2: ComponentClose[1,13](>)
|
||||
3: ClosingComponentOpen[1,14](</)
|
||||
4: TypedIdentifier[1,16](groowt.Test)
|
||||
5: ComponentClose[1,27](>)
|
||||
6: RawText[1,28](\n)
|
||||
@ -1,4 +1,4 @@
|
||||
RawText[1,1](Hello, )
|
||||
DollarReferenceStart[1,8]($)
|
||||
GroovyCode[1,9](target)
|
||||
RawText[1,15](!\n)
|
||||
0: RawText[1,1](Hello, )
|
||||
1: DollarReferenceStart[1,8]($)
|
||||
2: GroovyCode[1,9](target)
|
||||
3: RawText[1,15](!\n)
|
||||
@ -1 +1 @@
|
||||
RawText[1,1](Hello, World!\n)
|
||||
0: RawText[1,1](Hello, World!\n)
|
||||
@ -0,0 +1,98 @@
|
||||
package groowt.view.component.web.tools
|
||||
|
||||
import picocli.CommandLine.Option
|
||||
import picocli.CommandLine.Parameters
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.*
|
||||
import java.util.concurrent.Callable
|
||||
|
||||
abstract class AbstractSourceTransformerCli : Callable<Int> {
|
||||
|
||||
@Parameters(
|
||||
arity = "1..*",
|
||||
description = ["The wvc source file(s) to process."]
|
||||
)
|
||||
protected lateinit var targets: List<Path>
|
||||
|
||||
@Option(
|
||||
names = ["--n", "--dry-run"],
|
||||
description = ["Do a dry run; do not output files to disk."]
|
||||
)
|
||||
protected var dryRun: Boolean = false
|
||||
|
||||
@Option(
|
||||
names = ["-y", "--yes"],
|
||||
description = ["Automatically write output files if there are no processing errors."]
|
||||
)
|
||||
protected var autoYes: Boolean = false
|
||||
|
||||
@Option(
|
||||
names = ["-W", "--write-over"],
|
||||
description = ["If an output file already exists, write over it without asking."]
|
||||
)
|
||||
protected var autoWriteOver: Boolean = false
|
||||
|
||||
@Option(
|
||||
names = ["-v", "--verbose"],
|
||||
description = ["Log verbosely to the console."]
|
||||
)
|
||||
protected var verbose: Boolean = false
|
||||
|
||||
private val scanner = Scanner(System.`in`)
|
||||
|
||||
protected fun getYesNo(prompt: String, allowAuto: Boolean = true): Boolean {
|
||||
if (this.autoYes && allowAuto) {
|
||||
return true
|
||||
} else {
|
||||
print("$prompt (y/n) ")
|
||||
while (true) {
|
||||
if (this.scanner.hasNextLine()) {
|
||||
val input = this.scanner.nextLine()
|
||||
if (input.contains("n")) {
|
||||
return false
|
||||
} else if (input.contains("y")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun doWrite(resolvedTarget: Path, text: String) {
|
||||
if (this.dryRun) {
|
||||
println("Dry-run: would write to $resolvedTarget")
|
||||
} else {
|
||||
println("Writing to $resolvedTarget...")
|
||||
Files.writeString(resolvedTarget, text)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun writeToDisk(target: Path, text: String) {
|
||||
val outputDir = getOutputDir()
|
||||
if (outputDir != null) {
|
||||
Files.createDirectories(outputDir)
|
||||
}
|
||||
val resolvedTarget = outputDir?.resolve(target) ?: target
|
||||
if (Files.exists(resolvedTarget) && !autoWriteOver) {
|
||||
if (getYesNo("$resolvedTarget already exists. Write over?")) {
|
||||
doWrite(resolvedTarget, text)
|
||||
} else {
|
||||
println("Skipping writing to $resolvedTarget")
|
||||
}
|
||||
} else {
|
||||
doWrite(resolvedTarget, text)
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract fun transform(target: Path): Int
|
||||
|
||||
protected abstract fun getOutputDir(): Path?
|
||||
|
||||
override fun call(): Int {
|
||||
return this.targets.fold(0) { acc, target ->
|
||||
acc.or(this.transform(target))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,129 @@
|
||||
package groowt.view.component.web.tools
|
||||
|
||||
import groowt.view.component.web.antlr.*
|
||||
import org.antlr.v4.runtime.CharStreams
|
||||
import org.antlr.v4.runtime.ConsoleErrorListener
|
||||
import org.antlr.v4.runtime.Token
|
||||
import picocli.CommandLine
|
||||
import picocli.CommandLine.Command
|
||||
import picocli.CommandLine.Option
|
||||
import java.nio.file.Path
|
||||
import java.util.*
|
||||
import kotlin.io.path.nameWithoutExtension
|
||||
import kotlin.system.exitProcess
|
||||
|
||||
@Command(
|
||||
name = "tokenizeWvc",
|
||||
description = ["Tokenizes a Web View Component source file."],
|
||||
mixinStandardHelpOptions = true,
|
||||
version = ["0.1.0"]
|
||||
)
|
||||
open class TokenizeWvc : AbstractSourceTransformerCli() {
|
||||
|
||||
companion object {
|
||||
|
||||
@JvmStatic
|
||||
fun main(args: Array<String>) {
|
||||
exitProcess(CommandLine(TokenizeWvc()).execute(*args))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Option(
|
||||
names = ["-s", "--suffix"],
|
||||
description = ["The suffix (not extension!) to append to the output file names."]
|
||||
)
|
||||
protected lateinit var suffix: Optional<String>
|
||||
|
||||
@Option(
|
||||
names = ["-e", "--extension"],
|
||||
description = ["The extension for output files."],
|
||||
defaultValue = ".txt"
|
||||
)
|
||||
protected lateinit var extension: String
|
||||
|
||||
@Option(
|
||||
names = ["-d", "--output-dir"],
|
||||
description = ["The output directory."],
|
||||
defaultValue = ".",
|
||||
paramLabel = "outputDir"
|
||||
)
|
||||
private var myOutputDir: Path? = null
|
||||
|
||||
protected fun onErrors(errors: List<LexerError>): Boolean {
|
||||
println("There were errors during tokenization.")
|
||||
errors.forEach { println(format(it)) }
|
||||
return this.getYesNo("Do you wish to try again?", false)
|
||||
}
|
||||
|
||||
private fun getOutputPath(target: Path): Path =
|
||||
Path.of(target.nameWithoutExtension + suffix.orElse("") + extension)
|
||||
|
||||
protected fun onSuccess(target: Path, allTokens: List<Token>): Boolean {
|
||||
val formatted = allTokens.mapIndexed { index, token ->
|
||||
"$index: ${formatToken(token)}"
|
||||
}.joinToString(separator = "\n")
|
||||
if (!this.autoYes) {
|
||||
println("Please review the following tokens:\n$formatted")
|
||||
if (this.getYesNo("Write to disk?")) {
|
||||
this.writeToDisk(getOutputPath(target), formatted)
|
||||
return false
|
||||
} else {
|
||||
return this.getYesNo("Do you wish to redo this file?", false)
|
||||
}
|
||||
} else {
|
||||
this.writeToDisk(getOutputPath(target), formatted)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
protected fun onException(e: Exception): Boolean {
|
||||
println("There was an exception during processing: $e")
|
||||
if (this.verbose) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return this.getYesNo("Do you wish to try again?", false)
|
||||
}
|
||||
|
||||
override fun getOutputDir(): Path? = myOutputDir
|
||||
|
||||
override fun transform(target: Path): Int {
|
||||
println("Processing $target")
|
||||
var code = 0
|
||||
while (true) {
|
||||
try {
|
||||
val input = CharStreams.fromPath(target)
|
||||
|
||||
val lexer = WebViewComponentsLexer(input)
|
||||
lexer.removeErrorListener(ConsoleErrorListener.INSTANCE)
|
||||
val errorListener = LexerErrorListener()
|
||||
lexer.addErrorListener(errorListener)
|
||||
|
||||
val tokenStream = WebViewComponentsTokenStream(lexer)
|
||||
val allTokens = tokenStream.getAllTokensSkipEOF()
|
||||
|
||||
val errors = errorListener.getErrors()
|
||||
if (errors.isNotEmpty()) {
|
||||
val recover = this.onErrors(errors)
|
||||
if (!recover) {
|
||||
code = 1
|
||||
break
|
||||
}
|
||||
} else {
|
||||
val redo = this.onSuccess(target, allTokens)
|
||||
if (!redo) {
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
val recover = this.onException(e)
|
||||
if (!recover) {
|
||||
code = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return code
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user