Further testing of ComponentTokenizer.

template/src/main/groovy/com/jessebrault/ssg/template/gspe/ComponentToken.groovy

@@ -32,4 +32,9 @@ class ComponentToken {
     Type type
     String text
 
+    @Override
+    String toString() {
+        "ComponentToken(${ this.type }, ${ this.text })"
+    }
+
 }

template/src/main/groovy/com/jessebrault/ssg/template/gspe/ComponentTokenizer.groovy

@@ -32,6 +32,8 @@ class ComponentTokenizer {
 
     static enum State {
         START,
+        IDENTIFIER,
+        KEYS_AND_VALUES,
         DOUBLE_QUOTE_STRING,
         SINGLE_QUOTE_STRING,
         DOLLAR_GROOVY,
@@ -50,15 +52,27 @@ class ComponentTokenizer {
             initialState = State.START
 
             whileIn(State.START) {
-                on lessThan exec {
+                on lessThan shiftTo State.IDENTIFIER exec {
                     tokens << new ComponentToken(Type.LT, it)
                 }
+                onNoMatch() exec {
+                    throw new IllegalArgumentException()
+                }
+            }
+
+            whileIn(State.IDENTIFIER) {
+                on identifier shiftTo State.KEYS_AND_VALUES exec {
+                    tokens << new ComponentToken(Type.IDENTIFIER, it)
+                }
+                onNoMatch() exec {
+                    throw new IllegalArgumentException()
+                }
+            }
+
+            whileIn(State.KEYS_AND_VALUES) {
                 on greaterThan shiftTo State.DONE exec {
                     tokens << new ComponentToken(Type.GT, it)
                 }
-                on identifier exec {
-                    tokens << new ComponentToken(Type.IDENTIFIER, it)
-                }
                 on whitespace exec { }
                 on key exec {
                     tokens << new ComponentToken(Type.KEY, it)
@@ -95,7 +109,7 @@ class ComponentTokenizer {
                 on doubleQuoteStringContent exec {
                     tokens << new ComponentToken(Type.STRING, it)
                 }
-                on doubleQuote shiftTo State.START exec {
+                on doubleQuote shiftTo State.KEYS_AND_VALUES exec {
                     tokens << new ComponentToken(Type.DOUBLE_QUOTE, it)
                 }
                 onNoMatch() exec {
@@ -107,7 +121,7 @@ class ComponentTokenizer {
                 on singleQuoteStringContent exec {
                     tokens << new ComponentToken(Type.STRING, it)
                 }
-                on singleQuote shiftTo State.START exec {
+                on singleQuote shiftTo State.KEYS_AND_VALUES exec {
                     tokens << new ComponentToken(Type.SINGLE_QUOTE, it)
                 }
                 onNoMatch() exec {
@@ -136,7 +150,7 @@ class ComponentTokenizer {
                         }
                     }
                     b.toString()
-                } shiftTo State.START exec { String s ->
+                } shiftTo State.KEYS_AND_VALUES exec { String s ->
                     tokens << new ComponentToken(Type.GROOVY, s.substring(0, s.length() - 1))
                     tokens << new ComponentToken(Type.CURLY_CLOSE, '}')
                 }
@@ -146,7 +160,7 @@ class ComponentTokenizer {
             }
 
             whileIn(State.EXPRESSION_SCRIPTLET_GROOVY) {
-                on expressionScriptletGroovy shiftTo State.START exec { String s ->
+                on expressionScriptletGroovy shiftTo State.KEYS_AND_VALUES exec { String s ->
                     tokens << new ComponentToken(Type.GROOVY, s.substring(0, s.length() - 2))
                     tokens << new ComponentToken(Type.PERCENT, '%')
                     tokens << new ComponentToken(Type.GT, '>')

template/src/test/groovy/com/jessebrault/ssg/template/gspe/ComponentTokenizerTests.groovy

@@ -1,107 +1,164 @@
 package com.jessebrault.ssg.template.gspe
 
 import org.junit.jupiter.api.Test
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 
 import static com.jessebrault.ssg.template.gspe.ComponentToken.Type.*
-
 import static org.junit.jupiter.api.Assertions.assertEquals
+import static org.junit.jupiter.api.Assertions.assertTrue
 
 class ComponentTokenizerTests {
 
+    private static final Logger logger = LoggerFactory.getLogger(ComponentTokenizerTests)
+
+    private static class TokenSpec {
+
+        ComponentToken.Type type
+        String text
+
+        TokenSpec(ComponentToken.Type type, String text = null) {
+            this.type = Objects.requireNonNull(type)
+            this.text = text
+        }
+
+        void compare(ComponentToken actual) {
+            assertEquals(this.type, actual.type)
+            if (this.text != null) {
+                assertEquals(this.text, actual.text)
+            }
+        }
+
+        @Override
+        String toString() {
+            "TokenSpec(${ this.type }, ${ this.text })"
+        }
+
+    }
+
+    private static class TesterConfigurator {
+
+        Queue<TokenSpec> specs = new LinkedList<>()
+
+        void expect(ComponentToken.Type type, String text) {
+            this.specs << new TokenSpec(type, text)
+        }
+
+        void expect(ComponentToken.Type type) {
+            this.specs << new TokenSpec(type)
+        }
+
+    }
+
     private final ComponentTokenizer tokenizer = new ComponentTokenizer()
 
+    private void test(
+            String src,
+            @DelegatesTo(value = TesterConfigurator, strategy = Closure.DELEGATE_FIRST)
+            Closure<Void> configure
+    ) {
+        def configurator = new TesterConfigurator()
+        configure.setDelegate(configurator)
+        configure.setResolveStrategy(Closure.DELEGATE_FIRST)
+        configure()
+
+        def r = this.tokenizer.tokenize(src)
+        logger.debug('r: {}', r)
+        logger.debug('configurator.specs: {}', configurator.specs)
+
+        assertEquals(configurator.specs.size(), r.size())
+
+        def resultIterator = r.iterator()
+        configurator.specs.each {
+            assertTrue(resultIterator.hasNext())
+            it.compare(resultIterator.next())
+        }
+    }
+
     @Test
     void selfClosingComponent() {
-        def r = this.tokenizer.tokenize('<Test />')
+        this.test('<Test />') {
-        assertEquals(4, r.size())
+            expect LT
-        assertEquals(LT, r[0].type)
+            expect IDENTIFIER, 'Test'
-        assertEquals(IDENTIFIER, r[1].type)
+            expect FORWARD_SLASH
-        assertEquals('Test', r[1].text)
+            expect GT
-        assertEquals(FORWARD_SLASH, r[2].type)
+        }
-        assertEquals(GT, r[3].type)
     }
 
     @Test
     void selfClosingComponentWithDoubleQuotedString() {
-        def r = this.tokenizer.tokenize('<Test key="value" />')
+        this.test('<Test key="value" />') {
-        assertEquals(9, r.size())
+            expect LT
-
+            expect IDENTIFIER, 'Test'
-        assertEquals(LT, r[0].type)
+            expect KEY, 'key'
-
+            expect EQUALS
-        assertEquals(IDENTIFIER, r[1].type)
+            expect DOUBLE_QUOTE
-        assertEquals('Test', r[1].text)
+            expect STRING, 'value'
-
+            expect DOUBLE_QUOTE
-        assertEquals(KEY, r[2].type)
+            expect FORWARD_SLASH
-        assertEquals('key', r[2].text)
+            expect GT
-
+        }
-        assertEquals(EQUALS, r[3].type)
-
-        assertEquals(DOUBLE_QUOTE, r[4].type)
-
-        assertEquals(STRING, r[5].type)
-        assertEquals('value', r[5].text)
-
-        assertEquals(DOUBLE_QUOTE, r[6].type)
-
-        assertEquals(FORWARD_SLASH, r[7].type)
-
-        assertEquals(GT, r[8].type)
     }
 
     @Test
     void selfClosingComponentWithSingleQuotedString() {
-        def r = this.tokenizer.tokenize("<Test key='value' />")
+        this.test("<Test key='value' />") {
-        assertEquals(9, r.size())
+            expect LT
-
+            expect IDENTIFIER, 'Test'
-        assertEquals(LT, r[0].type)
+            expect KEY, 'key'
-
+            expect EQUALS
-        assertEquals(IDENTIFIER, r[1].type)
+            expect SINGLE_QUOTE
-        assertEquals('Test', r[1].text)
+            expect STRING, 'value'
-
+            expect SINGLE_QUOTE
-        assertEquals(KEY, r[2].type)
+            expect FORWARD_SLASH
-        assertEquals('key', r[2].text)
+            expect GT
-
+        }
-        assertEquals(EQUALS, r[3].type)
-
-        assertEquals(SINGLE_QUOTE, r[4].type)
-
-        assertEquals(STRING, r[5].type)
-        assertEquals('value', r[5].text)
-
-        assertEquals(SINGLE_QUOTE, r[6].type)
-
-        assertEquals(FORWARD_SLASH, r[7].type)
-
-        assertEquals(GT, r[8].type)
     }
 
     @Test
     void componentWithSimpleDollarGroovy() {
-        def r = this.tokenizer.tokenize('<Test key=${ test } />')
+        this.test('<Test key=${ test } />') {
-        assertEquals(10, r.size())
+            expect LT
+            expect IDENTIFIER, 'Test'
+            expect KEY, 'key'
+            expect EQUALS
+            expect DOLLAR
+            expect CURLY_OPEN
+            expect GROOVY, ' test '
+            expect CURLY_CLOSE
+            expect FORWARD_SLASH
+            expect GT
+        }
+    }
 
-        assertEquals(LT, r[0].type)
+    @Test
+    void dollarGroovyNestedBraces() {
+        this.test('<Test key=${ test.each { it.test() } } />') {
+            expect LT
+            expect IDENTIFIER, 'Test'
+            expect KEY, 'key'
+            expect EQUALS
+            expect DOLLAR
+            expect CURLY_OPEN
+            expect GROOVY, ' test.each { it.test() } '
+            expect CURLY_CLOSE
+            expect FORWARD_SLASH
+            expect GT
+        }
+    }
 
-        assertEquals(IDENTIFIER, r[1].type)
+    @Test
-        assertEquals('Test', r[1].text)
+    void dollarReference() {
-
+        this.test('<Test key=$test />') {
-        assertEquals(KEY, r[2].type)
+            expect LT
-        assertEquals('key', r[2].text)
+            expect IDENTIFIER, 'Test'
-
+            expect KEY, 'key'
-        assertEquals(EQUALS, r[3].type)
+            expect EQUALS
-
+            expect DOLLAR
-        assertEquals(DOLLAR, r[4].type)
+            expect GROOVY_IDENTIFIER, 'test'
-
+            expect FORWARD_SLASH
-        assertEquals(CURLY_OPEN, r[5].type)
+            expect GT
-
+        }
-        assertEquals(GROOVY, r[6].type)
-        assertEquals(' test ', r[6].text)
-
-        assertEquals(CURLY_CLOSE, r[7].type)
-
-        assertEquals(FORWARD_SLASH, r[8].type)
-
-        assertEquals(GT, r[9].type)
     }
 
 }