Append punctuation tokens

DQNEO · Aug 29, 2019 · a7f2cc8 · a7f2cc8
1 parent 8df1ec0
commit a7f2cc8
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/main.go b/main.go
@@ -24,7 +24,7 @@ func ungetChar() {
 }
 
 type Token struct {
-	kind  string // "intliteral"
+	kind  string // "intliteral", "punct"
 	value string
 }
 
@@ -66,6 +66,13 @@ func tokenize() []*Token {
 			}
 			tokens = append(tokens, token)
 			fmt.Printf(" '%s'", token.value)
+		case ';':
+			token := &Token{
+				kind: "punct",
+				value: string([]byte{char}),
+			}
+			tokens = append(tokens, token)
+			fmt.Printf(" '%s'", token.value)
 		default:
 			panic(fmt.Sprintf("tokenizer: Invalid char: '%c'", char))
 		}