5 is the appears to be the sweet spot for average token len

2024-09-19 09:40:00 -04:00 · 2024-09-19 09:40:00 -04:00 · ff8f86e034
commit ff8f86e034
parent 288a1fb9ef
1 changed files with 1 additions and 1 deletions
--- a/parse.go
+++ b/parse.go
@ -19,7 +19,7 @@ func parseString (text string) ([]runeLayout, []token) {
 	}

 	// alloc initial token slice
-	tokens := make([]token, 0, len(runes) / 3)
+	tokens := make([]token, 0, len(runes) / 5)

 	var index         int
 	var startingIndex int