Add measurement stage

Closes #1

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 8
+charset = utf-8
+
+[*.md]
+indent_style = space
+indent_size = 2

README.md

@@ -1,6 +1,21 @@
 # typeset
 
-Typeset provides utilities for text layout, wrapping, and rendering.
+[![Go Reference](https://pkg.go.dev/badge/git.tebibyte.media/tomo/typeset.svg)](https://pkg.go.dev/git.tebibyte.media/tomo/typeset)
 
-The state of a text layout is stored in a TypeSetter, and it can be drawn to any
-image.Image using a Drawer which "extends" TypeSetter.
+Typeset provides utilities for text layout, wrapping, and rendering. It is
+designed to avoid redundant work and minimize memory allocations wherever
+posible in situations where the bounds of a section of text may change
+frequently and its content semi-frequently. Text layout is performed by the
+TypeSetter struct, which operates in a three-phase process:
+
+1. Tokenization
+2. Measurement
+3. Layout, alignment
+
+The results of these phases are memoized. When the state of the TypeSetter is
+queried, it will run through only the required phases before returning a value.
+
+The contents of a TypeSetter can be drawn onto any draw.Image using the Draw
+function included within this package, but it is entirely possible to create a
+custom draw function that iterates over TypeSetter.Runes that uses some other
+method of drawing that's faster than five gazillion virtual method calls.

drawer.go

@@ -1,52 +0,0 @@
-package typeset
-
-import "image"
-import "unicode"
-import "image/draw"
-import "image/color"
-import "golang.org/x/image/math/fixed"
-
-// Drawer is an extended TypeSetter that is able to draw text. Much like
-// TypeSetter, It has no constructor and its zero value can be used safely.
-type Drawer struct { TypeSetter }
-
-// Draw draws the drawer's text onto the specified canvas at the given offset.
-func (drawer Drawer) Draw (
-	destination draw.Image,
-	color       color.Color,
-	offset      image.Point,
-) (
-	updatedRegion image.Rectangle,
-) {
-	source := image.NewUniform(color)
-	
-	drawer.For (func (
-		index    int,
-		char     rune,
-		position fixed.Point26_6,
-	) bool {
-		destinationRectangle,
-		mask, maskPoint, _, ok := drawer.face.Glyph (
-			fixed.P (
-				offset.X + position.X.Round(),
-				offset.Y + position.Y.Round()),
-			char)
-		if !ok || unicode.IsSpace(char) || char == 0 {
-			return true
-		}
-
-		// FIXME:? clip destination rectangle if we are on the cusp of
-		// the maximum height.
-
-		draw.DrawMask (
-			destination,
-			destinationRectangle,
-			source, image.Point { },
-			mask, maskPoint,
-			draw.Over)
-
-		updatedRegion = updatedRegion.Union(destinationRectangle)
-		return true
-	})
-	return
-}

measure.go

@@ -0,0 +1,45 @@
+package typeset
+
+import "golang.org/x/image/font"
+import "golang.org/x/image/math/fixed"
+
+func measure (tokens []token, face font.Face) {
+	var lastRune rune
+	for index, token := range tokens {
+		var x fixed.Int26_6
+		for index, runl := range token.runes {
+			advance, ok := face.GlyphAdvance(runl.run)
+			if !ok { advance = tofuAdvance(face) }
+			advance += face.Kern(lastRune, runl.run)
+			
+			runl.x = x
+			x += advance
+			lastRune = runl.run
+			token.runes[index] = runl
+		}
+		token.width = x
+		tokens[index] = token
+	}
+}
+
+const tofuStandinRune     = 'M'
+const fallbackTofuAdvance = 16
+const fallbackTofuWidth   = 14
+const fallbackTofuAscend  = 16
+
+func tofuAdvance (face font.Face) fixed.Int26_6 {
+	if advance, ok := face.GlyphAdvance(tofuStandinRune); ok {
+		return advance
+	} else {
+		return fallbackTofuAdvance
+	}
+}
+
+func tofuBounds (face font.Face) (fixed.Rectangle26_6, fixed.Int26_6) {
+	if bounds, advance, ok := face.GlyphBounds(tofuStandinRune); ok {
+		return bounds, advance
+	} else {
+		return fixed.R(0, -fallbackTofuAscend, fallbackTofuWidth, 0),
+			fallbackTofuAdvance
+	}
+}

measure_test.go

@@ -0,0 +1,70 @@
+package typeset
+
+import "testing"
+import "golang.org/x/image/math/fixed"
+import "golang.org/x/image/font/basicfont"
+
+const basicfontFace7x13advance = 7
+
+func tkw (kind tokenKind, value string, width fixed.Int26_6) token {
+	tok := tk(kind, value)
+	tok.width = width
+	for index, runl := range tok.runes {
+		runl.x = fixed.I(basicfontFace7x13advance * index)
+		tok.runes[index] = runl
+	}
+	return tok
+}
+
+func TestMeasure (test *testing.T) {
+	// ---- processing ----
+	tokens := []token {
+		tk(tokenKindWord, "hello"),
+		tk(tokenKindSpace, " "),
+		tk(tokenKindWord, "\rworld!"),
+		tk(tokenKindLineBreak, "\n"),
+		tk(tokenKindWord, "foo"),
+		tk(tokenKindLineBreak, "\n"),
+		tk(tokenKindLineBreak, "\r\n"),
+		tk(tokenKindWord, "bar"),
+		tk(tokenKindTab, "\t"),
+		tk(tokenKindWord, "baz"),
+		tk(tokenKindTab, "\t\t"),
+		tk(tokenKindWord, "something"),
+	}
+	measure(tokens, basicfont.Face7x13)
+	
+	// ---- correct data ----
+	correctTokens := []token {
+		tkw(tokenKindWord,      "hello",     fixed.I(35)),
+		tkw(tokenKindSpace,     " ",         fixed.I( 7)),
+		tkw(tokenKindWord,      "\rworld!",  fixed.I(49)),
+		tkw(tokenKindLineBreak, "\n",        fixed.I( 7)),
+		tkw(tokenKindWord,      "foo",       fixed.I(21)),
+		tkw(tokenKindLineBreak, "\n",        fixed.I( 7)),
+		tkw(tokenKindLineBreak, "\r\n",      fixed.I(14)),
+		tkw(tokenKindWord,      "bar",       fixed.I(21)),
+		tkw(tokenKindTab,       "\t",        fixed.I( 7)),
+		tkw(tokenKindWord,      "baz",       fixed.I(21)),
+		tkw(tokenKindTab,       "\t\t",      fixed.I(14)),
+		tkw(tokenKindWord,      "something", fixed.I(63)),
+	}
+
+	// ---- testing ----
+	if len(tokens) != len(correctTokens) {
+		test.Logf("len(tokens) != len(correctTokens): %d, %d", len(tokens), len(correctTokens))
+		test.Log("GOT")
+		logTokens(test, tokens)
+		test.Log("CORRECT")
+		logTokens(test, correctTokens)
+		test.FailNow()
+	}
+	if !compareTokens(tokens, correctTokens) {
+		test.Log("tokens != correctTokens:")
+		test.Log("GOT")
+		logTokens(test, tokens)
+		test.Log("CORRECT")
+		logTokens(test, correctTokens)
+		test.FailNow()
+	}
+}

old/drawer.go

@@ -0,0 +1,84 @@
+package typeset
+
+import "image"
+import "unicode"
+import "image/draw"
+import "image/color"
+import "golang.org/x/image/math/fixed"
+
+// Drawer is an extended TypeSetter that is able to draw text. Much like
+// TypeSetter, It has no constructor and its zero value can be used safely.
+type Drawer struct { TypeSetter }
+
+// Draw draws the drawer's text onto the specified canvas at the given offset.
+func (drawer Drawer) Draw (
+	destination draw.Image,
+	col         color.Color,
+	offset      image.Point,
+) (
+	updatedRegion image.Rectangle,
+) {
+	source := image.NewUniform(col)
+	
+	drawer.ForRunes (func (
+		index    int,
+		char     rune,
+		position fixed.Point26_6,
+	) bool {
+		// leave empty space for space characters
+		if unicode.IsSpace(char) {
+			return true
+		}
+		
+		dot := fixed.P (
+			offset.X + position.X.Round(),
+			offset.Y + position.Y.Round())
+		destinationRectangle,
+		mask, maskPoint, _, ok := drawer.face.Glyph(dot, char)
+		// tofu
+		if !ok {
+			drawer.drawTofu(char, destination, col, dot)
+			return true
+		}
+
+		// FIXME:? clip destination rectangle if we are on the cusp of
+		// the maximum height.
+
+		draw.DrawMask (
+			destination,
+			destinationRectangle,
+			source, image.Point { },
+			mask, maskPoint,
+			draw.Over)
+
+		updatedRegion = updatedRegion.Union(destinationRectangle)
+		return true
+	})
+	return
+}
+
+func (drawer Drawer) drawTofu (
+	char rune,
+	destination draw.Image,
+	col         color.Color,
+	position    fixed.Point26_6,
+) {
+	bounds, _ := tofuBounds(drawer.face)
+	rectBounds := image.Rect (
+		bounds.Min.X.Round(),
+		bounds.Min.Y.Round(),
+		bounds.Max.X.Round(),
+		bounds.Max.Y.Round()).Add(image.Pt(
+			position.X.Round(),
+			position.Y.Round()))
+	for x := rectBounds.Min.X; x < rectBounds.Max.X; x ++ {
+		destination.Set(x, rectBounds.Min.Y, col)
+	}
+	for y := rectBounds.Min.Y; y < rectBounds.Max.Y; y ++ {
+		destination.Set(rectBounds.Min.X, y, col)
+		destination.Set(rectBounds.Max.X - 1, y, col)
+	}
+	for x := rectBounds.Min.X; x < rectBounds.Max.X; x ++ {
+		destination.Set(x, rectBounds.Max.Y - 1, col)
+	}
+}

old/layout.go

@@ -58,13 +58,16 @@ func DoWord (text []rune, face font.Face) (word WordLayout, remaining []rune) {
 		
 		// consume and process the rune
 		remaining = remaining[1:]
-		_, advance, ok := face.GlyphBounds(char)
-		if !ok { continue }
-		word.Runes = append (word.Runes, RuneLayout {
+		advance, ok := face.GlyphAdvance(char)
+		if !ok {
+			advance = tofuAdvance(face)
+		}
+		runeLayout := RuneLayout {
 			X:     x,
 			Width: advance,
 			Rune:  char,
-		})
+		}
+		word.Runes = append(word.Runes, runeLayout)
 
 		// advance
 		if gettingSpace {
@@ -113,12 +116,10 @@ type LineLayout struct {
 func DoLine (text []rune, face font.Face, wrap bool, width fixed.Int26_6) (line LineLayout, remaining []rune) {
 	remaining    = text
 	x           := fixed.Int26_6(0)
-	lastWord    := WordLayout { }
 	isFirstWord := true
 	for {
 		// process one word
 		word, remainingFromWord := DoWord(remaining, face)
-		word.X = x
 		x += word.Width
 
 		// if we have gone over the preferred width, stop processing
@@ -132,7 +133,6 @@ func DoLine (text []rune, face font.Face, wrap bool, width fixed.Int26_6) (line
 
 		// if the word actually has contents, add it
 		if word.Runes != nil {
-			lastWord   = word
 			line.Words = append(line.Words, word)
 		}
 
@@ -148,9 +148,13 @@ func DoLine (text []rune, face font.Face, wrap bool, width fixed.Int26_6) (line
 	}
 
 	// set the width of the line's content.
-	line.ContentWidth = lastWord.X + lastWord.Width
 	line.Width = width
-	line.SpaceAfter = lastWord.SpaceAfter
+	// TODO: just have RecommendedHeight want aligned layout?
+	if len(line.Words) > 0 {
+		lastWord := line.Words[len(line.Words) - 1]
+		line.ContentWidth = x - lastWord.SpaceAfter
+		line.SpaceAfter   = lastWord.SpaceAfter
+	}
 	return
 }
 
@@ -167,30 +171,45 @@ func (line *LineLayout) Length () int {
 
 // Align aligns the text in the line according to the specified alignment
 // method.
-func (line *LineLayout) Align (align Align) {
+func (line *LineLayout) Align (align Align, tabWidth fixed.Int26_6) {
 	if len(line.Words) == 0 { return }
 	
 	if align == AlignEven {
-		line.justify()
-		return
-	}
+		line.justify(tabWidth)
+	} else {
+		line.contract(tabWidth)
+		
+		var leftOffset fixed.Int26_6
+		if align == AlignMiddle {
+			leftOffset = (line.Width - line.ContentWidth) / 2
+		} else if align == AlignEnd {
+			leftOffset = line.Width - line.ContentWidth
+		}
 
-	leftOffset := -line.Words[0].X
-
-	if align == AlignMiddle {
-		leftOffset += (line.Width - line.ContentWidth) / 2
-	} else if align == AlignEnd {
-		leftOffset += line.Width - line.ContentWidth
-	}
-
-	for index := range line.Words {
-		line.Words[index].X += leftOffset
+		for index := range line.Words {
+			line.Words[index].X += leftOffset
+		}
 	}
 }
 
-func (line *LineLayout) justify () {
-	if len(line.Words) < 2 {
-		line.Align(AlignStart)
+// assume line has content > 0
+func (line *LineLayout) contract (tabWidth fixed.Int26_6) {
+	x := fixed.Int26_6(0)
+	for index, word := range line.Words {
+		word.X = x
+		x += word.Width
+		x += word.SpaceAfter
+		line.Words[index] = word
+	}
+	lastWord := line.Words[len(line.Words) - 1]
+	line.ContentWidth = lastWord.X + lastWord.Width
+	line.SpaceAfter   = lastWord.SpaceAfter
+}
+
+// assume line has content > 0
+func (line *LineLayout) justify (tabWidth fixed.Int26_6) {
+	if len(line.Words) <= 1 {
+		line.Align(AlignStart, tabWidth)
 		return
 	}
 
@@ -209,3 +228,23 @@ func (line *LineLayout) justify () {
 		x += spacePerWord + word.Width
 	}
 }
+
+func tabStop (x, tabWidth fixed.Int26_6, delta int) fixed.Int26_6 {
+	return fixed.I((tabWidth * 64 / x).Floor() + delta).Mul(tabWidth)
+}
+
+func tofuAdvance (face font.Face) fixed.Int26_6 {
+	if advance, ok := face.GlyphAdvance('M'); ok {
+		return advance
+	} else {
+		return 16
+	}
+}
+
+func tofuBounds (face font.Face) (fixed.Rectangle26_6, fixed.Int26_6) {
+	if bounds, advance, ok := face.GlyphBounds('M'); ok {
+		return bounds, advance
+	} else {
+		return fixed.R(0, -16, 14, 0), 16
+	}
+}

old/setter.go

@@ -18,6 +18,7 @@ type TypeSetter struct {
 	face           font.Face
 	width, height  int
 	wrap           bool
+	tabWidth       fixed.Int26_6
 	
 	minWidth          fixed.Int26_6
 	layoutBounds      image.Rectangle
@@ -100,6 +101,7 @@ func (setter *TypeSetter) needAlignedLayout () {
 	setter.alignVertically()
 }
 
+// should only be called from within setter.needAlignedLayout
 func (setter *TypeSetter) alignHorizontally () {
 	if len(setter.lines) == 0 { return }
 	
@@ -116,10 +118,11 @@ func (setter *TypeSetter) alignHorizontally () {
 		}
 
 		// align line
-		setter.lines[index].Align(align)
+		setter.lines[index].Align(align, setter.tabWidth)
 	}
 }
 
+// should only be called from within setter.needAlignedLayout
 func (setter *TypeSetter) alignVertically () {
 	if setter.height     == 0 { return }
 	if len(setter.lines) == 0 { return }
@@ -129,20 +132,27 @@ func (setter *TypeSetter) alignVertically () {
 	}
 
 	// determine how much to shift lines
-	topOffset     := 0
+	topOffset     := fixed.I(0)
 	contentHeight := setter.layoutBoundsSpace.Dy()
 	if setter.vAlign == AlignMiddle {
-		topOffset += (setter.height - contentHeight) / 2
+		topOffset += fixed.I((setter.height - contentHeight) / 2)
 	} else if setter.vAlign == AlignEnd {
-		topOffset += setter.height - contentHeight
+		topOffset += fixed.I(setter.height - contentHeight)
+	}
+
+	// we may be re-aligning already aligned text. if the text is shifted
+	// away from the origin, account for that.
+	if len(setter.lines) > 0 {
+		topOffset -= setter.lines[0].Y
 	}
 
 	// shift lines
 	for index := range setter.lines {
-		setter.lines[index].Y += fixed.I(topOffset)
+		setter.lines[index].Y += topOffset
 	}
 }
 
+// should only be called from within setter.alignVertically
 func (setter *TypeSetter) justifyVertically () {
 	spaceCount    := len(setter.lines) - 1
 	contentHeight := setter.layoutBoundsSpace.Dy()
@@ -206,6 +216,14 @@ func (setter *TypeSetter) SetHeight (heignt int) {
 	setter.height = heignt
 }
 
+// SetTabWidth sets the distance between tab stops.
+func (setter *TypeSetter) SetTabWidth (tabWidth fixed.Int26_6) {
+	if setter.tabWidth == tabWidth { return }
+	setter.layoutClean = false
+	setter.alignClean  = false
+	setter.tabWidth = tabWidth
+}
+
 // Em returns the width of one emspace according to the typesetter's font, which
 // is the width of the capital letter 'M'.
 func (setter *TypeSetter) Em () (width fixed.Int26_6) {
@@ -250,8 +268,18 @@ type RuneIterator func (
 )
 
 // For calls the specified iterator for every rune in the typesetter. If the
-// iterator returns false, the loop will immediately stop.
+// iterator returns false, the loop will immediately stop. This method will
+// insert a fake null rune at the end.
 func (setter *TypeSetter) For (iterator RuneIterator) {
+	setter.forInternal(iterator, true)
+}
+
+// ForRunes is like For, but leaves out the fake null rune.
+func (setter *TypeSetter) ForRunes (iterator RuneIterator) {
+	setter.forInternal(iterator, false)
+}
+
+func (setter *TypeSetter) forInternal (iterator RuneIterator, fakeNull bool) {
 	setter.needAlignedLayout()
 
 	index := 0
@@ -262,7 +290,7 @@ func (setter *TypeSetter) For (iterator RuneIterator) {
 		for _, word := range line.Words {
 		for _, char := range word.Runes {
 			lastCharRightBound = word.X + char.X + char.Width
-			keepGoing := iterator (index, char.Rune, fixed.Point26_6 {
+			keepGoing := iterator(index, char.Rune, fixed.Point26_6 {
 				X: word.X + char.X,
 				Y: line.Y,
 			})
@@ -271,21 +299,24 @@ func (setter *TypeSetter) For (iterator RuneIterator) {
 		}}
 		
 		if line.BreakAfter {
-			keepGoing := iterator (index, '\n', fixed.Point26_6 {
+			keepGoing := iterator(index, '\n', fixed.Point26_6 {
 				X: lastCharRightBound,
 				Y: line.Y,
 			})
 			if !keepGoing { return }
 			index ++
+			lastCharRightBound = fixed.Int26_6(0)
 		}
 	}
 	
-	keepGoing := iterator (index, '\000', fixed.Point26_6 {
-		X: lastCharRightBound,
-		Y: lastLineY,
-	})
-	if !keepGoing { return }
-	index ++
+	if fakeNull {
+		keepGoing := iterator (index, '\000', fixed.Point26_6 {
+			X: lastCharRightBound,
+			Y: lastLineY,
+		})
+		if !keepGoing { return }
+		index ++
+	}
 }
 
 // AtPosition returns the index of the rune at the specified position.
@@ -364,10 +395,10 @@ func (setter *TypeSetter) MinimumSize () image.Point {
 	return image.Pt(width.Round(), height.Round())
 }
 
-// ReccomendedHeightFor returns the reccomended max height if the text were to
-// have its maximum width set to the given width. This does not alter the
+// RecommendedHeight returns the reccomended max height if the text were to have
+// its maximum width set to the given width. This does not alter the
 // typesetter's state.
-func (setter *TypeSetter) ReccomendedHeightFor (width int) (height int) {
+func (setter *TypeSetter) RecommendedHeight (width int) (height int) {
 	setter.needLayout()
 	
 	if setter.lines == nil { return }

parse.go

@@ -0,0 +1,70 @@
+package typeset
+
+import "unicode"
+
+// TODO perhaps follow https://unicode.org/reports/tr14/
+
+func parseString (text string) ([]runeLayout, []token) {
+	// TODO find an optimal size for both of these to minimize allocs. will
+	// require some testing.
+	runes  := make([]runeLayout, 0, len(text) * 2 / 3)
+	tokens := make([]token,      0, len(text) / 4)
+
+	var index         int
+	var startingIndex int
+	var run           rune
+	var lastRune      rune
+
+	var tok token
+	tokenBoundary := func () {
+		if startingIndex != index {
+			tok.runes = runes[startingIndex:index]
+			startingIndex = index
+			tokens = append(tokens, tok)
+		}
+		tok = token { }
+	}
+	mustBeInToken := func (kind tokenKind) {
+		if tok.kind != kind {
+			tokenBoundary()
+			tok.kind = kind
+		}
+	}
+
+	for index, run = range text {
+		runes = append(runes, runeLayout {
+			run: run,
+		})
+
+		switch {
+		case run == '\r':
+			tokenBoundary()
+			// we don't know the token type yet. if next rune is a
+			// \n then this is a CRLF line break. if not, this is
+			// just a word.
+			
+		case run == '\n':
+			if lastRune == '\r' {
+				// continue the \r to make a CRLF line break
+				tok.kind = tokenKindLineBreak
+			} else {
+				tokenBoundary()
+				tok.kind = tokenKindLineBreak
+			}
+
+		case run == '\t':
+			mustBeInToken(tokenKindTab)
+
+		case unicode.IsSpace(run):
+			mustBeInToken(tokenKindSpace)
+
+		default:
+			mustBeInToken(tokenKindWord)
+		}
+		lastRune = run
+	}
+	index ++ // make index equal to len([]rune(text))
+	
+	tokenBoundary()
+	return runes, tokens
+}

parse_test.go

@@ -0,0 +1,126 @@
+package typeset
+
+import "slices"
+import "testing"
+
+func rl (run rune) runeLayout { return runeLayout { run: run } }
+func tk (kind tokenKind, value string) token {
+	tok := token {
+		kind: kind,
+	}
+	runeValue := []rune(value)
+	tok.runes = make([]runeLayout, len(runeValue))
+	for index, run := range runeValue {
+		tok.runes[index] = rl(run)
+	}
+	return tok
+}
+func compareTokens (got, correct []token) bool {
+	for index, tok := range got {
+		correctTok := correct[index]
+		isCorrect :=
+			correctTok.kind  == tok.kind &&
+			correctTok.width == tok.width &&
+			slices.Equal(correctTok.runes, tok.runes)
+		if !isCorrect { return false }
+	}
+	return true
+}
+func logTokens (test *testing.T, tokens []token) {
+	for _, token := range tokens {
+		test.Logf("- %-40v | %v", token, token.runes)
+	}
+}
+
+func TestParseString (test *testing.T) {
+	// ---- processing ----
+	runes, tokens := parseString("hello \rworld!\nfoo\n\r\nbar\tbaz\t\tsomething")
+
+	// ---- correct data ----
+	correctRunes := []runeLayout {
+		rl('h'),
+		rl('e'),
+		rl('l'),
+		rl('l'),
+		rl('o'),
+		rl(' '),
+		rl('\r'),
+		rl('w'),
+		rl('o'),
+		rl('r'),
+		rl('l'),
+		rl('d'),
+		rl('!'),
+		rl('\n'),
+		rl('f'),
+		rl('o'),
+		rl('o'),
+		rl('\n'),
+		rl('\r'),
+		rl('\n'),
+		rl('b'),
+		rl('a'),
+		rl('r'),
+		rl('\t'),
+		rl('b'),
+		rl('a'),
+		rl('z'),
+		rl('\t'),
+		rl('\t'),
+		rl('s'),
+		rl('o'),
+		rl('m'),
+		rl('e'),
+		rl('t'),
+		rl('h'),
+		rl('i'),
+		rl('n'),
+		rl('g'),
+	}
+	correctTokens := []token {
+		tk(tokenKindWord, "hello"),
+		tk(tokenKindSpace, " "),
+		tk(tokenKindWord, "\rworld!"),
+		tk(tokenKindLineBreak, "\n"),
+		tk(tokenKindWord, "foo"),
+		tk(tokenKindLineBreak, "\n"),
+		tk(tokenKindLineBreak, "\r\n"),
+		tk(tokenKindWord, "bar"),
+		tk(tokenKindTab, "\t"),
+		tk(tokenKindWord, "baz"),
+		tk(tokenKindTab, "\t\t"),
+		tk(tokenKindWord, "something"),
+	}
+
+	// ---- testing ----
+	if len(runes) != len(correctRunes) {
+		test.Logf("len(runes) != len(correctRunes): %d, %d", len(runes), len(correctRunes))
+		test.Log(runes)
+		test.Log(correctRunes)
+		test.FailNow()
+	}
+	if !slices.Equal(runes, correctRunes) {
+		test.Log("runes != correctRunes:")
+		test.Log(runes)
+		test.Log(correctRunes)
+		test.FailNow()
+	}
+	if len(tokens) != len(correctTokens) {
+		test.Logf("len(tokens) != len(correctTokens): %d, %d", len(tokens), len(correctTokens))
+		test.Log("GOT")
+		logTokens(test, tokens)
+		test.Log("CORRECT")
+		logTokens(test, correctTokens)
+		test.FailNow()
+	}
+	if !compareTokens(tokens, correctTokens) {
+		test.Log("tokens != correctTokens:")
+		test.Log("GOT")
+		logTokens(test, tokens)
+		test.Log("CORRECT")
+		logTokens(test, correctTokens)
+		test.FailNow()
+	}
+	// TODO: ensure runeLayout slices in the tokens reference the same
+	// memory as the complete runes slice
+}