typeset/parse.go

79 lines
1.7 KiB
Go
Raw Permalink Normal View History

2024-09-10 09:19:57 -06:00
package typeset
import "unicode"
// TODO perhaps follow https://unicode.org/reports/tr14/
func parseString (text string) ([]runeLayout, []token) {
// alloc initial rune slice
runes := make([]runeLayout, 0, len(text) * 2 / 3)
// build the rune slice
// we need to do this before parsing into tokens, because otherwise
// a realloc will occur in the middle of it and the tokens at the start
// will be referencing old memory
for _, run := range text {
runes = append(runes, runeLayout {
run: run,
})
}
// alloc initial token slice
tokens := make([]token, 0, len(runes) / 5)
2024-09-10 09:19:57 -06:00
var index int
var startingIndex int
2024-09-18 21:48:21 -06:00
var runl runeLayout
2024-09-10 09:19:57 -06:00
var lastRune rune
var tok token
tokenBoundary := func () {
if startingIndex != index {
tok.runes = runes[startingIndex:index]
startingIndex = index
tokens = append(tokens, tok)
}
tok = token { }
}
mustBeInToken := func (kind tokenKind) {
if tok.kind != kind {
tokenBoundary()
tok.kind = kind
}
}
2024-09-18 21:48:21 -06:00
// parse tokens
for index, runl = range runes {
2024-09-10 09:19:57 -06:00
switch {
2024-09-18 21:48:21 -06:00
case runl.run == '\r':
2024-09-10 09:19:57 -06:00
tokenBoundary()
// we don't know the token type yet. if next rune is a
// \n then this is a CRLF line break. if not, this is
// just a word.
2024-09-18 21:48:21 -06:00
case runl.run == '\n':
2024-09-10 09:19:57 -06:00
if lastRune == '\r' {
// continue the \r to make a CRLF line break
tok.kind = tokenKindLineBreak
} else {
tokenBoundary()
tok.kind = tokenKindLineBreak
}
2024-09-18 21:48:21 -06:00
case runl.run == '\t':
2024-09-10 09:19:57 -06:00
mustBeInToken(tokenKindTab)
2024-09-18 21:48:21 -06:00
case unicode.IsSpace(runl.run):
2024-09-10 09:19:57 -06:00
mustBeInToken(tokenKindSpace)
default:
mustBeInToken(tokenKindWord)
}
2024-09-18 21:48:21 -06:00
lastRune = runl.run
2024-09-10 09:19:57 -06:00
}
index ++ // make index equal to len([]rune(text))
tokenBoundary()
return runes, tokens
}