tomo/typeset
tomo/typeset
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 12 Wiki Activity

Benchmarks report token waste

Browse Source
This commit is contained in:
Sasha Koshka 2024-09-19 09:33:14 -04:00
parent aa486fe660
commit a91816df6c
1 changed files with 26 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
parse_test.go
View File

@ -147,28 +147,40 @@ func TestParseString (test *testing.T) {
func BenchmarkParseStringLatin (benchmark *testing.B) {
benchmark.ReportAllocs()
var meanLen, meanCap int
var rmeanLen, rmeanCap int
var tmeanLen, tmeanCap int
for i := 0; i < benchmark.N; i ++ {
runes, _ := parseString(lipsumLt)
meanLen += len(runes)
meanCap += cap(runes)
runes, tokens := parseString(lipsumLt)
rmeanLen += len(runes)
rmeanCap += cap(runes)
tmeanLen += len(tokens)
tmeanCap += cap(tokens)
}
meanLen /= benchmark.N
meanCap /= benchmark.N
benchmark.ReportMetric(float64(meanCap) / float64(meanLen), "waste")
rmeanLen /= benchmark.N
rmeanCap /= benchmark.N
tmeanLen /= benchmark.N
tmeanCap /= benchmark.N
benchmark.ReportMetric(float64(rmeanCap) / float64(rmeanLen), "rune-waste")
benchmark.ReportMetric(float64(tmeanCap) / float64(tmeanLen), "token-waste")
}
func BenchmarkParseStringChinese (benchmark *testing.B) {
benchmark.ReportAllocs()
var meanLen, meanCap int
var rmeanLen, rmeanCap int
var tmeanLen, tmeanCap int
for i := 0; i < benchmark.N; i ++ {
runes, _ := parseString(lipsumCn)
meanLen += len(runes)
meanCap += cap(runes)
runes, tokens := parseString(lipsumCn)
rmeanLen += len(runes)
rmeanCap += cap(runes)
tmeanLen += len(tokens)
tmeanCap += cap(tokens)
}
meanLen /= benchmark.N
meanCap /= benchmark.N
benchmark.ReportMetric(float64(meanCap) / float64(meanLen), "waste")
rmeanLen /= benchmark.N
rmeanCap /= benchmark.N
tmeanLen /= benchmark.N
tmeanCap /= benchmark.N
benchmark.ReportMetric(float64(rmeanCap) / float64(rmeanLen), "rune-waste")
benchmark.ReportMetric(float64(tmeanCap) / float64(tmeanLen), "token-waste")
}
const lipsumLt =