mars
/
tml
1
2
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Minor WithSource cleanup

This commit is contained in:
mars 2022-10-11 01:56:11 -06:00
parent a0edf84c01
commit 10a324172b
2 changed files with 70 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
src/ast.rs
View File

@ -1,12 +1,12 @@
use std::fmt::Debug;
use std::ops::Range;
use std::sync::Arc;
use std::{fmt::Debug, ops::Deref};
use logos::{Lexer, Logos};
#[derive(Logos, Clone, Debug, PartialEq, Eq)]
#[derive(Logos, Copy, Clone, Debug, PartialEq, Eq)]
#[logos(subpattern ident = r"[a-zA-Z][a-zA-Z0-9]*")]
pub enum Token {
pub enum TokenKind {
#[token("(")]
ParenOpen,
@ -39,16 +39,16 @@ pub enum Token {
}
#[derive(Debug, Clone)]
pub struct SpannedToken {
pub kind: Token,
pub struct TokenInner {
pub kind: TokenKind,
pub content: String,
pub source: Arc<Source>,
pub range: SourceRange,
}
pub type Token = WithSource<TokenInner>;
#[derive(Debug)]
pub struct AstBuilder<'a> {
lexer: Lexer<'a, Token>,
lexer: Lexer<'a, TokenKind>,
source: &'a Arc<Source>,
current_row_idx: usize,
current_row_start: usize,
@ -58,7 +58,7 @@ pub struct AstBuilder<'a> {
impl<'a> AstBuilder<'a> {
pub fn new(source: &'a Arc<Source>) -> Self {
Self {
lexer: Token::lexer(&source.full),
lexer: TokenKind::lexer(&source.full),
current_row_idx: 0,
current_row_start: 0,
paren_level: 0,
@ -84,17 +84,17 @@ impl<'a> AstBuilder<'a> {
self.span_to_range(self.lexer.span())
}
pub fn make_error(&self, kind: ParseErrorKind, range: SourceRange) -> ParseError {
ParseError {
pub fn with_source<T>(&self, inner: T, range: SourceRange) -> WithSource<T> {
WithSource {
inner,
source: self.source.clone(),
kind,
range,
}
}
pub fn token_to_string(&self, token: SpannedToken) -> ParseResult<WithSource<String>> {
pub fn token_to_string(&self, token: Token) -> ParseResult<WithSource<String>> {
match token.kind {
Token::String => {
TokenKind::String => {
let slice = self.lexer.slice();
let end = slice.len() - 1;
let inner = slice[1..end].to_string();
@ -104,57 +104,56 @@ impl<'a> AstBuilder<'a> {
range: token.range,
})
}
Token::Symbol => Ok(WithSource {
inner: token.content,
TokenKind::Symbol => Ok(WithSource {
inner: token.content.clone(),
source: token.source,
range: token.range,
}),
Token::Keyword => Ok(WithSource {
TokenKind::Keyword => Ok(WithSource {
inner: token.content[1..].to_string(),
source: token.source,
range: token.range,
}),
_ => Err(self.make_error(
ParseErrorKind::UnexpectedToken(token.kind, None),
token.range,
)),
_ => Err(token.map(ParseErrorKind::UnexpectedToken(token.kind, None))),
}
}
pub fn next(&mut self) -> ParseResult<SpannedToken> {
pub fn next(&mut self) -> ParseResult<Token> {
loop {
let token = self.lexer.next().unwrap_or(Token::Eof);
let token = self.lexer.next().unwrap_or(TokenKind::Eof);
let span = self.lexer.span();
if let Token::Newline = token {
if let TokenKind::Newline = token {
self.current_row_idx += 1;
self.current_row_start = span.start;
} else if let Token::Error = token {
} else if let TokenKind::Error = token {
let range = self.span_to_range(span);
break Err(self.make_error(ParseErrorKind::InvalidToken, range));
break Err(self.with_source(ParseErrorKind::InvalidToken, range));
} else {
let range = self.span_to_range(span);
if let Token::ParenOpen = token {
if let TokenKind::ParenOpen = token {
self.paren_level += 1;
}
if let Token::ParenClose = token {
if let TokenKind::ParenClose = token {
if self.paren_level == 0 {
return Err(self.make_error(ParseErrorKind::UnmatchedParen, range));
return Err(self.with_source(ParseErrorKind::UnmatchedParen, range));
}
self.paren_level -= 1;
}
if let Token::Eof = token {
if let TokenKind::Eof = token {
if self.paren_level > 0 {
return Err(self.make_error(ParseErrorKind::UnmatchedParen, range));
return Err(self.with_source(ParseErrorKind::UnmatchedParen, range));
}
}
break Ok(SpannedToken {
kind: token,
content: self.lexer.slice().to_string(),
break Ok(Token {
inner: TokenInner {
kind: token,
content: self.lexer.slice().to_string(),
},
source: self.source.to_owned(),
range,
});
@ -162,11 +161,10 @@ impl<'a> AstBuilder<'a> {
}
}
pub fn expect_next(&mut self) -> ParseResult<SpannedToken> {
pub fn expect_next(&mut self) -> ParseResult<Token> {
let tok = self.next()?;
if let Token::Eof = tok.kind {
let range = self.span_to_range(self.lexer.span());
Err(self.make_error(ParseErrorKind::UnexpectedEof, range))
if let TokenKind::Eof = tok.kind {
Err(tok.map(ParseErrorKind::UnexpectedEof))
} else {
Ok(tok)
}
@ -177,9 +175,9 @@ impl<'a> AstBuilder<'a> {
let children = loop {
let token = self.next()?;
match token.kind {
Token::Eof => break vec![],
Token::ParenClose => break vec![],
Token::Keyword => {
TokenKind::Eof => break vec![],
TokenKind::ParenClose => break vec![],
TokenKind::Keyword => {
let name = self.token_to_string(token)?;
let val = self.expect_value()?;
args.insert(Arg { name, val })?;
@ -191,8 +189,8 @@ impl<'a> AstBuilder<'a> {
Ok(TagBody { args, children })
}
pub fn parse_list_from(&mut self, mut token: SpannedToken) -> ParseResult<Vec<Value>> {
if let Token::Symbol = token.kind {
pub fn parse_list_from(&mut self, mut token: Token) -> ParseResult<Vec<Value>> {
if let TokenKind::Symbol = token.kind {
let start = token.range.start.clone();
let id = self.token_to_string(token)?;
let body = self.parse_tag_body()?;
@ -214,12 +212,12 @@ impl<'a> AstBuilder<'a> {
Ok(vals)
}
pub fn parse_value_from(&mut self, token: SpannedToken) -> ParseResult<Option<Value>> {
pub fn parse_value_from(&mut self, token: Token) -> ParseResult<Option<Value>> {
match token.kind {
Token::Eof | Token::ParenClose => Ok(None),
Token::String => Ok(Some(self.token_to_string(token)?.to_val_string())),
Token::Symbol => Ok(Some(self.token_to_string(token)?.to_val_symbol())),
Token::ParenOpen => {
TokenKind::Eof | TokenKind::ParenClose => Ok(None),
TokenKind::String => Ok(Some(self.token_to_string(token)?.to_val_string())),
TokenKind::Symbol => Ok(Some(self.token_to_string(token)?.to_val_symbol())),
TokenKind::ParenOpen => {
let start = token.range.end;
let next = self.expect_next()?;
let vals = self.parse_list_from(next)?;
@ -230,17 +228,14 @@ impl<'a> AstBuilder<'a> {
range: start..end,
}))
}
_ => Err(self.make_error(
ParseErrorKind::UnexpectedToken(token.kind, None),
self.span_to_range(self.lexer.span()),
)),
_ => Err(token.map(ParseErrorKind::UnexpectedToken(token.kind, None))),
}
}
pub fn expect_value(&mut self) -> ParseResult<Value> {
let next = self.expect_next()?;
self.parse_value_from(next)?
.ok_or_else(|| self.make_error(ParseErrorKind::UnexpectedEof, self.lexer_range()))
.ok_or_else(|| self.with_source(ParseErrorKind::UnexpectedEof, self.lexer_range()))
}
}
@ -273,6 +268,14 @@ impl<T> AsRef<T> for WithSource<T> {
}
}
impl<T> Deref for WithSource<T> {
type Target = T;
fn deref(&self) -> &T {
&self.inner
}
}
impl WithSource<String> {
pub fn to_val_string(self) -> Value {
Value {
@ -291,6 +294,16 @@ impl WithSource<String> {
}
}
impl<T> WithSource<T> {
pub fn map<O>(&self, other: O) -> WithSource<O> {
WithSource {
inner: other,
source: self.source.to_owned(),
range: self.range.clone(),
}
}
}
#[derive(Clone, Debug, Default)]
pub struct Args(Vec<Arg>);
@ -303,9 +316,9 @@ impl Args {
for existing in self.0.iter() {
if arg.name.inner == existing.name.inner {
return Err(ParseError {
kind: ParseErrorKind::DuplicateArgument(arg.name.inner),
source: arg.name.source,
range: arg.name.range,
inner: ParseErrorKind::DuplicateArgument(arg.name.inner),
});
}
}
@ -356,20 +369,14 @@ pub struct SourcePosition {
}
pub type SourceRange = Range<SourcePosition>;
pub type ParseError = WithSource<ParseErrorKind>;
pub type ParseResult<T> = Result<T, ParseError>;
#[derive(Clone, Debug)]
pub struct ParseError {
pub source: Arc<Source>,
pub range: SourceRange,
pub kind: ParseErrorKind,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ParseErrorKind {
InvalidToken,
DuplicateArgument(String),
UnmatchedParen,
UnexpectedToken(Token, Option<Token>),
UnexpectedToken(TokenKind, Option<TokenKind>),
UnexpectedEof,
}

2
src/main.rs
View File

@ -13,7 +13,7 @@ fn main() {
let src_path = std::env::args().nth(1).unwrap();
let src = std::fs::read_to_string(src_path).unwrap();
let mut lexer = ast::Token::lexer(&src);
let lexer = ast::TokenKind::lexer(&src);
let tokens: Vec<_> = lexer.collect();
println!("{:?}", tokens);