Add AST parser

Cargo.lock

@@ -31,6 +31,18 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "beef"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -43,6 +55,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01af5c4fc8506204d2280509e9d57d1b41e54e587c185ac09baef30d21ae63ed"
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "getrandom"
 version = "0.2.7"
@@ -104,6 +122,29 @@ version = "0.2.134"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb"
 
+[[package]]
+name = "logos"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf8b031682c67a8e3d5446840f9573eb7fe26efe7ec8d195c9ac4c0647c502f1"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c"
+dependencies = [
+ "beef",
+ "fnv",
+ "proc-macro2",
+ "quote",
+ "regex-syntax",
+ "syn",
+]
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -224,8 +265,10 @@ name = "tml"
 version = "0.1.0"
 dependencies = [
  "ansi_term",
+ "bitflags",
  "figlet-rs",
  "lexpr",
+ "logos",
  "strum",
  "textwrap",
 ]

Cargo.toml

@@ -4,8 +4,10 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+bitflags = "1.3"
 figlet-rs = "0.1"
 lexpr = "0.2"
+logos = "0.12"
 strum = { version = "0.24", features = ["derive"] }
 ansi_term = "0.12"
 textwrap = "0.15"

src/ast.rs

@@ -0,0 +1,375 @@
+use std::fmt::Debug;
+use std::ops::Range;
+use std::sync::Arc;
+
+use logos::{Lexer, Logos};
+
+#[derive(Logos, Clone, Debug, PartialEq, Eq)]
+#[logos(subpattern ident = r"[a-zA-Z][a-zA-Z0-9]*")]
+pub enum Token {
+    #[token("(")]
+    ParenOpen,
+
+    #[token(")")]
+    ParenClose,
+
+    #[regex(r"[a-zA-Z-][a-zA-Z0-9-]*")]
+    Symbol,
+
+    #[regex(":[a-zA-Z-][a-zA-Z0-9-]*")]
+    Keyword,
+
+    #[regex(r#""([^"\\]|\\t|\\u|\\n|\\")*""#)]
+    String,
+
+    #[regex(r";[^\n]*", logos::skip)]
+    Comment,
+
+    #[token("\n")]
+    Newline,
+
+    #[regex(r"[ \t\f\r]+", logos::skip)]
+    Whitespace,
+
+    #[logos(ignore)]
+    Eof,
+
+    #[error]
+    Error,
+}
+
+#[derive(Debug, Clone)]
+pub struct SpannedToken {
+    pub kind: Token,
+    pub content: String,
+    pub source: Arc<Source>,
+    pub range: SourceRange,
+}
+
+#[derive(Debug)]
+pub struct AstBuilder<'a> {
+    lexer: Lexer<'a, Token>,
+    source: &'a Arc<Source>,
+    current_row_idx: usize,
+    current_row_start: usize,
+    paren_level: usize,
+}
+
+impl<'a> AstBuilder<'a> {
+    pub fn new(source: &'a Arc<Source>) -> Self {
+        Self {
+            lexer: Token::lexer(&source.full),
+            current_row_idx: 0,
+            current_row_start: 0,
+            paren_level: 0,
+            source,
+        }
+    }
+
+    pub fn idx_to_position(&self, idx: usize) -> SourcePosition {
+        SourcePosition {
+            row: self.current_row_idx,
+            col: idx - self.current_row_start,
+        }
+    }
+
+    pub fn span_to_range(&self, span: Range<usize>) -> SourceRange {
+        Range {
+            start: self.idx_to_position(span.start),
+            end: self.idx_to_position(span.end),
+        }
+    }
+
+    pub fn lexer_range(&self) -> SourceRange {
+        self.span_to_range(self.lexer.span())
+    }
+
+    pub fn make_error(&self, kind: ParseErrorKind, range: SourceRange) -> ParseError {
+        ParseError {
+            source: self.source.clone(),
+            kind,
+            range,
+        }
+    }
+
+    pub fn token_to_string(&self, token: SpannedToken) -> ParseResult<WithSource<String>> {
+        match token.kind {
+            Token::String => {
+                let slice = self.lexer.slice();
+                let end = slice.len() - 1;
+                let inner = slice[1..end].to_string();
+                Ok(WithSource {
+                    inner,
+                    source: token.source,
+                    range: token.range,
+                })
+            }
+            Token::Symbol => Ok(WithSource {
+                inner: token.content,
+                source: token.source,
+                range: token.range,
+            }),
+            Token::Keyword => Ok(WithSource {
+                inner: token.content[1..].to_string(),
+                source: token.source,
+                range: token.range,
+            }),
+            _ => Err(self.make_error(
+                ParseErrorKind::UnexpectedToken(token.kind, None),
+                token.range,
+            )),
+        }
+    }
+
+    pub fn next(&mut self) -> ParseResult<SpannedToken> {
+        loop {
+            let token = self.lexer.next().unwrap_or(Token::Eof);
+            let span = self.lexer.span();
+            if let Token::Newline = token {
+                self.current_row_idx += 1;
+                self.current_row_start = span.start;
+            } else if let Token::Error = token {
+                let range = self.span_to_range(span);
+                break Err(self.make_error(ParseErrorKind::InvalidToken, range));
+            } else {
+                let range = self.span_to_range(span);
+
+                if let Token::ParenOpen = token {
+                    self.paren_level += 1;
+                }
+
+                if let Token::ParenClose = token {
+                    if self.paren_level == 0 {
+                        return Err(self.make_error(ParseErrorKind::UnmatchedParen, range));
+                    }
+
+                    self.paren_level -= 1;
+                }
+
+                if let Token::Eof = token {
+                    if self.paren_level > 0 {
+                        return Err(self.make_error(ParseErrorKind::UnmatchedParen, range));
+                    }
+                }
+
+                break Ok(SpannedToken {
+                    kind: token,
+                    content: self.lexer.slice().to_string(),
+                    source: self.source.to_owned(),
+                    range,
+                });
+            }
+        }
+    }
+
+    pub fn expect_next(&mut self) -> ParseResult<SpannedToken> {
+        let tok = self.next()?;
+        if let Token::Eof = tok.kind {
+            let range = self.span_to_range(self.lexer.span());
+            Err(self.make_error(ParseErrorKind::UnexpectedEof, range))
+        } else {
+            Ok(tok)
+        }
+    }
+
+    pub fn parse_tag_body(&mut self) -> ParseResult<TagBody> {
+        let mut args = Args::new();
+        let children = loop {
+            let token = self.next()?;
+            match token.kind {
+                Token::Eof => break vec![],
+                Token::ParenClose => break vec![],
+                Token::Keyword => {
+                    let name = self.token_to_string(token)?;
+                    let val = self.expect_value()?;
+                    args.insert(Arg { name, val })?;
+                }
+                _ => break self.parse_list_from(token)?,
+            }
+        };
+
+        Ok(TagBody { args, children })
+    }
+
+    pub fn parse_list_from(&mut self, mut token: SpannedToken) -> ParseResult<Vec<Value>> {
+        if let Token::Symbol = token.kind {
+            let start = token.range.start.clone();
+            let id = self.token_to_string(token)?;
+            let body = self.parse_tag_body()?;
+            let end = self.idx_to_position(self.lexer.span().end);
+            return Ok(vec![Value {
+                inner: ValueKind::Tag(Tag { id, body }),
+                source: self.source.to_owned(),
+                range: start..end,
+            }]);
+        }
+
+        let mut vals = Vec::new();
+
+        while let Some(val) = self.parse_value_from(token)? {
+            vals.push(val);
+            token = self.next()?;
+        }
+
+        Ok(vals)
+    }
+
+    pub fn parse_value_from(&mut self, token: SpannedToken) -> ParseResult<Option<Value>> {
+        match token.kind {
+            Token::Eof | Token::ParenClose => Ok(None),
+            Token::String => Ok(Some(self.token_to_string(token)?.to_val_string())),
+            Token::Symbol => Ok(Some(self.token_to_string(token)?.to_val_symbol())),
+            Token::ParenOpen => {
+                let start = token.range.end;
+                let next = self.expect_next()?;
+                let vals = self.parse_list_from(next)?;
+                let end = self.idx_to_position(self.lexer.span().end);
+                Ok(Some(Value {
+                    inner: ValueKind::List(vals),
+                    source: self.source.to_owned(),
+                    range: start..end,
+                }))
+            }
+            _ => Err(self.make_error(
+                ParseErrorKind::UnexpectedToken(token.kind, None),
+                self.span_to_range(self.lexer.span()),
+            )),
+        }
+    }
+
+    pub fn expect_value(&mut self) -> ParseResult<Value> {
+        let next = self.expect_next()?;
+        self.parse_value_from(next)?
+            .ok_or_else(|| self.make_error(ParseErrorKind::UnexpectedEof, self.lexer_range()))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum ValueKind {
+    String(String),
+    Symbol(String),
+    Tag(Tag),
+    List(Vec<Value>),
+}
+
+pub type Value = WithSource<ValueKind>;
+
+#[derive(Clone)]
+pub struct WithSource<T> {
+    pub inner: T,
+    pub source: Arc<Source>,
+    pub range: SourceRange,
+}
+
+impl<T: Debug> Debug for WithSource<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        self.inner.fmt(f)
+    }
+}
+
+impl<T> AsRef<T> for WithSource<T> {
+    fn as_ref(&self) -> &T {
+        &self.inner
+    }
+}
+
+impl WithSource<String> {
+    pub fn to_val_string(self) -> Value {
+        Value {
+            inner: ValueKind::String(self.inner),
+            source: self.source,
+            range: self.range,
+        }
+    }
+
+    pub fn to_val_symbol(self) -> Value {
+        Value {
+            inner: ValueKind::Symbol(self.inner),
+            source: self.source,
+            range: self.range,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct Args(Vec<Arg>);
+
+impl Args {
+    pub fn new() -> Self {
+        Self(Vec::new())
+    }
+
+    pub fn insert(&mut self, arg: Arg) -> ParseResult<()> {
+        for existing in self.0.iter() {
+            if arg.name.inner == existing.name.inner {
+                return Err(ParseError {
+                    kind: ParseErrorKind::DuplicateArgument(arg.name.inner),
+                    source: arg.name.source,
+                    range: arg.name.range,
+                });
+            }
+        }
+
+        self.0.push(arg);
+
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct Arg {
+    pub name: WithSource<String>,
+    pub val: Value,
+}
+
+#[derive(Clone, Debug)]
+pub struct Tag {
+    pub id: WithSource<String>,
+    pub body: TagBody,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct TagBody {
+    pub args: Args,
+    pub children: Vec<Value>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Source {
+    pub full: String,
+    pub lines: Vec<String>,
+}
+
+impl Source {
+    pub fn from_string(full: String) -> Self {
+        Self {
+            lines: full.lines().map(ToString::to_string).collect(),
+            full,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct SourcePosition {
+    pub row: usize,
+    pub col: usize,
+}
+
+pub type SourceRange = Range<SourcePosition>;
+pub type ParseResult<T> = Result<T, ParseError>;
+
+#[derive(Clone, Debug)]
+pub struct ParseError {
+    pub source: Arc<Source>,
+    pub range: SourceRange,
+    pub kind: ParseErrorKind,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ParseErrorKind {
+    InvalidToken,
+    DuplicateArgument(String),
+    UnmatchedParen,
+    UnexpectedToken(Token, Option<Token>),
+    UnexpectedEof,
+}

src/main.rs

@@ -1,17 +1,28 @@
-use std::str::FromStr;
-use strum::EnumString;
-
+pub mod ast;
 pub mod dom;
 pub mod layout;
 pub mod parse;
 pub mod style;
 pub mod tag;
+pub mod text;
 
+use logos::Logos;
 use style::Stylesheet;
 
 fn main() {
     let src_path = std::env::args().nth(1).unwrap();
     let src = std::fs::read_to_string(src_path).unwrap();
+    
+    let mut lexer = ast::Token::lexer(&src);
+    let tokens: Vec<_> = lexer.collect();
+    println!("{:?}", tokens);
+
+    let source = std::sync::Arc::new(ast::Source::from_string(src.clone()));
+    let mut ast_builder = ast::AstBuilder::new(&source);
+    let body = ast_builder.expect_value();
+    
+    println!("{:#?}", body);
+
     let options = lexpr::parse::Options::new()
         .with_string_syntax(lexpr::parse::StringSyntax::Elisp)
         .with_keyword_syntax(lexpr::parse::KeywordSyntax::ColonPrefix);

src/text.rs

@@ -0,0 +1,105 @@
+use strum::EnumString;
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumString)]
+#[strum(serialize_all = "kebab-case", ascii_case_insensitive)]
+pub enum TextTagKind {
+    Black,
+    Red,
+    Green,
+    Yellow,
+    Blue,
+    Magenta,
+    Cyan,
+    White,
+    Italic,
+    Bold,
+    Underline,
+    Strike,
+    Style,
+}
+
+impl TextTagKind {
+    pub fn to_color(&self) -> Option<Color> {
+        use Color as C;
+        use TextTagKind::*;
+        match self {
+            Black => Some(C::Black),
+            Red => Some(C::Red),
+            Green => Some(C::Green),
+            Yellow => Some(C::Yellow),
+            Blue => Some(C::Blue),
+            Magenta => Some(C::Magenta),
+            Cyan => Some(C::Cyan),
+            White => Some(C::White),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumString)]
+#[strum(serialize_all = "kebab-case", ascii_case_insensitive)]
+pub enum Color {
+    Foreground,
+    Background,
+    Black,
+    Red,
+    Green,
+    Yellow,
+    Blue,
+    Magenta,
+    Cyan,
+    White,
+}
+
+bitflags::bitflags! {
+    pub struct TextStyleFlags: u32 {
+        const ITALIC = 1 << 0;
+        const BOLD = 1 << 1;
+        const UNDERLINE = 1 << 2;
+        const STRIKETHROUGH = 1 << 3;
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct TextStyle {
+    pub fg: Color,
+    pub bg: Color,
+    pub flags: TextStyleFlags,
+}
+
+impl Default for TextStyle {
+    fn default() -> Self {
+        Self {
+            fg: Color::Foreground,
+            bg: Color::Background,
+            flags: TextStyleFlags::empty(),
+        }
+    }
+}
+
+impl TextStyle {
+    pub fn apply_tag(&mut self, kind: TextTagKind) {
+        if let Some(color) = kind.to_color() {
+            self.fg = color;
+            return;
+        }
+
+        use TextStyleFlags as F;
+        use TextTagKind::*;
+        let flags = match kind {
+            Italic => F::ITALIC,
+            Bold => F::BOLD,
+            Strike => F::STRIKETHROUGH,
+            Underline => F::UNDERLINE,
+            _ => unreachable!(),
+        };
+
+        self.flags |= flags;
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct StyledText {
+    pub text: String,
+    pub style: TextStyle,
+}