Basic lexer

.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "katana"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+console = "0.15.0"
+logos = "0.12.0"

example.ktn

@@ -0,0 +1,92 @@
+// complex data is passed and manipulated in structures
+struct World {
+    // each member is defined by name and type
+    u32 xsize,
+    u32 ysize,
+    BitArray current,
+    u32 numdots,
+    BitArray next,
+
+    // members can have default initialization values
+    i32 xmin = 1'000'000,
+    i32 xmax = -1,
+    i32 ymin = 1'000'000,
+    i32 ymax = -1,
+}
+
+// non-associated function, operates in the scope of its arguments only
+// `fn fn_name(args) return_value { body }`
+fn count_neighbors(BitArray array, i32 i, i32 j) i32 {
+    // local variables are declared with `let`
+    let c1 = i - 1;
+    let c2 = i;
+    let c3 = i + 1;
+
+    let r1 = i - 1;
+    let r2 = i;
+    let c3 = i + 1;
+
+    // associated functions of a struct can be stored in a variable
+    // now `g(x, y)` is shorthand for `array.get(x, y)`
+    let g = array.get;
+
+    // return result with trailing expression at the end of function
+    g(c1, r1) + g(c1, r2) + g(c1, r3) +
+    g(c2, r1) +             g(c2, r3) +
+    g(c3, r1) + g(c3, r2) + g(c3, r3)
+}
+
+// associated function for the World struct
+World fn set_next(i32 i, i32 j) {
+    // members on this World struct can be accessed with `.`
+    var numdots = .numdots; // mutable variables are defined with `var`
+    let neighbors = count_neighbors(.current, i, j);
+
+    // `if` statements are expressions
+    let next = if .current.get(i, j) == 0 {
+        if neighbors != 3 {
+            0
+        } else {
+            numdots++;
+            1
+        }
+    } else {
+        // Python-like `or` operator
+        if neighbors == 2 or neighbors == 3 {
+            1
+        } else {
+            numdots--;
+            0
+        }
+    };
+
+    if next != 0 {
+        // TODO: mutability rules for arguments?
+        if i < .xmin { .xmin = i; }
+        if i > .xmax { .xmax = i; }
+        if j < .ymin { .ymin = j; }
+        if j > .ymax { .ymax = j; }
+    }
+}
+
+World fn next_cycle() {
+    let xlb = .xmin - 1;
+    let xub = .xmax + 1;
+    let ylb = .ymin - 1;
+    let yub = .ymax + 1;
+
+    // TODO: flesh out `for` semantics
+    for y in ylb..yub {
+        for x in xlb..xub {
+            .set_next(x, y);
+        }
+    }
+
+    // TODO: figure out better range definitions, or ditch them altogether
+    for y in 0..(.ysize) {
+        for x in 0..(.xsize) {
+            // TODO: mutability rules
+            .current.set_bit(x, y, .next.get_bit(x, y));
+        }
+    }
+}

src/lib.rs

@@ -0,0 +1,109 @@
+use console::Style;
+use logos::Logos;
+
+#[rustfmt::skip]
+#[derive(Logos, Debug, PartialEq)]
+pub enum Token {
+    // keywords
+    #[token("struct")] Struct,
+    #[token("fn")]     Function,
+    #[token("for")]    For,
+    #[token("if")]     If,
+    #[token("else")]   Else,
+    #[token("in")]     In,
+    #[token("let")]    Let,
+    #[token("var")]    Var,
+
+    // separators
+    #[token("{")] BraceOpen,
+    #[token("}")] BraceClose,
+    #[token("(")] ParanOpen,
+    #[token(")")] ParanClose,
+    #[token(";")] Semicolon,
+    #[token(",")] Comma,
+    #[token(".")] Dot,
+
+    // arithmetic operators
+    #[token("+")] OpAdd,
+    #[token("-")] OpSub,
+    #[token("*")] OpMul,
+    #[token("/")] OpDiv,
+    #[token("=")] OpAssign,
+
+    // comparison operators
+    #[token("<")]  OpLess,
+    #[token("<=")] OpLessEq,
+    #[token(">")]  OpGreater,
+    #[token(">=")] OpGreaterEq,
+    #[token("==")] OpEq,
+    #[token("!=")] OpNeq,
+
+    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*")]
+    Identifier,
+
+    #[regex(r"//[^\n]*")]
+    SingleLineComment,
+
+    #[regex(r"0b_*[01][_01]*")]
+    BinaryInteger,
+
+    #[regex(r"0o_*[0-7][_0-7]*")]
+    OctalInteger,
+
+    #[regex(r"-?[0-9][_'0-9]*")]
+    DecimalInteger,
+
+    #[regex(r"[ \t\n\f ]+")] // , logos::skip)]
+    Whitespace,
+
+    #[error]
+    Error,
+}
+
+pub struct ColorTheme {
+    normal: Style,
+    keyword: Style,
+    literal: Style,
+    comment: Style,
+    error: Style,
+}
+
+impl ColorTheme {
+    pub fn new() -> Self {
+        Self {
+            normal: Style::default(),
+            keyword: Style::new().blue(),
+            literal: Style::new().cyan(),
+            comment: Style::new().white(),
+            error: Style::new().black().on_red().bold(),
+        }
+    }
+
+    pub fn token_style(&self, token: &Token) -> &Style {
+        use Token::*;
+        match token {
+            Struct | Function | For | If | Else | In | Let | Var => &self.keyword,
+            BinaryInteger | OctalInteger | DecimalInteger => &self.literal,
+            SingleLineComment => &self.comment,
+            Error => &self.error,
+            _ => &self.normal,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn color_file() {
+        let source = include_str!("../example.ktn");
+        let theme = ColorTheme::new();
+        let mut lex = Token::lexer(source);
+
+        while let Some(token) = lex.next() {
+            let style = theme.token_style(&token);
+            print!("{}", style.apply_to(lex.slice()));
+        }
+    }
+}