From 6479953531b08884ef6ec27ae8dc015e82f096ca Mon Sep 17 00:00:00 2001
From: Alex van de Sandt <alex@avandesa.dev>
Date: Wed, 15 May 2024 23:47:31 -0400
Subject: [PATCH] Implement ident and keyword scanning

---
 lox-examples/hello.lox |  4 +-
 src/scanner.rs         | 98 +++++++++++++++++++++++++++++++++++++++---
 2 files changed, 95 insertions(+), 7 deletions(-)

diff --git a/lox-examples/hello.lox b/lox-examples/hello.lox
index a06cad3..b108da0 100644
--- a/lox-examples/hello.lox
+++ b/lox-examples/hello.lox
@@ -1,2 +1,2 @@
-"Hello,
-world"
\ No newline at end of file
+print "Hello, world"
+print 1 + 1 == 2 and true
\ No newline at end of file
diff --git a/src/scanner.rs b/src/scanner.rs
index 026a3bf..da34418 100644
--- a/src/scanner.rs
+++ b/src/scanner.rs
@@ -97,12 +97,14 @@ impl Scanner {
                 // discard whitespace
                 return Ok(None);
             }
-            c if c.is_ascii_digit() => {
+            d if d.is_ascii_digit() => {
                 let is_digit = |c: char| c.is_ascii_digit();
                 // consume all the remaining digits
                 self.pop_while(is_digit);
-                // if it's a decimal, consume the right side
-                if self.pop_if_matches('.').is_some() {
+                if self
+                    .pop_if(|a, b| a == '.' && b.is_some_and(is_digit))
+                    .is_some()
+                {
                     self.pop_while(is_digit);
                 }
 
@@ -112,8 +114,34 @@ impl Scanner {
                     .expect("could not parse number literal lexeme");
                 NumberLit(value)
             }
-            c => {
-                return Err(ScannerError::unexpected_character(c, current_token_start));
+            i if i.is_ascii_alphabetic() || i == '_' => {
+                self.pop_while(|c| c.is_ascii_alphanumeric() || c == '_');
+                let lexeme = self.lexeme(current_token_start, self.current_position);
+                match lexeme.as_str() {
+                    "and" => And,
+                    "class" => Class,
+                    "else" => Else,
+                    "false" => False,
+                    "fun" => Fun,
+                    "for" => For,
+                    "if" => If,
+                    "nil" => Nil,
+                    "or" => Or,
+                    "print" => Print,
+                    "return" => Return,
+                    "super" => Super,
+                    "this" => This,
+                    "true" => True,
+                    "var" => Var,
+                    "while" => While,
+                    _ => Ident(lexeme),
+                }
+            }
+            unexpected => {
+                return Err(ScannerError::unexpected_character(
+                    unexpected,
+                    current_token_start,
+                ));
             }
         };
 
@@ -172,6 +200,22 @@ impl Scanner {
         }
     }
 
+    /// Takes a predicate that receives the next two characters.
+    ///
+    /// Pops the next character and returns it if the predicate returns `true`. If it returns `false` or we are at the end of the input, this function returns `None`.
+    fn pop_if<F>(&mut self, should_pop: F) -> Option<char>
+    where
+        F: FnOnce(char, Option<char>) -> bool,
+    {
+        let next = self.peek()?;
+        let after = self.peek_2();
+        if should_pop(next, after) {
+            self.pop()
+        } else {
+            None
+        }
+    }
+
     /// Advance until the next character matches `until`, or we reach the end of the input. That
     /// character is not consumed.
     ///
@@ -212,6 +256,11 @@ impl Scanner {
         self.source.get(self.current_position).copied()
     }
 
+    /// Look at the character after next but do not advance
+    fn peek_2(&self) -> Option<char> {
+        self.source.get(self.current_position + 1).copied()
+    }
+
     /// Returns `true` if we are at the end of the input
     fn at_end(&self) -> bool {
         self.current_position >= self.source.len()
@@ -303,6 +352,30 @@ mod test {
         }
     }
 
+    #[test]
+    fn scans_keywords() {
+        for (input, expected) in [
+            ("and", And),
+            ("class", Class),
+            ("else", Else),
+            ("false", False),
+            ("fun", Fun),
+            ("for", For),
+            ("if", If),
+            ("nil", Nil),
+            ("or", Or),
+            ("print", Print),
+            ("return", Return),
+            ("super", Super),
+            ("this", This),
+            ("true", True),
+            ("var", Var),
+            ("while", While),
+        ] {
+            assert_input_scans_to(input, &[expected]);
+        }
+    }
+
     #[test]
     fn scans_slash() {
         assert_input_scans_to("/", &[Slash]);
@@ -336,6 +409,21 @@ mod test {
         assert_input_scans_to(&PI.to_string(), &[NumberLit(PI)]);
     }
 
+    #[test]
+    fn no_trailing_decimals() {
+        assert_input_scans_to("1.", &[NumberLit(1.0), Dot]);
+    }
+
+    #[test]
+    fn scans_identifiers() {
+        assert_input_scans_to("foobar", &[Ident("foobar".to_string())]);
+    }
+
+    #[test]
+    fn idents_can_start_with_underscore() {
+        assert_input_scans_to("_a", &[Ident("_a".to_string())]);
+    }
+
     #[test]
     fn errs_on_unexpected_char() {
         let err = scan("+++@").unwrap_err();
-- 
2.51.2