Progress on grammar routines.

Author: pmfirestone

Cargo.toml

@@ -16,7 +16,7 @@ regex-automata = "0.4.9"
 # candle-transformers = "0.9.1"
 # tokenizers = { version = "0.21.0", features = ["http"] }
 # hf-hub = "0.4.2"
-# rayon = "1.11.0"
+rayon = "1.11.0"
 # serde = "1.0.225"
 # serde_json = "1.0.145"
 

src/ebnf.rs

@@ -787,36 +787,54 @@ mod tests {
     use std::fs;
     extern crate test;
 
-    // #[test]
-    // fn parse_simple_grammar() {
-    //     let parser = EBNFParser::new("s: c c\n c: \"C\" c | \"D\"", "s");
-    //     let Ok(grammar) = parser.parse() else {
-    //         panic!()
-    //     };
-    //     let expected_grammar = Grammar {
-    //         symbol_set: vec![
-    //             "__ANONYMOUS_LITERAL_C_1".to_string(),
-    //             "__ANONYMOUS_LITERAL_D_2".to_string(),
-    //             "c".to_string(),
-    //             "s".to_string(),
-    //             "$".to_string(),
-    //         ],
-    //         terminals: vec![
-    //             Terminal::new("__ANONYMOUS_LITERAL_C_1", "C", 0),
-    //             Terminal::new("__ANONYMOUS_LITERAL_D_2", "D", 0),
-    //             Terminal::new("$", "", 0),
-    //         ],
-    //         start_symbol: "s".to_string(),
-    //         productions: vec![
-    //             Production::new("s", vec!["c", "c"]),
-    //             Production::new("c", vec!["__ANONYMOUS_LITERAL_C_1", "c"]),
-    //             Production::new("c", vec!["__ANONYMOUS_LITERAL_D_2"]),
-    //         ],
-    //         ignore_terminals: vec![],
-    //     };
-
-    //     assert_eq!(grammar, expected_grammar);
-    // }
+    #[test]
+    fn parse_simple_grammar() {
+        let parser = EBNFParser::new(
+            r#"
+s: c c
+c: "C" c | "D"
+"#,
+            "s",
+        );
+        let Ok(grammar) = parser.parse() else {
+            panic!()
+        };
+        let expected_grammar = Grammar {
+            start_symbol: Symbol::NonTerminal("s".to_string()),
+            productions: vec![
+                Production {
+                    lhs: Symbol::NonTerminal("s".to_string()).into(),
+                    rhs: vec![
+                        Symbol::NonTerminal("c".to_string()),
+                        Symbol::NonTerminal("c".to_string()),
+                    ]
+                    .into(),
+                },
+                Production {
+                    lhs: Symbol::NonTerminal("c".to_string()).into(),
+                    rhs: vec![
+                        Symbol::NonTerminal("__ANONYMOUS_LITERAL_C_1".to_string()),
+                        Symbol::NonTerminal("c".to_string()),
+                    ]
+                    .into(),
+                },
+                Production {
+                    lhs: Symbol::NonTerminal("c".to_string()).into(),
+                    rhs: vec![Symbol::NonTerminal("__ANONYMOUS_LITERAL_D_2".to_string())].into(),
+                },
+                Production {
+                    lhs: Symbol::NonTerminal("__ANONYMOUS_LITERAL_C_1".to_string()).into(),
+                    rhs: vec![Symbol::Terminal(67)].into(),
+                },
+                Production {
+                    lhs: Symbol::NonTerminal("__ANONYMOUS_LITERAL_D_2".to_string()).into(),
+                    rhs: vec![Symbol::Terminal(68)].into(),
+                },
+            ],
+        };
+
+        assert_eq!(grammar, expected_grammar);
+    }
 
     #[test]
     fn parse_tokipona_grammar() {