Change from str to bytes.

This should make everything easier.

Author: pmfirestone

src/dfa.rs

@@ -8,7 +8,7 @@ use std::collections::{HashMap, VecDeque};
 use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 
-type DFACache = HashMap<String, Arc<DFA>>;
+type DFACache = HashMap<Box<[u8]>, Arc<DFA>>;
 type DFA = dense::DFA<Vec<u32>>;
 
 /// A DFA along with its state. Generic to facilitate experiementation with
@@ -17,7 +17,7 @@ type DFA = dense::DFA<Vec<u32>>;
 #[derive(Clone, Debug)]
 pub struct DFAState {
     /// The regex representing this dfa.
-    pub regex: Box<str>,
+    pub regex: Box<[u8]>,
     /// The actual DFA implementation from the library.
     pub dfa: Arc<DFA>,
     /// The state of this DFA. Defaults to the starting state of the DFA.
@@ -43,12 +43,13 @@ impl DFABuilder {
 
     /// Return a DFAState, either from the cache or building a new one from scratch.
     // FIXME: Remove the clones from this function to accelerate it further.
-    pub fn build_dfa(&mut self, regex: String) -> DFAState {
-        match self.cache.get(&regex) {
+    pub fn build_dfa(&mut self, regex: &[u8]) -> DFAState {
+        match self.cache.get(regex) {
             Some(dfa) => DFAState::new(regex, dfa.clone()),
             None => {
-                let new_dfa = Arc::new(DFA::new(&regex).unwrap());
-                self.cache.insert(regex.clone(), new_dfa.clone());
+                let regex_str: &str = &std::str::from_utf8(regex).unwrap();
+                let new_dfa = Arc::new(DFA::new(regex_str).unwrap());
+                self.cache.insert(regex.into(), new_dfa.clone());
                 DFAState::new(regex, new_dfa)
             }
         }
@@ -59,30 +60,30 @@ impl DFABuilder {
 impl DFAState {
     /// For the Python interface, make advance return the whole DFAState rather than the StateID.
     /// TODO: This is probably a better way to do it than returning the StateID.
-    #[pyo3(name="advance")]
-    pub fn py_advance(&self, input: String) -> DFAState {
+    #[pyo3(name = "advance")]
+    pub fn py_advance(&self, input: &[u8]) -> DFAState {
         let mut dfa = self.clone();
-        for c in input.chars() {
-            dfa.consume_character(c);
+        for b in input {
+            dfa.consume(b);
         }
         dfa.clone()
     }
 
     #[getter(state_id)]
     fn state_id(&self) -> u32 {
-	self.state_id.as_u32()
+        self.state_id.as_u32()
     }
 
     #[getter(regex)]
-    fn regex(&self) -> String {
-	self.regex.to_string()
+    fn regex(&self) -> &[u8] {
+        &self.regex
     }
 }
 
 /// A dense implementation of the DFAState abstraction.
 impl DFAState {
     /// Encapsulate the kluge necessary to set up the DFA correctly for Syncode's use case.
-    fn new(regex: String, dfa: Arc<DFA>) -> DFAState {
+    fn new(regex: &[u8], dfa: Arc<DFA>) -> DFAState {
         // We always want the DFA to match starting from the beginning of the string.
         let config = start::Config::new().anchored(Anchored::Yes);
         let state_id = dfa.start_state(&config).unwrap();
@@ -94,36 +95,17 @@ impl DFAState {
     }
 
     /// Convenience function to set the state how we want it.
-    pub fn advance(&mut self, input: String) -> StateID {
-        for c in input.chars() {
-            self.consume_character(c);
+    pub fn advance(&mut self, input: &[u8]) -> StateID {
+        for b in input {
+            self.consume(b);
         }
         self.state_id
     }
 
-    /// Consume a character, starting at the current state, setting and
+    /// Consume a byte, starting at the current state, setting and
     /// returning the new state.
-    ///
-    /// The logic here is non-trivial, because UTF-8 characters are a variable
-    /// number of bytes long, and the underlying DFA has bytes as its input
-    /// alphabet.
-    pub fn consume_character(&mut self, c: char) -> StateID {
-        let char_len = c.len_utf8();
-        // Buffer to store character as bytes. UFT-8 characters are at most 4
-        // bytes long, so allocate a buffer big enough to store the whole
-        // character regardless of how long it turns out to be.
-        let mut buf = [0; 4];
-        c.encode_utf8(&mut buf);
-        for (i, &b) in buf.iter().enumerate() {
-            // The number of bytes per character is variable: we only need to
-            // feed the number of bytes that the character actually is into the
-            // DFA; any more would be incorrect. Break the loop once we've gone
-            // past the end of the character.
-            if i >= char_len {
-                break;
-            }
-            self.state_id = self.dfa.next_state(self.state_id, b);
-        }
+    pub fn consume(&mut self, b: &u8) -> StateID {
+        self.state_id = self.dfa.next_state(self.state_id, *b);
         self.state_id
     }
 
@@ -179,14 +161,14 @@ impl Hash for DFAState {
     }
 }
 /// Compute the union of all states of a list of regexes.
-pub fn all_dfa_states(terminals: &Vec<String>) -> Vec<DFAState> {
+pub fn all_dfa_states(terminals: &Vec<&[u8]>) -> Vec<DFAState> {
     let mut res = Vec::new();
     let mut builder = DFABuilder::new();
     for terminal in terminals.iter() {
-        let dfa = builder.build_dfa(terminal.clone());
+        let dfa = builder.build_dfa(terminal);
         for state in dfa.states() {
             res.push(DFAState {
-                regex: terminal.to_string().into(),
+                regex: (*terminal).into(),
                 dfa: dfa.dfa.clone(),
                 state_id: state,
             });
@@ -201,40 +183,40 @@ mod tests {
 
     #[test]
     fn test_consume_character_match() {
-        let mut dfa_state = DFABuilder::new().build_dfa("a".to_string());
-        let mut state = dfa_state.consume_character('a');
+        let mut dfa_state = DFABuilder::new().build_dfa(b"a");
+        let mut state = dfa_state.consume(&b"a"[0]);
         state = dfa_state.dfa.next_eoi_state(state);
         assert!(dfa_state.dfa.is_match_state(state));
     }
 
     #[test]
     fn test_consume_character_fails_to_match() {
-        let mut dfa_state = DFABuilder::new().build_dfa("a".to_string());
-        let mut state = dfa_state.consume_character('b');
+        let mut dfa_state = DFABuilder::new().build_dfa(b"a");
+        let mut state = dfa_state.consume(&b"b"[0]);
         state = dfa_state.dfa.next_eoi_state(state);
         assert!(!dfa_state.dfa.is_match_state(state));
     }
 
     #[test]
     fn test_advance_match() {
-        let mut dfa_state = DFABuilder::new().build_dfa("[ab¥]*".to_string());
-        let mut state = dfa_state.advance("aabb¥aab".to_string());
+        let mut dfa_state = DFABuilder::new().build_dfa("[ab¥]*".as_bytes());
+        let mut state = dfa_state.advance("aabb¥aab".as_bytes());
         state = dfa_state.dfa.next_eoi_state(state);
         assert!(dfa_state.dfa.is_match_state(state));
     }
 
     #[test]
     fn test_advance_fails_to_match() {
-        let mut dfa_state = DFABuilder::new().build_dfa("[ab]*".to_string());
-        let mut state = dfa_state.advance("aabba¥ab".to_string());
+        let mut dfa_state = DFABuilder::new().build_dfa("[ab]*".as_bytes());
+        let mut state = dfa_state.advance("aabba¥ab".as_bytes());
         state = dfa_state.dfa.next_eoi_state(state);
         assert!(!dfa_state.dfa.is_match_state(state));
     }
 
     #[test]
     fn test_advance() {
-        let mut dfa_state = DFABuilder::new().build_dfa(r"[a-zA-Z_]*".to_string());
-        let state = dfa_state.advance("indeed".to_string());
+        let mut dfa_state = DFABuilder::new().build_dfa(r"[a-zA-Z_]*".as_bytes());
+        let state = dfa_state.advance("indeed".as_bytes());
         assert!(dfa_state.dfa.is_match_state(state));
     }
 }