use nom::{ IResult, Parser, branch::alt, multi::many0, combinator::all_consuming, bytes::complete::{tag, take_while1}, character::complete::multispace1, }; #[derive(Debug,PartialEq)] pub enum Token { ParOpen, ParClose, Num(i32), Sym(String), Whitespace(String), } use Token::*; use crate::parse::util::*; fn parse_token(s: &str) -> IResult<&str, Token> { alt(( tag("(").map(|_| ParOpen), tag(")").map(|_| ParClose), multispace1.map(whitespace), nom::character::complete::i32.map(Num), take_while1(|c| !(" \n\t()".contains(c))).map(sym), )).parse(s) } fn tokenize(s: &str) -> Result, String> { match many0(parse_token).parse(s) { Ok(("", res)) => Ok(res), Ok((rest, _)) => Err(format!("all data should be tokenizable, '{rest}' was not")), Err(e) => Err(e.to_string()), } } #[cfg(test)] mod private_parsing_tests { use super::{*, parse_token}; #[test] fn test_parse_token() { assert_eq!(parse_token("()"), Ok((")", ParOpen))); assert_eq!(parse_token(")"), Ok(("", ParClose))); assert_eq!(parse_token(" \t\n"), Ok(("", whitespace(" \t\n")))); assert_eq!(parse_token("1 23"), Ok((" 23", Num(1)))); assert_eq!(parse_token("23"), Ok(("", Num(23)))); assert_eq!(parse_token("Nil a"), Ok((" a", sym("Nil")))); assert_eq!(parse_token("a"), Ok(("", sym("a")))); assert!(parse_token("").is_err()) } #[test] fn test_tokenize() { assert_eq!( tokenize("(+ 1 2 (\t\n a)").unwrap(), vec![ ParOpen, sym("+"), whitespace(" "), Num(1), whitespace(" "), Num(2), whitespace(" "), ParOpen, whitespace("\t\n "), sym("a"), ParClose ] ); } }