add tokenator crate

also remove a lot of the "advanced" token parsing style
which was a bit too verbose for my tastes

Signed-off-by: William Casarin <jb55@jb55.com>

Cargo.lock

@@ -2828,6 +2828,7 @@ dependencies = [
  "strum_macros",
  "tempfile",
  "thiserror 2.0.7",
+ "tokenator",
  "tokio",
  "tracing",
  "tracing-appender",
@@ -4527,6 +4528,10 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
+[[package]]
+name = "tokenator"
+version = "0.1.0"
+
 [[package]]
 name = "tokio"
 version = "1.42.0"