tyiu/notedeck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parser: add utf8 seeking functions

Browse Source 
useful for peeking the previous utf8 char on a 0-copy utf8 buffer view
This commit is contained in:
William Casarin
2023-07-01 11:27:47 -07:00
parent cc043a4ec3
commit 3033943e52
4 changed files with 175 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/error.rs
View File

@@ -1,6 +1,9 @@
use crate::parser;
#[derive(Eq, PartialEq, Debug)]
pub enum Error {
Nostr(enostr::Error),
Parse(parser::Error),
Generic(String),
}
@@ -10,6 +13,12 @@ impl From<String> for Error {
}
}
impl From<parser::Error> for Error {
fn from(s: parser::Error) -> Self {
Error::Parse(s)
}
}
impl From<enostr::Error> for Error {
fn from(err: enostr::Error) -> Self {
Error::Nostr(err)

1
src/lib.rs
View File

@@ -3,6 +3,7 @@ mod app;
mod contacts;
mod error;
mod parser;
mod result;
pub use app::Damus;
pub use error::Error;

199
src/parser.rs
View File

@@ -1,28 +1,65 @@
use log::info;
use log::{debug, info};
#[derive(Debug, PartialEq)]
struct Parser<'a> {
#[derive(Debug, PartialEq, Eq)]
pub struct Parser<'a> {
data: &'a [u8],
pos: usize,
}
#[derive(Debug, PartialEq)]
enum ParseError {
NotFound,
BadUtf8Encoding,
EOF,
#[derive(Debug, PartialEq, Eq)]
pub enum Bound {
Start,
End,
}
type Result<T> = std::result::Result<T, ParseError>;
#[derive(Debug, PartialEq, Eq)]
pub enum Error {
NotFound,
BadUtf8Encoding,
OutOfBounds(Bound),
}
type Result<T> = std::result::Result<T, Error>;
pub fn is_oob<T>(r: Result<T>) -> bool {
match r {
Err(Error::OutOfBounds(_)) => true,
Err(_) => false,
Ok(_) => false,
}
}
impl<'a> Parser<'a> {
fn new(data: &'a [u8]) -> Parser {
pub fn from_bytes(data: &'a [u8]) -> Parser<'a> {
Parser { data: data, pos: 0 }
}
fn pull_byte(&mut self) -> Result<u8> {
if self.pos + 1 > self.data.len() {
return Err(ParseError::EOF);
pub fn from_str(string: &'a str) -> Parser<'a> {
Parser {
data: string.as_bytes(),
pos: 0,
}
}
pub fn set_pos(&mut self, pos: usize) {
self.pos = pos;
}
pub fn pos(&self) -> usize {
self.pos
}
pub fn data(&self) -> &[u8] {
self.data
}
pub fn len(&self) -> usize {
self.data.len()
}
pub fn pull_byte(&mut self) -> Result<u8> {
if self.pos + 1 > self.len() {
return Err(Error::OutOfBounds(Bound::End));
}
let c = self.data[self.pos];
@@ -30,6 +67,25 @@ impl<'a> Parser<'a> {
return Ok(c);
}
pub fn skip<F: Fn(char) -> bool>(&mut self, should_skip: F) -> Result<()> {
let len = self.len();
while self.pos < len {
let prev = self.pos();
if should_skip(self.pull_char()?) {
continue;
} else {
self.set_pos(prev);
return Ok(());
}
}
return Err(Error::OutOfBounds(Bound::End));
}
pub fn skip_whitespace(&mut self) -> Result<()> {
self.skip(|c| c.is_ascii_whitespace())
}
pub fn peek_char(&mut self) -> Result<char> {
let peek = true;
self.pull_or_peek_char(peek)
@@ -40,6 +96,65 @@ impl<'a> Parser<'a> {
self.pull_or_peek_char(peek)
}
pub fn seek_prev_byte(&mut self) -> Result<()> {
if self.pos == 0 {
return Err(Error::OutOfBounds(Bound::Start));
}
self.pos -= 1;
Ok(())
}
fn peek_prev_char(&self) -> Result<char> {
let mut i = 1;
let mut codepoint = 0u32;
let mut bs: [u32; 4] = [0; 4];
if self.pos == 0 {
return Err(Error::OutOfBounds(Bound::Start));
}
while i <= 4 && ((self.pos as i32) - (i as i32) >= 0) {
let byte = self.data[self.pos - i] as u32;
let masked = byte & 0b11000000;
if masked == 0b10000000 {
// continuation byte
bs[i - 1] = byte & 0b00111111;
i += 1;
} else if masked == 0b11000000 {
// start byte
match i {
4 => {
codepoint = ((bs[3] & 0x07) << 18)
| ((bs[2] & 0x3F) << 12)
| ((bs[1] & 0x3F) << 6)
| (bs[0] & 0x3F)
}
3 => {
codepoint = ((bs[2] & 0x0F) << 12) | ((bs[1] & 0x3F) << 6) | (bs[0] & 0x3F)
}
2 => codepoint = ((bs[1] & 0x0F) << 6) | (bs[0] & 0x3F),
_ => return Err(Error::BadUtf8Encoding),
}
return parser_codepoint_char(codepoint);
} else {
return parser_codepoint_char(byte);
}
}
// If we reached here, we reached the start of the string without finding a non-continuation byte.
Err(Error::BadUtf8Encoding)
}
pub fn seek_prev_char(&mut self) -> Result<()> {
self.seek_prev_byte()?;
while self.pos > 0 && (self.data[self.pos] & 0b11000000) == 0b10000000 {
self.pos -= 1;
}
Ok(())
}
fn pull_or_peek_char(&mut self, peek: bool) -> Result<char> {
let mut codepoint: u32 = 0;
@@ -47,10 +162,10 @@ impl<'a> Parser<'a> {
let b0 = self.pull_byte()? as u32;
if b0 & 0x80 != 0 {
if (b0 & 0xE0) == 0xC0 {
if (b0 & 0b11100000) == 0b11000000 {
// Two-byte sequence
let b1 = self.pull_byte()? as u32;
codepoint = ((b0 & 0x1F) << 6) | (b1 & 0x3F);
codepoint = ((b0 & 0b00011111) << 6) | (b1 & 0b00111111);
} else if (b0 & 0xF0) == 0xE0 {
// Three-byte sequence
let b1 = self.pull_byte()? as u32;
@@ -75,35 +190,32 @@ impl<'a> Parser<'a> {
match std::char::from_u32(codepoint) {
Some(c) => Ok(c),
None => Err(ParseError::BadUtf8Encoding),
None => Err(Error::BadUtf8Encoding),
}
}
fn current(&mut self) -> Result<char> {
let last_pos = self.pos;
let c = self.pull_char();
if c.is_ok() {
self.pos = last_pos;
}
return c;
pub fn parse_until_char(&mut self, needle: char) -> Result<()> {
self.parse_until(|c| c == needle)
}
fn parse_until_char(&mut self, needle: char) -> Result<()> {
self.parse_until(|c| c == needle)?;
Ok(())
}
fn parse_until<F: Fn(char) -> bool>(&mut self, matches: F) -> Result<()> {
let len = self.data.len();
pub fn parse_until<F: Fn(char) -> bool>(&mut self, matches: F) -> Result<()> {
let len = self.len();
while self.pos < len {
let prev_pos = self.pos;
let prev = self.pos;
if matches(self.pull_char()?) {
self.pos = prev_pos;
self.pos = prev;
return Ok(());
}
}
Err(ParseError::NotFound)
Err(Error::OutOfBounds(Bound::End))
}
}
fn parser_codepoint_char(codepoint: u32) -> Result<char> {
match std::char::from_u32(codepoint) {
Some(c) => Ok(c),
None => Err(Error::BadUtf8Encoding),
}
}
@@ -116,7 +228,7 @@ mod test {
// v alien v
// 00000000: 20f0 9f91 bd23 6861 7368 7461 670a _....#hashtag.
let s = " #hashtag ";
let mut parser = Parser::new(s.as_bytes());
let mut parser = Parser::from_str(s);
let mut res = parser.parse_until_char('#');
assert_eq!(res, Ok(()));
assert_eq!(parser.pos, 1);
@@ -126,16 +238,29 @@ mod test {
Ok(())
}
#[test]
fn test_peek_prev_char() {
let s = ".👽.";
let mut parser = Parser::from_str(s);
let r1 = parser.parse_until_char('👽');
assert_eq!(r1, Ok(()));
let r2 = parser.pull_char();
assert_eq!(r2, Ok('👽'));
let r3 = parser.peek_prev_char();
assert_eq!(r3, Ok('👽'));
assert_eq!(parser.pos(), 5);
}
#[test]
fn test_utf8_parsing() -> Result<()> {
let s = "hey there #👽.";
let mut parser = Parser::new(s.as_bytes());
let mut parser = Parser::from_str(s);
let _ = parser.parse_until_char('👽');
assert_eq!(parser.current(), Ok('👽'));
assert_eq!(parser.peek_char(), Ok('👽'));
assert_eq!(parser.pos, 11);
let res = parser.parse_until(|c| c.is_ascii_whitespace() || c.is_ascii_punctuation());
assert_eq!(res, Ok(()));
assert_eq!(parser.current(), Ok('.'));
assert_eq!(parser.peek_char(), Ok('.'));
Ok(())
}
}

3
src/result.rs Normal file
View File

@@ -0,0 +1,3 @@
use crate::error::Error;
type Result<T> = std::result::Result<T, Error>;