Skip to content
Snippets Groups Projects
Commit 759e52b7 authored by Stephen D's avatar Stephen D
Browse files

Merge branch 'reaction_fixes' into 'master'

better reaction tokenization

Closes #1

See merge request stephen/cat-disruptor-6500!10
parents 119260e7 0197cc7d
No related branches found
No related tags found
No related merge requests found
...@@ -42,7 +42,7 @@ impl LineHandler for ReactHandler { ...@@ -42,7 +42,7 @@ impl LineHandler for ReactHandler {
.content .content
.chars() .chars()
.map(|c| match c { .map(|c| match c {
'!' | '?' | ',' | '.' => ' ', '!' | '?' | ',' | '.' | '(' | ')' | '[' | ']' | '\n' | '\r' => ' ',
_ => c, _ => c,
}) })
.group_by(|c| *c == ' '); .group_by(|c| *c == ' ');
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment