From 0551f8f97c4f27895b5e6ba3e1685cb6bb886365 Mon Sep 17 00:00:00 2001
From: Kainoa Kanter <44733677+ThatOneCalculator@users.noreply.github.com>
Date: Sun, 2 Jul 2023 11:43:19 -0700
Subject: [PATCH] Add support for Japanese punctuation

---
 lib/tokenizer.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/tokenizer.js b/lib/tokenizer.js
index e6e5917..40c2ae3 100644
--- a/lib/tokenizer.js
+++ b/lib/tokenizer.js
@@ -39,7 +39,7 @@ Tokenizer.prototype = {
     if (!this.entry) return [];
     var words = this.entry.split(' ');
     var endingWords = words.filter(function(w) {
-      return w.endsWith('.') || w.endsWith('!') || w.endsWith('?');
+      return w.endsWith('.') || w.endsWith('!') || w.endsWith('?') || w.endsWith('。') || w.endsWith('！') || w.endsWith('？');
     });
 
     var self = this;
@@ -75,4 +75,4 @@ Tokenizer.prototype = {
   }
 };
 
-module.exports = Tokenizer;
\ No newline at end of file
+module.exports = Tokenizer;