From 61becd03b57dd1c841719e28ae90cc4c19a5ba20 Mon Sep 17 00:00:00 2001
From: IsaacBreen <mail@isaacbreen.com>
Date: Mon, 24 Apr 2023 10:44:33 +0800
Subject: [PATCH 1/4] Color tokens based on the index (in the token stream) of
 the first occurrence a token with the same ID

---
 src/sections/TokenViewer.tsx | 160 +++++++++++++++++++----------------
 1 file changed, 86 insertions(+), 74 deletions(-)

diff --git a/src/sections/TokenViewer.tsx b/src/sections/TokenViewer.tsx
index 009d67b..3759dd4 100644
--- a/src/sections/TokenViewer.tsx
+++ b/src/sections/TokenViewer.tsx
@@ -63,6 +63,18 @@ export function TokenViewer(props: {
 
   const [showWhitespace, setShowWhitespace] = useState(false);
 
+  // Keep track of the first occurrence of each token
+  const firstOccurrences: Record<number, number> = {};
+  props.data?.forEach(({ tokens }) => {
+    tokens.forEach(({ id, idx }) => {
+      if (firstOccurrences[id] == null) {
+        firstOccurrences[id] = idx;
+      }
+    });
+  });
+
+  console.log(props.data)
+
   return (
     <>
       <div className="flex gap-4">
@@ -75,83 +87,83 @@ export function TokenViewer(props: {
           <div className="flex-grow rounded-md border bg-slate-50 p-4 shadow-sm">
             <p className="text-sm ">Price per prompt</p>
             <p className="text-lg">
-              ${pricing?.multipliedBy(tokenCount)?.toFixed()}
-            </p>
-          </div>
-        )}
+              ${pricing?.multipliedBy(tokenCount)?.        toFixed()}
+        </p>
       </div>
+    )}
+  </div>
 
-      <pre className="min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-all rounded-md border bg-slate-50 p-4 shadow-sm">
-        {props.data?.map(({ text }, idx) => (
-          <span
-            key={idx}
-            onMouseEnter={() => setIndexHover(idx)}
-            onMouseLeave={() => setIndexHover(null)}
-            className={cn(
-              "transition-all",
-              (indexHover == null || indexHover === idx) &&
-                COLORS[idx % COLORS.length],
-              props.isFetching && "opacity-50"
-            )}
-          >
-            {showWhitespace || indexHover === idx
-              ? encodeWhitespace(text)
-              : text}
-          </span>
-        ))}
-      </pre>
+  <pre className="min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-all rounded-md border bg-slate-50 p-4 shadow-sm">
+    {props.data?.map(({ text, tokens }, idx) => (
+      <span
+        key={idx}
+        onMouseEnter={() => setIndexHover(idx)}
+        onMouseLeave={() => setIndexHover(null)}
+        className={cn(
+          "transition-all",
+          (indexHover == null || indexHover === idx) &&
+            COLORS[firstOccurrences[tokens[0]!.id]! % COLORS.length],
+          props.isFetching && "opacity-50"
+        )}
+      >
+        {showWhitespace || indexHover === idx
+          ? encodeWhitespace(text)
+          : text}
+      </span>
+    ))}
+  </pre>
 
-      <pre
-        className={
-          "min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-all rounded-md border bg-slate-50 p-4 shadow-sm"
-        }
+  <pre
+    className={
+      "min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-all rounded-md border bg-slate-50 p-4 shadow-sm"
+    }
+  >
+    {props.data && tokenCount > 0 && (
+      <span
+        className={cn(
+          "transition-opacity",
+          props.isFetching && "opacity-50"
+        )}
       >
-        {props.data && tokenCount > 0 && (
-          <span
-            className={cn(
-              "transition-opacity",
-              props.isFetching && "opacity-50"
-            )}
-          >
-            [
-            {props.data.map((segment, segmentIdx) => (
-              <Fragment key={segmentIdx}>
-                {segment.tokens.map((token) => (
-                  <Fragment key={token.idx}>
-                    <span
-                      onMouseEnter={() => setIndexHover(segmentIdx)}
-                      onMouseLeave={() => setIndexHover(null)}
-                      className={cn(
-                        "transition-colors",
-                        indexHover === segmentIdx &&
-                          COLORS[segmentIdx % COLORS.length]
-                      )}
-                    >
-                      {token.id}
-                    </span>
-                    <span className="last-of-type:hidden">{", "}</span>
-                  </Fragment>
-                ))}
+        [
+        {props.data.map(({ tokens }, segmentIdx) => (
+          <Fragment key={segmentIdx}>
+            {tokens.map(({ id, idx }) => (
+              <Fragment key={idx}>
+                <span
+                  onMouseEnter={() => setIndexHover(segmentIdx)}
+                  onMouseLeave={() => setIndexHover(null)}
+                  className={cn(
+                    "transition-colors",
+                    indexHover === segmentIdx &&
+                      COLORS[firstOccurrences[id]! % COLORS.length]
+                  )}
+                >
+                  {id}
+                </span>
+                <span className="last-of-type:hidden">{", "}</span>
               </Fragment>
             ))}
-            ]
-          </span>
-        )}
-      </pre>
-
-      <div className="flex items-center space-x-2">
-        <Checkbox
-          id="terms"
-          checked={showWhitespace}
-          onClick={() => setShowWhitespace((v) => !v)}
-        />
-        <label
-          htmlFor="terms"
-          className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
-        >
-          Show whitespace
-        </label>
-      </div>
-    </>
-  );
-}
+          </Fragment>
+        ))}
+        ]
+      </span>
+    )}
+  </pre>
+
+  <div className="flex items-center space-x-2">
+    <Checkbox
+      id="terms"
+      checked={showWhitespace}
+      onClick={() => setShowWhitespace((v) => !v)}
+    />
+    <label
+      htmlFor="terms"
+      className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+    >
+      Show whitespace
+    </label>
+  </div>
+</>
+);
+}
\ No newline at end of file

From 83a792f0d1ef21c336ed5cb901442a8a8ec1e227 Mon Sep 17 00:00:00 2001
From: IsaacBreen <mail@isaacbreen.com>
Date: Mon, 24 Apr 2023 10:56:45 +0800
Subject: [PATCH 2/4] Remove debug log

---
 src/sections/TokenViewer.tsx | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/sections/TokenViewer.tsx b/src/sections/TokenViewer.tsx
index 3759dd4..7bbc6a4 100644
--- a/src/sections/TokenViewer.tsx
+++ b/src/sections/TokenViewer.tsx
@@ -73,8 +73,6 @@ export function TokenViewer(props: {
     });
   });
 
-  console.log(props.data)
-
   return (
     <>
       <div className="flex gap-4">

From dfcbe566a57b7461b6db7ee3dcfe18249d333f6d Mon Sep 17 00:00:00 2001
From: IsaacBreen <mail@isaacbreen.com>
Date: Mon, 24 Apr 2023 10:57:06 +0800
Subject: [PATCH 3/4] Don't break words (or numbers)

---
 src/sections/TokenViewer.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sections/TokenViewer.tsx b/src/sections/TokenViewer.tsx
index 7bbc6a4..d995d86 100644
--- a/src/sections/TokenViewer.tsx
+++ b/src/sections/TokenViewer.tsx
@@ -91,7 +91,7 @@ export function TokenViewer(props: {
     )}
   </div>
 
-  <pre className="min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-all rounded-md border bg-slate-50 p-4 shadow-sm">
+  <pre className="min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-word rounded-md border bg-slate-50 p-4 shadow-sm">
     {props.data?.map(({ text, tokens }, idx) => (
       <span
         key={idx}
@@ -113,7 +113,7 @@ export function TokenViewer(props: {
 
   <pre
     className={
-      "min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-all rounded-md border bg-slate-50 p-4 shadow-sm"
+      "min-h-[256px] max-w-[100vw] overflow-auto whitespace-pre-wrap break-word rounded-md border bg-slate-50 p-4 shadow-sm"
     }
   >
     {props.data && tokenCount > 0 && (

From 0bf226912741ba297309fe89654a17544aaa6478 Mon Sep 17 00:00:00 2001
From: IsaacBreen <mail@isaacbreen.com>
Date: Mon, 24 Apr 2023 11:06:10 +0800
Subject: [PATCH 4/4] Retain color assignments between text changes

---
 src/sections/TokenViewer.tsx | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/sections/TokenViewer.tsx b/src/sections/TokenViewer.tsx
index d995d86..d16d01e 100644
--- a/src/sections/TokenViewer.tsx
+++ b/src/sections/TokenViewer.tsx
@@ -48,6 +48,8 @@ function encodeWhitespace(str: string) {
   return result;
 }
 
+const firstOccurrences: Record<number, number> = {};
+
 export function TokenViewer(props: {
   isFetching: boolean;
   model: string | undefined;
@@ -64,10 +66,10 @@ export function TokenViewer(props: {
   const [showWhitespace, setShowWhitespace] = useState(false);
 
   // Keep track of the first occurrence of each token
-  const firstOccurrences: Record<number, number> = {};
   props.data?.forEach(({ tokens }) => {
     tokens.forEach(({ id, idx }) => {
       if (firstOccurrences[id] == null) {
+        console.log("firstOccurrences", id, idx);
         firstOccurrences[id] = idx;
       }
     });