diff --git a/src/sections/TokenViewer.tsx b/src/sections/TokenViewer.tsx index 009d67b..d16d01e 100644 --- a/src/sections/TokenViewer.tsx +++ b/src/sections/TokenViewer.tsx @@ -48,6 +48,8 @@ function encodeWhitespace(str: string) { return result; } +const firstOccurrences: Record = {}; + export function TokenViewer(props: { isFetching: boolean; model: string | undefined; @@ -63,6 +65,16 @@ export function TokenViewer(props: { const [showWhitespace, setShowWhitespace] = useState(false); + // Keep track of the first occurrence of each token + props.data?.forEach(({ tokens }) => { + tokens.forEach(({ id, idx }) => { + if (firstOccurrences[id] == null) { + console.log("firstOccurrences", id, idx); + firstOccurrences[id] = idx; + } + }); + }); + return ( <>
@@ -75,83 +87,83 @@ export function TokenViewer(props: {

Price per prompt

- ${pricing?.multipliedBy(tokenCount)?.toFixed()} -

-
- )} + ${pricing?.multipliedBy(tokenCount)?. toFixed()} +

+ )} + -
-        {props.data?.map(({ text }, idx) => (
-           setIndexHover(idx)}
-            onMouseLeave={() => setIndexHover(null)}
-            className={cn(
-              "transition-all",
-              (indexHover == null || indexHover === idx) &&
-                COLORS[idx % COLORS.length],
-              props.isFetching && "opacity-50"
-            )}
-          >
-            {showWhitespace || indexHover === idx
-              ? encodeWhitespace(text)
-              : text}
-          
-        ))}
-      
+
+    {props.data?.map(({ text, tokens }, idx) => (
+       setIndexHover(idx)}
+        onMouseLeave={() => setIndexHover(null)}
+        className={cn(
+          "transition-all",
+          (indexHover == null || indexHover === idx) &&
+            COLORS[firstOccurrences[tokens[0]!.id]! % COLORS.length],
+          props.isFetching && "opacity-50"
+        )}
+      >
+        {showWhitespace || indexHover === idx
+          ? encodeWhitespace(text)
+          : text}
+      
+    ))}
+  
-
+    {props.data && tokenCount > 0 && (
+      
-        {props.data && tokenCount > 0 && (
-          
-            [
-            {props.data.map((segment, segmentIdx) => (
-              
-                {segment.tokens.map((token) => (
-                  
-                     setIndexHover(segmentIdx)}
-                      onMouseLeave={() => setIndexHover(null)}
-                      className={cn(
-                        "transition-colors",
-                        indexHover === segmentIdx &&
-                          COLORS[segmentIdx % COLORS.length]
-                      )}
-                    >
-                      {token.id}
-                    
-                    {", "}
-                  
-                ))}
+        [
+        {props.data.map(({ tokens }, segmentIdx) => (
+          
+            {tokens.map(({ id, idx }) => (
+              
+                 setIndexHover(segmentIdx)}
+                  onMouseLeave={() => setIndexHover(null)}
+                  className={cn(
+                    "transition-colors",
+                    indexHover === segmentIdx &&
+                      COLORS[firstOccurrences[id]! % COLORS.length]
+                  )}
+                >
+                  {id}
+                
+                {", "}
               
             ))}
-            ]
-          
-        )}
-      
- -
- setShowWhitespace((v) => !v)} - /> - -
- - ); -} + + ))} + ] + + )} + + +
+ setShowWhitespace((v) => !v)} + /> + +
+ +); +} \ No newline at end of file