Skip to content

Improve search results #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iwillig merged 4 commits into from
Dec 3, 2025
Merged

Improve search results #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c3a2173
Add better search results and output
iwillig Dec 2, 2025
ce179df
Clean plan, task and task-list
iwillig Dec 3, 2025
3b36a5b
Clean up build script
iwillig Dec 3, 2025
9c028db
Clean up readme
iwillig Dec 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
271 changes: 0 additions & 271 deletions bb.edn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,119 +106,6 @@
"Remove ANSI color codes from string"
(str/replace s (re-pattern "\033\\[[0-9;]*m") "")))

;; Build a specific prompt file by name
build
{:doc "Build a specific prompt file by name (without .md extension)\nUsage: bb build clojure_skill_builder"
:task (let [name (require-arg "name")
output-file (str "_build/" name ".md")]

(info-msg (str "Building " output-file "..."))
(fs/create-dirs "_build")

;; Use CLI to generate prompt content
(let [result (p/shell {:out :string
:err :string
:continue true}
(str "bb main prompt show " name))]
(if (zero? (:exit result))
(let [content (:out result)
cleaned (strip-ansi content)]
(spit output-file cleaned)
(success-msg (str "Build successful: " output-file)))
(do
(error-msg (str "Failed to build prompt: " name))
(println (:err result))
(System/exit 1)))))}

;; Build all prompt files
build-all
{:doc "Build all prompt files"
:task (do
(info-msg "Building all prompts...")

;; Build clojure_skill_builder
(shell "bb build clojure_skill_builder")

;; Build clojure_build
(shell "bb build clojure_build")
(shell "bb build code_archivist")
(shell "bb build plan_executor")

(shell "bb build shortcut_cli_builder")
(shell "bb build software_alchemist_build")

(success-msg "All builds complete!"))}

;; List built prompts with token counts
list-prompts
{:doc "List all built prompts with file sizes and token counts"
:task (let [build-dir "_build"]

(if-not (fs/exists? build-dir)
(do
(warning-msg "No _build directory found. Run 'bb build-all' first.")
(System/exit 1))

(let [prompt-files (sort (filter (fn [f]
(str/ends-with? (str f) ".md"))
(fs/list-dir build-dir)))

;; Collect data for each prompt with metadata
prompts (for [file prompt-files]
(let [file-path (str file)
filename (fs/file-name file)
base-name (-> filename
(str/replace ".compressed.md" "")
(str/replace ".md" ""))
source-prompt (str "prompts/" base-name ".md")
metadata (if (fs/exists? source-prompt)
(extract-metadata-with-pandoc source-prompt)
(extract-metadata-with-pandoc file-path))
content (slurp file-path)
file-size (fs/size file)
char-count (count content)
token-count (estimate-tokens content)
prompt-name (or (get metadata :name)
(get metadata :title)
base-name)
description (strip-html-tags (get metadata :description ""))]
{:filename filename
:name prompt-name
:description description
:file file-path
:size file-size
:size-str (format-size file-size)
:chars char-count
:tokens (long token-count)}))

;; Calculate totals
total-size (reduce + (map :size prompts))
total-chars (reduce + (map :chars prompts))
total-tokens (reduce + (map :tokens prompts))]

(info-msg "Built Prompts")

(println)
(table/print-table [:filename :name :description :size :chars :tokens]
(concat
(for [p prompts]
{:filename (:filename p)
:name (:name p)
:description (truncate-string (:description p) 60)
:size (:size-str p)
:chars (format-number (:chars p))
:tokens (format-number (:tokens p))})
[{:filename "TOTAL"
:name ""
:description ""
:size (format-size total-size)
:chars (format-number total-chars)
:tokens (format-number total-tokens)}]))

(println)
(println (bling/bling [:dim "Note: Token count is estimated (~4 chars per token)"]))
(println (bling/bling [:dim " Anthropic's Claude models use this approximation."])))))}

;; Clean all build artifacts and temporary files
clean
{:doc "Clean all build artifacts and temporary files"
Expand Down Expand Up @@ -406,164 +293,6 @@
(run 'test)
(success-msg "CI pipeline completed successfully"))}

list-skills
{:doc "List all skills with metadata in a table format"
:task (let [skills-dir "skills"
skill-files (sort (map str (fs/glob skills-dir "**/*.md")))

;; Parse all skills with pandoc metadata
skills (for [file-path skill-files]
(let [metadata (extract-metadata-with-pandoc file-path)
file-size (fs/size file-path)
content (slurp file-path)
char-count (count content)
token-count (long (estimate-tokens content))
relative-path (str/replace file-path
(str (fs/file skills-dir) "/")
"")
category (first (str/split relative-path (re-pattern "/")))
skill-name (or (get metadata :name)
(get metadata :title)
(str/replace (fs/file-name file-path) ".md" ""))
description (strip-html-tags (get metadata :description ""))]
{:category category
:name skill-name
:path relative-path
:size (format-size file-size)
:tokens (format-number token-count)
:description description}))

;; Calculate totals
total-size (reduce + (map (fn [s] (fs/size (str "skills/" (:path s)))) skills))
total-tokens (reduce + (map (fn [s] (long (estimate-tokens (slurp (str "skills/" (:path s)))))) skills))]

(info-msg "Available Skills")

(println)
(table/print-table [:category :name :description :size :tokens]
(concat
(for [s (sort-by (juxt :category :name) skills)]
(assoc s :description (truncate-string (:description s) 80)))
[{:category "TOTAL"
:name (str (count skills) " skills")
:description ""
:size (format-size total-size)
:tokens (format-number total-tokens)}]))

(println)
(println (bling/bling [:dim "Note: Token count is estimated (~4 chars per token)"])))}

watch
{:doc "Watch for changes and rebuild prompts automatically\nUsage: bb watch [prompt-name]"
:task (let [name (first *command-line-args*)]
(info-msg (if name
(str "Watching for changes to rebuild " name "...")
"Watching for changes to rebuild all prompts..."))

;; Use babashka.fs/watch
(let [watch-paths ["prompts" "skills" "prompt_configs"]]
(apply fs/watch
(for [path watch-paths
:when (fs/exists? path)]
path)
(fn [{:keys [type path]}]
(when (and (= :modify type)
(str/ends-with? (str path) ".md"))
(info-msg (str "Change detected: " path))
(try
(if name
(shell (str "bb build " name))
(shell "bb build-all"))
(catch Exception e
(error-msg (str "Build failed: " (.getMessage e))))))))))}

;; LLMLingua compression tasks

setup-python
{:doc "Install Python dependencies using pipenv"
:task (do
(info-msg "Installing Python dependencies with pipenv...")

(shell "pipenv install")

(success-msg "Python environment ready!")

(println)
(println "You can now use:")
(println " bb compress <name> --ratio 2")
(println " bb build-compressed <name> --ratio 3")
(println " bb compress-skill skills/path/to/file.md --ratio 2"))}

compress
{:doc "Compress a built prompt file using LLMLingua (default: 2x ratio)\nUsage: bb compress clojure_skill_builder [--ratio 2]"
:task (let [args *command-line-args*
name (first args)
ratio (get-ratio-arg)

input (str "_build/" name ".md")
output (str "_build/" name ".compressed.md")]

(when-not name
(throw (ex-info "Usage: bb compress <name> [--ratio N]" {})))

(when-not (fs/exists? input)
(error-msg (str "Built file not found: " input))
(println "\nRun 'bb build " name "' first.")
(System/exit 1))

(info-msg (str "Compressing " input " with " ratio "x ratio..."))

;; Run Python compression script via pipenv
(let [result (shell {:out :string
:err :string
:continue true}
(str "pipenv run python scripts/compress_prompt.py"
" --input " input
" --output " output
" --ratio " ratio))]
(if (zero? (:exit result))
(do
(println (:out result))
(success-msg (str "Compression successful: " output)))
(do
(println (:err result))
(error-msg "Compression failed")
(System/exit 1)))))}

compress-skill
{:doc "Compress a single skill file (default: 2x ratio)\nUsage: bb compress-skill skills/libraries/data_validation/malli.md [--ratio 2]"
:task (let [args *command-line-args*
input-file (first args)
ratio (get-ratio-arg)

;; Create output path by adding .compressed before .md
output-file (str/replace input-file (re-pattern "\\.md$") ".compressed.md")]

(when-not input-file
(throw (ex-info "Usage: bb compress-skill <path/to/skill.md> [--ratio N]" {})))

(when-not (fs/exists? input-file)
(error-msg (str "Skill file not found: " input-file))
(System/exit 1))

(info-msg (str "Compressing " input-file " with " ratio "x ratio..."))

(let [result (shell {:out :string
:err :string
:continue true}
(str "pipenv run python scripts/compress_prompt.py"
" --input " input-file
" --output " output-file
" --ratio " ratio))]
(if (zero? (:exit result))
(do
(println (:out result))
(success-msg (str "Compressed: " output-file)))
(do
(println (:err result))
(error-msg "Compression failed")
(System/exit 1)))))}

migrate
{:doc "Run database migrations"
:task (do
Expand Down
7 changes: 2 additions & 5 deletions deps.edn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,14 @@
funcool/lentes {:mvn/version "1.3.3"}
clj-commons/clj-yaml {:mvn/version "1.0.29"}
org.clj-commons/pretty {:mvn/version "3.6.7"}
dev.weavejester/hashp {:mvn/version "0.5.1"}
io.github.paintparty/bling {:mvn/version "0.8.8"}

mvxcvi/puget {:mvn/version "1.3.4"}

org.slf4j/slf4j-nop {:mvn/version "2.0.17"}
;; SQL database dependencies
com.github.seancorfield/next.jdbc {:mvn/version "1.3.955"}
com.github.seancorfield/honeysql {:mvn/version "2.7.1350"}
dev.weavejester/ragtime.next-jdbc {:mvn/version "0.9.4"}
org.xerial/sqlite-jdbc {:mvn/version "3.47.1.0"}
com.github.clojure-lsp/clojure-lsp {:mvn/version "2025.08.25-14.21.46"}}
org.xerial/sqlite-jdbc {:mvn/version "3.47.1.0"}}

:paths ["src"
"classes"
Expand Down
Loading