fun: fun presonal project research. gpt token optimizer

fennel/tsql.fnl

@@ -1,8 +1,9 @@
 ;; basically transpiled from Fennel's website https://fennel-lang.org/see
 ;; Lua (git:603e6cc, ->fennel->lua to remove docs): (reduced) 4000 GPT-3 tokens, 3800 Codex tokens, 9800 chars
 ;; Fennel (git:603e6cc): 5106 GPT-3 tokens, 3488 Codex tokens, 10110 chars
-;; Fennel (git:603e6cc, manually removed whitespace)
+
-;; Fennel got hit mostly by white-spaces
+;; Fennel got hit mostly by white-spaces in GPT-3
+;; Fennel (git:603e6cc, manually removed whitespace): 4048 GPT-3 tokens, 3443 Codex tokens, 9051 chars
 
 (set buf-select.QBuf {})
 (set buf-select.QBuf.__index buf-select.QBuf)
@@ -19,7 +20,7 @@
     (when (not status) (local path (vim.api.nvim_buf_get_name bufnr))
       (local err lang)
       (error (string.format "Error determining language for buffer %d: %s\n%s"
-                            bufnr path err)))
+        bufnr path err)))
     lang))
 (fn buf-select.QBuf.from_nvim_bufnr [bufnr]
   (let [path (vim.api.nvim_buf_get_name bufnr)
@@ -44,40 +45,40 @@
 (fn buf-select.buf_match.filetype [...]
   (let [filetypes [...]]
     (buf-select.BufMatch.new (fn [buf]
-                               (each [_ filetype (ipairs filetypes)]
+       (each [_ filetype (ipairs filetypes)]
-                                 (when (= buf.filetype filetype)
+         (when (= buf.filetype filetype)
-                                   (lua "return true")))
+           (lua "return true")))
-                               false))))
+       false))))
 (fn buf-select.buf_match.any []
   (buf-select.BufMatch.new (fn [_] true)))
 (fn buf-select.buf_match.path [...]
   (let [paths [...]]
     (buf-select.BufMatch.new (fn [buf]
-                               (each [_ path (ipairs paths)]
+       (each [_ path (ipairs paths)]
-                                 (when (not= (string.find buf.path path) nil)
+         (when (not= (string.find buf.path path) nil)
-                                   (lua "return true")))
+           (lua "return true")))
-                               false))))
+       false))))
 (fn buf-select.buf_match.path_or [...] (buf-select.buf_match.path ...))
 (fn buf-select.buf_match.path_and [...]
   (let [paths [...]]
     (buf-select.BufMatch.new (fn [buf]
-                               (each [_ path (ipairs paths)]
+       (each [_ path (ipairs paths)]
-                                 (when (= (string.find buf.path path) nil)
+         (when (= (string.find buf.path path) nil)
-                                   (lua "return false")))
+           (lua "return false")))
-                               true))))
+       true))))
 (fn buf-select.buf_match.ext [...]
   (let [exts [...]]
     (buf-select.BufMatch.new (fn [buf]
-                               (each [_ ext (ipairs exts)]
+       (each [_ ext (ipairs exts)]
-                                 (when (= (buf.path:sub (- (length ext))) ext)
+         (when (= (buf.path:sub (- (length ext))) ext)
-                                   (lua "return true")))
+           (lua "return true")))
-                               false))))
+       false))))
 (fn buf-select.BufMatch.or_ [self q]
   (buf-select.BufMatch.new (fn [buf]
-                             (or (self.matched_fn buf) (q.matched_fn buf)))))
+       (or (self.matched_fn buf) (q.matched_fn buf)))))
 (fn buf-select.BufMatch.then_ [self q]
   (buf-select.BufMatch.new (fn [buf]
-                             (and (self.matched_fn buf) (q.matched_fn buf)))))
+       (and (self.matched_fn buf) (q.matched_fn buf)))))
 (fn buf-select.BufMatch.not_ [self]
   (buf-select.BufMatch.new (fn [buf] (not (self.matched_fn buf)))))
 (fn buf-select.BufMatch.filter_on [self itr]
@@ -121,27 +122,27 @@
     (each [_ file (ipairs files)]
       (when (not= file.lang nil)
         (local (sts parser) (pcall vim.treesitter.get_parser file.bufnr
-                                   file.filetype))
+           file.filetype))
         (when sts
           (local tree (. (parser:parse) 1))
           (local root (tree:root))
           (local (sts query) (pcall vim.treesitter.query.parse file.lang
-                                    self.query))
+            self.query))
           (when (not sts)
             (error (.. "Error parsing query \"" self.query "\" on lang "
-                       file.lang " for file " file.path)))
+               file.lang " for file " file.path)))
           (each [_ ___match___ _ (query:iter_matches root file.bufnr 0 (- 1))]
             (each [id node (pairs ___match___)]
               (local (start-row start-col end-row end-col) (node:range false))
               (local start
-                     (token-select.FileLoc.new {:col_0 start-col
+                 (token-select.FileLoc.new {:col_0 start-col
-                                                :row_0 start-row}))
+                  :row_0 start-row}))
               (local end-ex-col
-                     (token-select.FileLoc.new {:col_0 end-col :row_0 end-row}))
+                 (token-select.FileLoc.new {:col_0 end-col :row_0 end-row}))
               (local qnode
-                     (token-select.QNode.new {:buf file
+                 (token-select.QNode.new {:buf file
-                                              :end_ex_col end-ex-col
+                  :end_ex_col end-ex-col
-                                              : start}))
+                  : start}))
               (table.insert result qnode))))))
     result))
 (local M {})
@@ -155,21 +156,21 @@
     self))
 (fn M.sink_by.highlight []
   (M.sink_by.pure_fn (fn [nodes]
-                       (each [_ node (ipairs nodes)]
+     (each [_ node (ipairs nodes)]
-                         (vim.highlight.range node.buf.bufnr M.config.nvim_ns
+       (vim.highlight.range node.buf.bufnr M.config.nvim_ns
-                                              M.config.nvim_hl_group
+          M.config.nvim_hl_group
-                                              [node.start.row_0
+          [node.start.row_0
-                                               node.start.col_0]
+           node.start.col_0]
-                                              [node.end_ex_col.row_0
+          [node.end_ex_col.row_0
-                                               node.end_ex_col.col_0]
+           node.end_ex_col.col_0]
-                                              {:inclusive false})))))
+          {:inclusive false})))))
 (fn M.format.display [nodes]
   (let [texts {}]
     (each [_ node (ipairs nodes)]
       (local text
-             (vim.api.nvim_buf_get_text node.buf.bufnr node.start.row_0
+         (vim.api.nvim_buf_get_text node.buf.bufnr node.start.row_0
-                                        node.start.col_0 node.end_ex_col.row_0
+            node.start.col_0 node.end_ex_col.row_0
-                                        node.end_ex_col.col_0 {}))
+            node.end_ex_col.col_0 {}))
       (table.insert texts (table.concat text "\n")))
     (table.concat texts "\n\n")))
 (fn M.format.dump [nodes] (vim.inspect nodes {:indent "  " :newline "\n"}))
@@ -181,7 +182,7 @@
 (fn M.sink_by.nvim_yank_buf [format]
   (when (= format nil) (set-forcibly! format M.format.default))
   (M.sink.pure_fn (fn [nodes]
-                    (let [text (format nodes)] (vim.fn.setreg "\"" text)))))
+      (let [text (format nodes)] (vim.fn.setreg "\"" text)))))
 (set M.buf_match buf-select.buf_match)
 (set M.BufMatch buf-select.BufMatch)
 (set M.QBuf buf-select.QBuf)
@@ -222,14 +223,14 @@
 (fn M.clear_highlights [] (M._delete_all_highlights M.config M.store))
 (fn M.setup [config]
   (set M.config
-       (vim.tbl_deep_extend :force M.config (or config M.config_default)))
+     (vim.tbl_deep_extend :force M.config (or config M.config_default)))
   (set M.config.nvim_ns (vim.api.nvim_create_namespace :tsql))
   (vim.api.nvim_create_user_command :Noh (fn [_] (M.clear_highlights))
-                                    {:desc "Clear tsql highlights" :nargs 0})
+    {:desc "Clear tsql highlights" :nargs 0})
   (vim.api.nvim_create_user_command :Tdsl
-                                    (fn [cmd]
+    (fn [cmd]
-                                      (: (M.s cmd.args) :do_nvim M.store))
+      (: (M.s cmd.args) :do_nvim M.store))
-                                    {:desc "tsql DSL invocation" :nargs "?"})
+    {:desc "tsql DSL invocation" :nargs "?"})
   (set M.store (M.Store:new)))
 M
 

repl.md

@@ -45,7 +45,7 @@ TL;DR:
 - Buffer-oriented parser 
   - `local buf_parser = vim.treesitter.parser(bufnr: number): LanguageTree`: Buffer-specific parser
   - `buf_parser:trees()`: Gets into the nodes within the buffer "host" language (usually based on filetype)
-  - `buf_parser:children(): table<language(string), LanguageTree>`: 
+  - `buf_parser:children(): table<language(string), LanguageTree>`: Treesitter injection!
 
 ```lua
 -- Parser for current buffer