fun: fun presonal project research. gpt token optimizer

master
Hung 2023-06-03 23:48:38 -07:00
parent 969834a1ed
commit c70a653099
2 changed files with 49 additions and 48 deletions

View File

@ -1,8 +1,9 @@
;; basically transpiled from Fennel's website https://fennel-lang.org/see
;; Lua (git:603e6cc, ->fennel->lua to remove docs): (reduced) 4000 GPT-3 tokens, 3800 Codex tokens, 9800 chars
;; Fennel (git:603e6cc): 5106 GPT-3 tokens, 3488 Codex tokens, 10110 chars
;; Fennel (git:603e6cc, manually removed whitespace)
;; Fennel got hit mostly by white-spaces
;; Fennel got hit mostly by white-spaces in GPT-3
;; Fennel (git:603e6cc, manually removed whitespace): 4048 GPT-3 tokens, 3443 Codex tokens, 9051 chars
(set buf-select.QBuf {})
(set buf-select.QBuf.__index buf-select.QBuf)
@ -19,7 +20,7 @@
(when (not status) (local path (vim.api.nvim_buf_get_name bufnr))
(local err lang)
(error (string.format "Error determining language for buffer %d: %s\n%s"
bufnr path err)))
bufnr path err)))
lang))
(fn buf-select.QBuf.from_nvim_bufnr [bufnr]
(let [path (vim.api.nvim_buf_get_name bufnr)
@ -44,40 +45,40 @@
(fn buf-select.buf_match.filetype [...]
(let [filetypes [...]]
(buf-select.BufMatch.new (fn [buf]
(each [_ filetype (ipairs filetypes)]
(when (= buf.filetype filetype)
(lua "return true")))
false))))
(each [_ filetype (ipairs filetypes)]
(when (= buf.filetype filetype)
(lua "return true")))
false))))
(fn buf-select.buf_match.any []
(buf-select.BufMatch.new (fn [_] true)))
(fn buf-select.buf_match.path [...]
(let [paths [...]]
(buf-select.BufMatch.new (fn [buf]
(each [_ path (ipairs paths)]
(when (not= (string.find buf.path path) nil)
(lua "return true")))
false))))
(each [_ path (ipairs paths)]
(when (not= (string.find buf.path path) nil)
(lua "return true")))
false))))
(fn buf-select.buf_match.path_or [...] (buf-select.buf_match.path ...))
(fn buf-select.buf_match.path_and [...]
(let [paths [...]]
(buf-select.BufMatch.new (fn [buf]
(each [_ path (ipairs paths)]
(when (= (string.find buf.path path) nil)
(lua "return false")))
true))))
(each [_ path (ipairs paths)]
(when (= (string.find buf.path path) nil)
(lua "return false")))
true))))
(fn buf-select.buf_match.ext [...]
(let [exts [...]]
(buf-select.BufMatch.new (fn [buf]
(each [_ ext (ipairs exts)]
(when (= (buf.path:sub (- (length ext))) ext)
(lua "return true")))
false))))
(each [_ ext (ipairs exts)]
(when (= (buf.path:sub (- (length ext))) ext)
(lua "return true")))
false))))
(fn buf-select.BufMatch.or_ [self q]
(buf-select.BufMatch.new (fn [buf]
(or (self.matched_fn buf) (q.matched_fn buf)))))
(or (self.matched_fn buf) (q.matched_fn buf)))))
(fn buf-select.BufMatch.then_ [self q]
(buf-select.BufMatch.new (fn [buf]
(and (self.matched_fn buf) (q.matched_fn buf)))))
(and (self.matched_fn buf) (q.matched_fn buf)))))
(fn buf-select.BufMatch.not_ [self]
(buf-select.BufMatch.new (fn [buf] (not (self.matched_fn buf)))))
(fn buf-select.BufMatch.filter_on [self itr]
@ -121,27 +122,27 @@
(each [_ file (ipairs files)]
(when (not= file.lang nil)
(local (sts parser) (pcall vim.treesitter.get_parser file.bufnr
file.filetype))
file.filetype))
(when sts
(local tree (. (parser:parse) 1))
(local root (tree:root))
(local (sts query) (pcall vim.treesitter.query.parse file.lang
self.query))
self.query))
(when (not sts)
(error (.. "Error parsing query \"" self.query "\" on lang "
file.lang " for file " file.path)))
file.lang " for file " file.path)))
(each [_ ___match___ _ (query:iter_matches root file.bufnr 0 (- 1))]
(each [id node (pairs ___match___)]
(local (start-row start-col end-row end-col) (node:range false))
(local start
(token-select.FileLoc.new {:col_0 start-col
:row_0 start-row}))
(token-select.FileLoc.new {:col_0 start-col
:row_0 start-row}))
(local end-ex-col
(token-select.FileLoc.new {:col_0 end-col :row_0 end-row}))
(token-select.FileLoc.new {:col_0 end-col :row_0 end-row}))
(local qnode
(token-select.QNode.new {:buf file
:end_ex_col end-ex-col
: start}))
(token-select.QNode.new {:buf file
:end_ex_col end-ex-col
: start}))
(table.insert result qnode))))))
result))
(local M {})
@ -155,21 +156,21 @@
self))
(fn M.sink_by.highlight []
(M.sink_by.pure_fn (fn [nodes]
(each [_ node (ipairs nodes)]
(vim.highlight.range node.buf.bufnr M.config.nvim_ns
M.config.nvim_hl_group
[node.start.row_0
node.start.col_0]
[node.end_ex_col.row_0
node.end_ex_col.col_0]
{:inclusive false})))))
(each [_ node (ipairs nodes)]
(vim.highlight.range node.buf.bufnr M.config.nvim_ns
M.config.nvim_hl_group
[node.start.row_0
node.start.col_0]
[node.end_ex_col.row_0
node.end_ex_col.col_0]
{:inclusive false})))))
(fn M.format.display [nodes]
(let [texts {}]
(each [_ node (ipairs nodes)]
(local text
(vim.api.nvim_buf_get_text node.buf.bufnr node.start.row_0
node.start.col_0 node.end_ex_col.row_0
node.end_ex_col.col_0 {}))
(vim.api.nvim_buf_get_text node.buf.bufnr node.start.row_0
node.start.col_0 node.end_ex_col.row_0
node.end_ex_col.col_0 {}))
(table.insert texts (table.concat text "\n")))
(table.concat texts "\n\n")))
(fn M.format.dump [nodes] (vim.inspect nodes {:indent " " :newline "\n"}))
@ -181,7 +182,7 @@
(fn M.sink_by.nvim_yank_buf [format]
(when (= format nil) (set-forcibly! format M.format.default))
(M.sink.pure_fn (fn [nodes]
(let [text (format nodes)] (vim.fn.setreg "\"" text)))))
(let [text (format nodes)] (vim.fn.setreg "\"" text)))))
(set M.buf_match buf-select.buf_match)
(set M.BufMatch buf-select.BufMatch)
(set M.QBuf buf-select.QBuf)
@ -222,14 +223,14 @@
(fn M.clear_highlights [] (M._delete_all_highlights M.config M.store))
(fn M.setup [config]
(set M.config
(vim.tbl_deep_extend :force M.config (or config M.config_default)))
(vim.tbl_deep_extend :force M.config (or config M.config_default)))
(set M.config.nvim_ns (vim.api.nvim_create_namespace :tsql))
(vim.api.nvim_create_user_command :Noh (fn [_] (M.clear_highlights))
{:desc "Clear tsql highlights" :nargs 0})
{:desc "Clear tsql highlights" :nargs 0})
(vim.api.nvim_create_user_command :Tdsl
(fn [cmd]
(: (M.s cmd.args) :do_nvim M.store))
{:desc "tsql DSL invocation" :nargs "?"})
(fn [cmd]
(: (M.s cmd.args) :do_nvim M.store))
{:desc "tsql DSL invocation" :nargs "?"})
(set M.store (M.Store:new)))
M

View File

@ -45,7 +45,7 @@ TL;DR:
- Buffer-oriented parser
- `local buf_parser = vim.treesitter.parser(bufnr: number): LanguageTree`: Buffer-specific parser
- `buf_parser:trees()`: Gets into the nodes within the buffer "host" language (usually based on filetype)
- `buf_parser:children(): table<language(string), LanguageTree>`:
- `buf_parser:children(): table<language(string), LanguageTree>`: Treesitter injection!
```lua
-- Parser for current buffer