test: add qa through repl and hopefully self-documented development via repl

* brainstorm on tree-sitter injections

README.md

@@ -3,6 +3,8 @@
 A Neovim plugin allowing users to perform workspace-wise operations (highlighting,
 list processing, mutation) on existing Treesitter query in Scheme.
 
+Please see [repl.md](./repl.md) for examples
+
 ## Features
 
 ### Buffer selection (`require('tsql').buf_match`)
@@ -51,7 +53,7 @@ Here's a basic example of how to use tsql.nvim:
 local ts = require('tsql')
 -- ts.t(<buf_match>, <ts_query>, <sink>)
 -- Matches all strings in our neovim workspace.
-ts.t(ts.buf_match.any(), ts.ts_query.from_scm("string"), ts.sink_by.print()):do_nvim()
+ts.t(ts.buf_match.any(), ts.ts_query.from_scm("(string) @_"), ts.sink_by.print()):do_nvim()
 ```
 
 ## Installation

lua/tsql.lua

@@ -1,4 +1,7 @@
----TODO: how does this work with changing texts? TODO: add reducer as formatter
+---TODO: how does this work with changing texts? (extmarks, but can look at `TextChanged` and `TextChangedI` events, along with some buf events to re-run when needs to)
+-- TODO: Better `ts_query` construction experience
+-- TODO: injection tree-sitter query
+---TODO: add reducer as formatter
 ---TODO: Add reducer as buf_select predicate
 
 ---@type mod_buf_select
@@ -35,6 +38,8 @@ end
 
 ---@return Sink
 function M.sink_by.highlight()
+    -- NOTE: magic! Neovim has extmarks, so in a very general case, you don't need to
+    -- re-run this sink to have the highlights be up-to-date
     return M.sink_by.pure_fn(function(nodes)
         for _, node in ipairs(nodes) do
             vim.highlight.range(
@@ -43,7 +48,7 @@ function M.sink_by.highlight()
                 M.config.nvim_hl_group,
                 { node.start.row_0, node.start.col_0 },
                 { node.end_ex_col.row_0, node.end_ex_col.col_0 },
-                {}
+                { inclusive = false }
             )
         end
     end)
@@ -164,6 +169,7 @@ end
 --- @param store StaticStore
 function M.Tsql:do_nvim(store)
     self.sink:sink(self:q_nodes())
+    -- FIXME: this should be limited to only highlight sink
     store:add_highlight(self)
 end
 

lua/tsql/buf_select.lua

@@ -17,22 +17,6 @@ local M = {}
 M.QBuf = {}
 M.QBuf.__index = M.QBuf
 
----@return string language
----NOTE: may fail with string
-local function get_lang(bufnr)
-    local status, lang = pcall(function()
-        return vim.treesitter.get_parser(bufnr):lang()
-    end)
-
-    -- PURPOSE: enriches the message
-    if not status then
-        local path = vim.api.nvim_buf_get_name(bufnr)
-        error(string.format("Error determining language for buffer %d: %s", bufnr, path))
-    end
-
-    return lang
-end
-
 ---@param bufnr number
 ---@param path string
 ---@param filetype string
@@ -56,6 +40,23 @@ function M.QBuf:new(bufnr, path, filetype, lang, is_loaded)
     return qbuf
 end
 
+---@return string language
+---NOTE: may fail with string
+local function get_lang(bufnr)
+    local status, lang = pcall(function()
+        return vim.treesitter.get_parser(bufnr):lang()
+    end)
+
+    -- PURPOSE: enriches the message
+    if not status then
+        local path = vim.api.nvim_buf_get_name(bufnr)
+        local err = lang
+        error(string.format("Error determining language for buffer %d: %s\n%s", bufnr, path, err))
+    end
+
+    return lang
+end
+
 ---@param bufnr number
 ---@return QBuf | nil
 function M.QBuf.from_nvim_bufnr(bufnr)

lua/tsql/token_select.lua

@@ -82,8 +82,7 @@ function M.TSQuery:find_nodes(files)
             for id, node in pairs(match) do
                 local start_row, start_col, end_row, end_col = node:range(false)
                 local start = M.FileLoc.new { row_0 = start_row, col_0 = start_col }
-                -- CORRECTNESS:
+                -- CORRECTNESS, see repl.md
-                -- :lua local parser = vim.treesitter.get_parser(0, 'lua'); local tree = parser:parse()[1]; local query = vim.treesitter.parse_query('lua', '(identifier) @name'); for id, node in query:iter_captures(tree:root(), 0) do local name = query.captures[id]; if name == 'name' and vim.treesitter.get_node_text(node, 0) == 'TSQuery' then local sr, sc, er, ec = node:range(); print(string.format("TSQuery Start: (%d, %d), End: (%d, %d)", sr, sc, er, ec)); end; end
                 local end_ex_col = M.FileLoc.new { row_0 = end_row, col_0 = end_col }
                 local qnode = M.QNode.new { buf = file, start = start, end_ex_col = end_ex_col }
 

repl.md

@@ -1,4 +1,12 @@
-# Simple location for me to use as a neovim repl
+---
+this_is: front-matter
+---
+
+# REPL
+
+Walks through how I develop tsql interactively.
+
+Within neovim, I just use `V"+y:<ctrl-v><enter>` to execute these one-liners
 
 ```lua
 :lua local ts = require('tsql'); ts.t(ts.buf_match.filetype("lua"), ts.ts_query.from_scm("(string) @_"), ts.sink_by.print()):do_nvim(ts.store);
@@ -8,13 +16,7 @@
 :lua local ts = require('tsql'); print(vim.inspect(ts.ts_query.from_scm("(string) @_"):find_nodes({ts.QBuf.from_nvim_bufnr(0)})))
 ```
 
-Non-tsql poc
+`do_nvim` only
-```lua
-:lua local parser = vim.treesitter.get_parser(0, 'lua'); local tree = parser:parse()[1]; local query = vim.treesitter.parse_query('lua', '(identifier) @name'); for id, node in query:iter_captures(tree:root(), 0) do local name = query.captures[id]; if name == 'name' and vim.treesitter.get_node_text(node, 0) == 'TSQuery' then local sr, sc, er, ec = node:range(); print(string.format("TSQuery Start: (%d, %d), End: (%d, %d)", sr, sc, er, ec)); end; end
-
-```
-
-
 ```lua
 :lua local ts = require('tsql'); ts.t(ts.buf_match.filetype("lua"), ts.ts_query.from_scm("(string) @_"), ts.sink_by.highlight()):do_nvim(ts.store);
 :lua local ts = require('tsql'); ts.t(ts.buf_match.filetype("lua"), ts.ts_query.from_scm("(identifier) @_"), ts.sink_by.highlight()):do_nvim(ts.store);
@@ -25,6 +27,91 @@ Non-tsql poc
 -- Highlight function calls, but only the function identifier
 :lua local ts = require('tsql'); ts.t(ts.buf_match.filetype("lua"), ts.ts_query.from_scm("[(function_call name: (dot_index_expression) @_) (function_call name: (identifier) @_) (function_call name: (method_index_expression) @_)]"), ts.sink_by.highlight()):do_nvim(ts.store);
 
+-- remove all highlights by this plugin
 :Noh
+```
+
+Non-tsql poc
+```lua
+:lua local parser = vim.treesitter.get_parser(0, 'lua'); local tree = parser:parse()[1]; local query = vim.treesitter.parse_query('lua', '(identifier) @name'); for id, node in query:iter_captures(tree:root(), 0) do local name = query.captures[id]; if name == 'name' and vim.treesitter.get_node_text(node, 0) == 'TSQuery' then local sr, sc, er, ec = node:range(); print(string.format("TSQuery Start: (%d, %d), End: (%d, %d)", sr, sc, er, ec)); end; end
 
 ```
+
+
+## Injection features
+
+TL;DR: 
+
+- Buffer-oriented parser 
+  - `local buf_parser = vim.treesitter.parser(bufnr: number): LanguageTree`: Buffer-specific parser
+  - `buf_parser:trees()`: Gets into the nodes within the buffer "host" language (usually based on filetype)
+  - `buf_parser:children(): table<language(string), LanguageTree>`: 
+
+```lua
+-- Parser for current buffer
+:lua print(vim.inspect(vim.treesitter.get_parser(0)))
+
+-- Using nvim-treesitter (what nvim-treesitter-playground uses)
+:lua print(vim.inspect(require('nvim-treesitter.parsers').get_parser(0)))
+-- NOTE: currently, 
+-- `nvim-treesitter.parsers.get_parser(
+-- buf_nr: number = factory(vim.api.nvim_get_current_buf), 
+-- lang: string = factory(get_buf_lang))
+-- is just a thin wrapper that has some backwards compatibilty. Not so much of our concern atm.
+```
+
+### NOTE: language from vim api
+
+```lua
+vim.treesitter.language.get_lang(ft: string)
+-- what we're currently using:
+vim.treesitter.get_parser(bufnr: number):lang
+```
+
+### Parsing with language injection
+
+The current file should be a good tester
+
+```lua
+:lua print(vim.inspect(vim.treesitter.get_parser(0):lang())) -- "markdown"
+:lua print(vim.inspect(vim.treesitter.language.get_lang(vim.api.nvim_buf_get_option(0, "filetype")))) -- "markdown"
+-- lang tree. We can see there are js, css, lua, markdown "children"
+-- within each are trees. There is also a `lang_tree._trees`
+:lua local lang_tree = vim.treesitter.get_parser(0); print(vim.inspect(lang_tree))
+-- This is our main language's tree
+:lua local lang_tree = vim.treesitter.get_parser(0); print(vim.inspect(lang_tree:trees()))
+-- This is our injected language trees. At the time of writing, `[:markdown-inline, :javascript, ]`
+:lua local lang_tree = vim.treesitter.get_parser(0); print(vim.inspect(lang_tree:children()))
+
+-- Content inside injection (markdown)
+:lua local
+```
+
+## Corpus
+
+just ignore this maybe, these are for upper blocks where it may need language injection
+
+### css-in-js-in-markdown
+
+If your highlighter doesn't support this, maybe think about switching to tree-sitter :)
+
+
+```js
+const my_class = css`
+  h1, h2, p {
+  text-align: center;
+  color: red;
+}
+`;
+
+const js_ts_limitations = lang.css`
+  h1, h2, p {
+    text-align: center;
+    color: red;
+  }
+`
+```
+
+```js some more cases
+const my_class = css`h1, h2, p { text-align: center; color: red; }`
+```