From 7ec1acf1ef50fa091e49ec0fb529d904b90c550c Mon Sep 17 00:00:00 2001 From: dsent <8774536+dsent@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:16:49 +0200 Subject: [PATCH] build: vendor metalua and stringutils as tracked files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A submodule buys isolation from an upstream that moves on its own. Neither of these has one. Metalua's last commit from its author is from June 2014; the compy-toys fork is where all movement since then has happened, and its only forks are personal ones belonging to contributors here. compy-toys/stringutils has no fork and no consumer but this repository. So the isolation is bought against nothing, while the ceremony is paid in full. That ceremony is three rounds for a one-line change to stringutils: open a PR against stringutils, wait for the merge, bump the pin in metalua and open a PR there, wait for that merge, then bump the pin here. Skipping steps is the natural response, and the visible result is that src/lib/metalua is pinned at d0dbd0d9 — a pull-request head rather than a commit on a branch. The other cost is drift against itself. stringutils is carried twice at two different commits — src/util/string at ff9be4b9 and src/lib/metalua/stringutils at 3662789, three commits behind it — so one checkout holds two versions of the same library, and which one a module gets depends on the path it requires. src/lib/metalua now holds the part this project loads: the fifteen modules metalua packages as metalua-parser, plus compiler/ast_to_src.lua, the only piece of metalua-compiler that model/lang/lua/parser.lua reaches. Dropped: the bytecode compiler, metalua/loader.lua, compiler/globals.lua, the metalua.lua CLI and every .mlua source. Nothing loads the metalua loader at runtime, so no .mlua file was reachable. compiler/parser/common.lua returns an empty table for an upstream parser.init() this fork does not have and nothing requires it. Across those sixteen files the only change that is not a license header is one line: ast_to_src.lua now requires util.string.string instead of stringutils.string, so the surviving stringutils serves both. metalua is dual-licensed MIT and EPL-1.0. This copy is carried under MIT alone: the EPL paragraphs are dropped from each header, the authors' copyright and contributor lines stay, and LICENSE carries the MIT text. src/lib/metalua/README.md records the origin commit and what was left behind. The AST corpus moved from the submodule's spec/ to tests/interpreter/ast_inputs.lua, beside the suite's other inputs, so ast_spec no longer degrades to zero cases when submodules are missing. Tests: 693 successes / 0 failures, matching this branch's point before the change, with the ast tag at 148 both sides. --- .github/workflows/package.yml | 6 - .gitmodules | 8 - DEVELOPMENT.md | 19 +- src/lib/metalua | 1 - src/lib/metalua/LICENSE | 20 + src/lib/metalua/README.md | 56 + src/lib/metalua/checks.lua | 52 + src/lib/metalua/metalua/compiler.lua | 183 +++ .../metalua/metalua/compiler/ast_to_src.lua | 1231 +++++++++++++++ src/lib/metalua/metalua/compiler/parser.lua | 35 + .../compiler/parser/annot/generator.lua | 41 + .../metalua/compiler/parser/annot/grammar.lua | 105 ++ .../metalua/metalua/compiler/parser/expr.lua | 199 +++ .../metalua/metalua/compiler/parser/ext.lua | 89 ++ .../metalua/metalua/compiler/parser/lexer.lua | 36 + .../metalua/metalua/compiler/parser/meta.lua | 131 ++ .../metalua/metalua/compiler/parser/misc.lua | 168 ++ .../metalua/metalua/compiler/parser/stat.lua | 272 ++++ .../metalua/metalua/compiler/parser/table.lua | 70 + src/lib/metalua/metalua/grammar/generator.lua | 854 ++++++++++ src/lib/metalua/metalua/grammar/lexer.lua | 773 +++++++++ src/lib/metalua/metalua/pprint.lua | 288 ++++ src/util/string | 1 - src/util/string/README.md | 8 + src/util/string/string.lua | 411 +++++ src/util/string/utf.lua | 7 + tests/interpreter/ast_inputs.lua | 1388 +++++++++++++++++ tests/interpreter/ast_spec.lua | 10 +- 28 files changed, 6424 insertions(+), 38 deletions(-) delete mode 100644 .gitmodules delete mode 160000 src/lib/metalua create mode 100644 src/lib/metalua/LICENSE create mode 100644 src/lib/metalua/README.md create mode 100644 src/lib/metalua/checks.lua create mode 100644 src/lib/metalua/metalua/compiler.lua create mode 100644 src/lib/metalua/metalua/compiler/ast_to_src.lua create mode 100644 src/lib/metalua/metalua/compiler/parser.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/annot/generator.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/annot/grammar.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/expr.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/ext.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/lexer.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/meta.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/misc.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/stat.lua create mode 100644 src/lib/metalua/metalua/compiler/parser/table.lua create mode 100644 src/lib/metalua/metalua/grammar/generator.lua create mode 100644 src/lib/metalua/metalua/grammar/lexer.lua create mode 100644 src/lib/metalua/metalua/pprint.lua delete mode 160000 src/util/string create mode 100644 src/util/string/README.md create mode 100644 src/util/string/string.lua create mode 100644 src/util/string/utf.lua create mode 100644 tests/interpreter/ast_inputs.lua diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d7dd4f039..c03447ed1 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -32,8 +32,6 @@ jobs: cancel-in-progress: true steps: - uses: actions/checkout@v4 - with: - submodules: "recursive" - uses: jkl1337/gh-actions-lua@v11 with: @@ -58,7 +56,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: "recursive" fetch-depth: 0 fetch-tags: true - name: install just @@ -106,7 +103,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: "recursive" fetch-depth: 0 fetch-tags: "true" - name: Download love package @@ -152,7 +148,6 @@ jobs: - uses: actions/checkout@v4 with: - submodules: "recursive" fetch-depth: 0 fetch-tags: "true" - name: Download love package @@ -236,7 +231,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: "recursive" fetch-tags: "true" fetch-depth: 0 - name: Download love package diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 7dc987c0c..000000000 --- a/.gitmodules +++ /dev/null @@ -1,8 +0,0 @@ -[submodule "src/lib/metalua"] - path = src/lib/metalua - url = https://github.com/compy-toys/metalua.git - branch = dev - -[submodule "src/util/string"] - path = src/util/string - url = https://github.com/compy-toys/stringutils.git diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c805f1749..459aa169c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,22 +1,15 @@ ## Cloning -Clone this project recursively, because libraries are included -as submodules: +A plain clone gives you everything: ```shell -git clone --recurse-submodules -# or -git clone --recurse-submodules --shallow-submodules +git clone ``` -If it's already cloned without, and you don't want to start over, -they can be initialized with: - -```shell -git submodule update --init -# or -git submodule update --init --depth 1 -``` +The libraries under `src/lib/` are tracked files, so there is +nothing to initialize afterwards. Each carries its own `README.md` +recording where it came from and at which commit; edit those copies +here rather than upstream. ## Installing diff --git a/src/lib/metalua b/src/lib/metalua deleted file mode 160000 index d0dbd0d98..000000000 --- a/src/lib/metalua +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d0dbd0d982f87512b806949ea697ea71f39cd0b4 diff --git a/src/lib/metalua/LICENSE b/src/lib/metalua/LICENSE new file mode 100644 index 000000000..643693645 --- /dev/null +++ b/src/lib/metalua/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2006-2013 Fabien Fleutot and others. +Copyright (c) 2024-2025 the Compy contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/lib/metalua/README.md b/src/lib/metalua/README.md new file mode 100644 index 000000000..b68f3c3a4 --- /dev/null +++ b/src/lib/metalua/README.md @@ -0,0 +1,56 @@ +# metalua (vendored) + +The part of [Metalua](https://github.com/compy-toys/metalua) the Compy editor +reads: the front end that turns Lua source into a checked AST, plus the printer +that turns an AST back into source. + +## Origin + +- Upstream: `https://github.com/compy-toys/metalua.git`, branch `dev` +- Imported at commit `d0dbd0d982f87512b806949ea697ea71f39cd0b4` +- That commit is `refs/pull/3/head`. PR #3 was merged into `dev` by + rebase, so `dev` carries the same five commits under different + hashes and its head `a42b3918` has an identical tree + (`5e4d41cc483fad980d742b92a1ab44cd05b93665`). The import is `dev`'s + content, reached by the hash the IDE was pinned at. +- Upstream of that fork: Eclipse Koneki Metalua 0.7.2 + +## What is here + +`metalua-parser`'s modules, which upstream packages separately from the +bytecode compiler: + +- `metalua/grammar/lexer.lua`, `metalua/grammar/generator.lua` +- `metalua/compiler/parser.lua` and `metalua/compiler/parser/*` +- `metalua/compiler.lua` — the front end `model.lang.lua.parser` instantiates +- `metalua/pprint.lua` +- `checks.lua` — argument checking, required by the front end; installs a + global `checkers` table + +Plus one module from `metalua-compiler`: + +- `metalua/compiler/ast_to_src.lua` — the source printer the editor uses to + render a chunk back to text + +`checks.lua` and `ast_to_src.lua`'s stringutils dependency resolve through the +editor's own `src/util/string`, reached as `util.string.string`. + +## What was left behind + +The bytecode compiler (`metalua/compiler/bytecode*`), the metaprogramming +loader (`metalua/loader.lua`, `metalua/compiler/globals.lua`), the CLI +(`metalua.lua`), and every `.mlua` source — extensions, `repl`, `treequery`, +`dollar`. Nothing loads the Metalua loader at runtime, so no `.mlua` file is +reachable. The AST test corpus moved to `tests/interpreter/ast_inputs.lua`. + +## License + +Metalua is dual-licensed MIT and EPL-1.0 upstream. This copy is carried under +MIT alone; see `LICENSE`. The per-file headers keep the authors' copyright and +contributor lines and name MIT as the single grant. + +## Changing this code + +Edit it here. This is a vendored copy, not a submodule, and the fork it came +from already diverges from Eclipse Metalua in `grammar/lexer.lua`, +`grammar/generator.lua`, `compiler.lua`, and `compiler/ast_to_src.lua`. diff --git a/src/lib/metalua/checks.lua b/src/lib/metalua/checks.lua new file mode 100644 index 000000000..8698508b2 --- /dev/null +++ b/src/lib/metalua/checks.lua @@ -0,0 +1,52 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +-- Alternative implementation of checks() in Lua. Slower than +-- the C counterpart, but no compilation/porting concerns. + +checkers = { } + +local function check_one(expected, val) + if type(val)==expected then return true end + local mt = getmetatable(val) + if mt and mt.__type==expected then return true end + local f = checkers[expected] + if f and f(val) then return true end + return false +end + +local function check_many(name, expected, val) + if expected=='?' then return true + elseif expected=='!' then return (val~=nil) + elseif type(expected) ~= 'string' then + error 'strings expected by checks()' + elseif val==nil and expected :sub(1,1) == '?' then return true end + for one in expected :gmatch "[^|?]+" do + if check_one(one, val) then return true end + end + return false +end + +function checks(...) + for i, arg in ipairs{...} do + local name, val = debug.getlocal(2, i) + local success = check_many(name, arg, val) + if not success then + local fname = debug.getinfo(2, 'n').name + local fmt = "bad argument #%d to '%s' (%s expected, got %s)" + local msg = string.format(fmt, i, fname or "?", arg, type(val)) + error(msg, 3) + end + end +end + +return checks diff --git a/src/lib/metalua/metalua/compiler.lua b/src/lib/metalua/metalua/compiler.lua new file mode 100644 index 000000000..2fae5f201 --- /dev/null +++ b/src/lib/metalua/metalua/compiler.lua @@ -0,0 +1,183 @@ +--------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Convert between various code representation formats. Atomic +-- converters are written in extenso, others are composed automatically +-- by chaining the atomic ones together in a closure. +-- +-- Supported formats are: +-- +-- * srcfile: the name of a file containing sources. +-- * src: these sources as a single string. +-- * lexstream: a stream of lexemes. +-- * ast: an abstract syntax tree. +-- * proto: a (Yueliang) struture containing a high level +-- representation of bytecode. Largely based on the +-- Proto structure in Lua's VM +-- * bytecode: a string dump of the function, as taken by +-- loadstring() and produced by string.dump(). +-- * function: an executable lua function in RAM. +-- +-------------------------------------------------------------------------------- + +require 'checks' + +local M = {} + +-------------------------------------------------------------------------------- +-- Order of the transformations. if 'a' is on the left of 'b', then a 'a' can +-- be transformed into a 'b' (but not the other way around). +-- M.sequence goes for numbers to format names, M.order goes from format +-- names to numbers. +-------------------------------------------------------------------------------- +M.sequence = { + 'srcfile', 'src', 'lexstream', 'ast', 'proto', 'bytecode', 'function' } + +local arg_types = { + srcfile = { 'string', '?string' }, + src = { 'string', '?string' }, + lexstream = { 'lexer.stream', '?string' }, + ast = { 'table', '?string' }, + proto = { 'table', '?string' }, + bytecode = { 'string', '?string' }, +} + +if false then + -- if defined, runs on every newly-generated AST + function M.check_ast(ast) + local function rec(x, n, parent) + if not x.lineinfo and parent.lineinfo then + local pp = require 'metalua.pprint' + pp.printf("WARNING: Missing lineinfo in child #%s `%s{...} of node at %s", + n, x.tag or '', tostring(parent.lineinfo)) + end + for i, child in ipairs(x) do + if type(child) == 'table' then rec(child, i, x) end + end + end + rec(ast, -1, {}) + end +end + + +M.order = {}; for a, b in pairs(M.sequence) do M.order[b] = a end + +local CONV = {} -- conversion metatable __index + +function CONV:srcfile_to_src(x, name) + checks('metalua.compiler', 'string', '?string') + name = name or '@' .. x + local f, msg = io.open(x, 'rb') + if not f then error(msg) end + local r, msg = f:read '*a' + if not r then error("Cannot read file '" .. x .. "': " .. msg) end + f:close() + return r, name +end + +function CONV:src_to_lexstream(src, name) + checks('metalua.compiler', 'string', '?string') + local r = self.parser.lexer:newstream(src, name) + return r, name +end + +function CONV:lexstream_to_ast(lx, name) + checks('metalua.compiler', 'lexer.stream', '?string') + local r = self.parser.chunk(lx) + r.source = name + if M.check_ast then M.check_ast(r) end + return r, name +end + +local bytecode_compiler = nil -- cache to avoid repeated `pcall(require(...))` +local function get_bytecode_compiler() + if bytecode_compiler then + return bytecode_compiler + else + local status, result = pcall(require, 'metalua.compiler.bytecode') + if status then + bytecode_compiler = result + return result + elseif string.match(result, "not found") then + error "Compilation only available with full Metalua" + else + error(result) + end + end +end + +function CONV:ast_to_proto(ast, name) + checks('metalua.compiler', 'table', '?string') + return get_bytecode_compiler().ast_to_proto(ast, name), name +end + +function CONV:proto_to_bytecode(proto, name) + return get_bytecode_compiler().proto_to_bytecode(proto), name +end + +function CONV:bytecode_to_function(bc, name) + checks('metalua.compiler', 'string', '?string') + return loadstring(bc, name) +end + +-- Create all sensible combinations +for i = 1, #M.sequence do + local src = M.sequence[i] + for j = i + 2, #M.sequence do + local dst = M.sequence[j] + local dst_name = src .. "_to_" .. dst + local my_arg_types = arg_types[src] + local functions = {} + for k = i, j - 1 do + local name = M.sequence[k] .. "_to_" .. M.sequence[k + 1] + local f = assert(CONV[name], name) + table.insert(functions, f) + end + CONV[dst_name] = function(self, a, b) + checks('metalua.compiler', unpack(my_arg_types)) + for _, f in ipairs(functions) do + a, b = f(self, a, b) + end + return a, b + end + --printf("Created M.%s out of %s", dst_name, table.concat(n, ', ')) + end +end + + +-------------------------------------------------------------------------------- +-- This one goes in the "wrong" direction, cannot be composed. +-------------------------------------------------------------------------------- +function CONV:function_to_bytecode(...) return string.dump(...) end + +local ast2 = require 'metalua.compiler.ast_to_src' + +function CONV:a2s(...) + return ast2.new(...) +end + +function CONV:ast_to_src(...) + return ast2:oneshot(...) +end + +local MT = { __index = CONV, __type = 'metalua.compiler' } + +function M.new() + local parser = require 'metalua.compiler.parser'.new() + local self = { parser = parser } + setmetatable(self, MT) + return self +end + +return M diff --git a/src/lib/metalua/metalua/compiler/ast_to_src.lua b/src/lib/metalua/metalua/compiler/ast_to_src.lua new file mode 100644 index 000000000..ae97c29d3 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/ast_to_src.lua @@ -0,0 +1,1231 @@ +---------------------------------------------------------------- +--- Copyright (c) 2006-2013 Fabien Fleutot and others. +--- +--- Made available under the terms of the MIT public license, which +--- accompanies this distribution in LICENSE and is available at +--- http://www.lua.org/license.html +--- +--- Contributors: +--- Fabien Fleutot - API and implementation +--- +---------------------------------------------------------------- + +--- @class M +--- @field _acc table +--- @field current_indent integer +--- @field indent_step string +--- @field _line_len integer +--- @field _lines integer +--- @field comment_ids table +--- @field comment_lines table +--- @field wrap integer +local M = {} +M.__index = M + +local pp = require("metalua.pprint") +require("util.string.string") + + +---------------------------------------------------------------- +--- Instantiate a new AST->source synthetizer +--- @param seen_comments integer[]? +--- @param w integer? +--- @return M +---------------------------------------------------------------- +function M.new(seen_comments, w) + local self = { + -- Accumulates pieces of source as strings + _acc = {}, + -- Current level of line indentation + current_indent = 0, + -- Indentation symbol, normally spaces or '\t' + indent_step = " ", + -- Comments index accumulator + comment_ids = seen_comments or {}, + comment_lines = {}, + -- Number of characters since last linebreak + _line_len = 0, + -- Number of linebreaks + _lines = 0, + -- wrap length + wrap = w or 80, + -- Last source line number emitted (used to detect blank-line gaps) + _last_nonempty_src_line = 0 + } + return setmetatable(self, M) +end + +---------------------------------------------------------------- +--- Run a synthetizer on the `ast' arg and return the source as +--- a string. +--- Can also be used as a static method `M.run (ast)'; +--- in this case a temporary Metizer is instantiated on the fly. +--- @param seen_comments integer[]? +--- @param w integer? +--- @return string +--- @return integer[] +---------------------------------------------------------------- +function M:run(ast, seen_comments, w) + if not ast then + self, ast = M.new(seen_comments, w), self + end + self._acc = {} + self:node(ast) + return self:render() +end + +--- @return string +--- @return integer[] +function M:render() + return table.concat(self._acc), self.comment_ids +end + +---------------------------------------------------------------- +--- Spin up another instance and render the source for the +--- passed node +--- @param node token +--- @return string +---------------------------------------------------------------- +function M:prerender(node) + local comments = {} + for k, v in pairs(self.comment_ids) do + comments[k] = v + end + local w = self.wrap + local renderer = M.new(comments, w) + renderer:node(node) + local rendered, _ = renderer:render() + + return rendered +end + +---------------------------------------------------------------- +--- Accumulate a piece of source file in the synthetizer. +--- @param x string +---------------------------------------------------------------- +function M:acc(x) + if x then + local clen = self._line_len + local l = string.ulen(x) + local lines = string.lines(x) + local n_l = #lines + if l + clen > self.wrap + --- if the string has multiple lines, + --- handle it elsewhere + and n_l < 2 + --- don't create a leading empty line + --- for an overlong token + and clen > 0 + then + --- The formatter already emits a newline plus + --- continuation indent here; keep separator spacing from + --- adding a fifth visual indent column. + local wrapped_x = x:gsub("^%s+", "") + local ind = self.indent_step:rep(self.current_indent + 2) + self:acc("\n" .. ind) + self._line_len = #ind + table.insert(self._acc, wrapped_x) + else + self._line_len = clen + l + table.insert(self._acc, x) + end + end +end + +---------------------------------------------------------------- +--- Check if a piece of source can fit within the width limit +--- @param s string +---------------------------------------------------------------- +function M:fits(s) + if type(s) == 'string' then + local clen = self._line_len + local l = string.ulen(s) + local lines = string.lines(s) + local n_l = #lines + if n_l == 1 then return l + clen < self.wrap end + return true --- TODO + end +end + +---------------------------------------------------------------- +--- Accumulate an indented newline. +---------------------------------------------------------------- +function M:nl() + local ind = self.indent_step:rep(self.current_indent) + self:acc("\n" .. ind) + self._line_len = string.len(ind) + self._lines = self._lines + 1 +end + +---------------------------------------------------------------- +--- Increase indentation and accumulate a new line. +---------------------------------------------------------------- +function M:nlindent() + self.current_indent = self.current_indent + 1 + self:nl() +end + +---------------------------------------------------------------- +--- Decrease indentation and accumulate a new line. +---------------------------------------------------------------- +function M:nldedent() + self.current_indent = self.current_indent - 1 + self:nl() +end + +---------------------------------------------------------------- +--- Insert a new line indented differently. Default is one deeper. +--- @param extra integer? +---------------------------------------------------------------- +function M:nltempindent(extra) + local add = extra or 1 + local depth = self.current_indent + self.current_indent = depth + add + self:nl() + self.current_indent = depth +end + +---------------------------------------------------------------- +--- Reset last non-empty source line to the current position +--- @param pos position|{line: integer}|{l: integer} +---------------------------------------------------------------- +function M:emptyline_gap_reset(pos) + local line = pos and (pos.line or pos.l) + -- never go backwards + if line and line > self._last_nonempty_src_line then + self._last_nonempty_src_line = line + end +end + +---------------------------------------------------------------- +--- Detect the length of emptyline sequence in the source, +--- that precedes the current position +--- @param pos position|{line: integer}|{l: integer} +--- @return integer +---------------------------------------------------------------- +function M:emptyline_gap_before(pos) + local line = pos and (pos.line or pos.l) + if line and line > self._last_nonempty_src_line then + return line - self._last_nonempty_src_line + end + return 0 +end + +---------------------------------------------------------------- +--- Keywords, which are illegal as identifiers. +---------------------------------------------------------------- +local keywords_list = { + "and", + "break", + "do", + "else", + "elseif", + "end", + "false", + "for", + "function", + "if", + "in", + "local", + "nil", + "not", + "or", + "repeat", + "return", + "then", + "true", + "until", + "while", +} +local keywords = {} +for _, kw in pairs(keywords_list) do + keywords[kw] = true +end + +---------------------------------------------------------------- +--- Return true iff string `id' is a legal identifier name. +---------------------------------------------------------------- +local function is_ident(id) + -- HACK: + if type(id) ~= "string" then + return false + end + return + string["match"](id, "^[%a_][%w_]*$") and not keywords[id] +end + +---------------------------------------------------------------- +--- Return true iff ast represents a legal function name for +--- syntax sugar ``function foo.bar.gnat() ... end'': +--- a series of nested string indexes, with an identifier as +--- the innermost node. +--- @param ast table +--- @return boolean +---------------------------------------------------------------- +local function is_idx_stack(ast) + local tag = ast.tag + if tag == "Index" then + if ast[2] then + return ast[2].tag == "Id" or ast[2].tag == "String" + end + return is_idx_stack(ast[1]) + elseif tag == "Id" then + return true + else + return false + end +end + +---------------------------------------------------------------- +--- Returns the length of a call/index chain +--- @param ast table +--- @param depth integer? +--- @return integer +---------------------------------------------------------------- +local function is_call_chain(ast, depth) + local d = depth or 0 + local tag = ast.tag + if tag == "Index" or tag == "Invoke" then + return is_call_chain(ast[1], d + 1) + elseif tag == "Id" then + return d + 1 + else + return d + end +end + +---------------------------------------------------------------- +--- Operator precedences, in increasing order. +--- This is not directly used, it's used to generate op_prec below. +---------------------------------------------------------------- +local op_preprec = { + { "or", "and" }, + { "lt", "le", "eq", "ne" }, + { "concat" }, + { "add", "sub" }, + { "mul", "div", "mod" }, + { "unm", "unary", "not", "len" }, ---TODO: + { "pow" }, + { "index" }, +} + +---------------------------------------------------------------- +--- operator --> precedence table, generated from op_preprec. +---------------------------------------------------------------- +local op_prec = {} + +for prec, ops in ipairs(op_preprec) do + for _, op in ipairs(ops) do + op_prec[op] = prec + end +end + +---------------------------------------------------------------- +--- operator --> source representation. +---------------------------------------------------------------- +local op_symbol = { + add = " + ", + sub = " - ", + mul = " * ", + div = " / ", + mod = " % ", + pow = " ^ ", + concat = " .. ", + eq = " == ", + ne = " ~= ", + lt = " < ", + le = " <= ", + ["and"] = " and ", + ["or"] = " or ", + ["not"] = "not ", + len = "#", + unm = "-", +} + +---------------------------------------------------------------- +--- boolean operators that tie compound conditions together +---------------------------------------------------------------- +local op_cond = { ["and"] = true, ["or"] = true } +---------------------------------------------------------------- +--- right-binding associative operators +---------------------------------------------------------------- +local op_infixr_assoc = { concat = true } +---------------------------------------------------------------- +--- commutative operators +---------------------------------------------------------------- +local op_comm = { + add = true, + mul = true, + ["and"] = true, + ["or"] = true, +} + +--- @alias CommentPos 'first'|'last' +---------------------------------------------------------------- +--- Extract comments from AST +--- @param node token +--- @return table +---------------------------------------------------------------- +function M:extract_comments(node) + if not node.lineinfo then return {} end + local lfi = node.lineinfo.first + local lla = node.lineinfo.last + local comments = {} + + --- @param c table + --- @param pos CommentPos + local function add_comment(c, pos) + local idf = c.lineinfo.first.id + local idl = c.lineinfo.last.id + local line = c.lineinfo.first.line + --- the same comment might get picked up both as preceding + --- the next expression and succeeding the previous one; + --- chunk and statement nodes may also reference it with + --- different lexeme ids. + local present = self.comment_ids[idf] or self.comment_ids[idl] + or self.comment_lines[line] + if not present then + local comment_text = c[1] + --- [[]] comments get parsed into two lexemes + --- (the second is empty) + local has_next = c[2] + local cfi = c.lineinfo.first + local cla = c.lineinfo.last + local cfirst = { l = cfi.line, c = cfi.column } + local clast = { l = cla.line, c = cla.column } + --- if the number of lines in the text is less than the + --- apparent positions, add the newline back + local n_l = #(string.lines(comment_text)) + local l_d = cla.line - cfi.line + local newline = (n_l ~= 0 and n_l == l_d) + local li = { + idf = idf, + idl = idl, + first = cfirst, + last = clast, + text = comment_text, + multiline = has_next, + position = pos, + prepend_newline = newline + } + self.comment_ids[idf] = true + self.comment_ids[idl] = true + self.comment_lines[line] = true + table.insert(comments, li) + end + end + if lfi.comments then + for _, c in ipairs(lfi.comments) do + add_comment(c, 'first') + end + end + if lla.comments then + for _, c in ipairs(lla.comments) do + add_comment(c, 'last') + end + end + + return comments +end + +---------------------------------------------------------------- +--- Accumulate the source representation of AST `node' in +--- the synthetizer. Most of the work is done by delegating to +--- the method having the name of the AST tag. +--- If something can't be converted to normal sources, it's +--- instead dumped as a `-{ ... }' splice in the source accumulator. +--- @param node token +---------------------------------------------------------------- +function M:node(node) + assert(self ~= M and self._acc) + if node == nil then + self:acc("<>") + return + end + local comments = self:extract_comments(node) + --- @param pos 'first'|'last' + local function show_comments(pos) + --- to avoid double dipping, only show corresponding + for _, co in pairs(comments) do + if co.position == pos then + --- comes _after_ a previous expression + if co.position == 'last' then self:nl() end + --- if there was a 'gap' preceding the comment, + --- we preserve it by emitting exactly one empty line + if co.position == 'first' then + if self:emptyline_gap_before(co.first) >= 2 then + self:nl() + end + end + --- preserve existing newlines + local lines = string.lines(co.text) + if co.multiline then + --- multliine comment (--[[]]) + if co.prepend_newline then + table.insert(lines, 1, '') + end + local l1 = lines[1] or '' + if #lines == 1 then + lines[1] = '--[[' .. l1 .. ']]' + else + local llast = lines[#lines] or '' + lines[1] = '--[[' .. l1 + lines[#lines] = llast .. ']]' + end + local wrapped = string.wrap_array(lines, self.wrap) + for i, l in ipairs(wrapped) do + self:acc(l) + if i ~= #wrapped then self:nl() end + end + else + local ls = co.first.l + local le = co.last.l + local wrapped = + string.wrap_array(lines, self.wrap - 3) + if ls == le then + --- (originally) single line comment + if co.text == '' then + self:acc('--') + else + for i, l in ipairs(wrapped) do + local first = string.sub(l, 1, 1) + local pre = '--' + --- add a space if not present already + --- do not break up '---'-style comments + if i == 1 and first == ' ' or first == '-' + --- in this case, only for the first line + then + else + pre = pre .. ' ' + end + self:acc(pre .. l) + if i ~= #wrapped then self:nl() end + end + end + else + --- multiple single line comments + for i, l in ipairs(wrapped) do + local first = string.sub(l, 1, 1) + local pre = '--' + --- add a space if not present already + --- do not break up '---'-style comments + if first == ' ' or first == '-' + then + else + pre = pre .. ' ' + end + self:acc(pre .. l) + if i ~= #wrapped then self:nl() end + end + end + end + --- comes _before_ the next expression + if co.position == 'first' then self:nl() end + --- advance non-empty source line tracker to this comment's last line + self:emptyline_gap_reset(co.last) + end + end + end + + show_comments('first') + -- advance non-empty source line tracker, + -- used for detecting emptyline gaps in the source + if node.lineinfo and node.lineinfo.first then + self:emptyline_gap_reset(node.lineinfo.first) + end + if not node.tag then --- tagless block. + self:list(node, self.nl) + else + local f = M[node.tag] + if type(f) == "function" then --- Delegate to tag method. + f(self, node, unpack(node)) + elseif type(f) == "string" then --- tag string. + self:acc(f) + else + --- No appropriate method, fall back to splice dumping. + --- This cannot happen in a plain Lua AST. + self:acc(" -{ ") + self:acc(pp.tostring(node, + { metalua_tag = 1, hide_hash = 1, line_max = 80 })) + self:acc(" }") + end + end + if node.lineinfo and node.lineinfo.last then + self:emptyline_gap_reset(node.lineinfo.last) + end + show_comments('last') +end + +---------------------------------------------------------------- +--- Convert every node in the AST list passed as 1st arg. +--- @param list table +--- @param sep string|function +--- Optional separator to be accumulated between each list +--- element, it can be a string or a synth method. +--- @param start integer? +--- Optional number (default == 1), indicating which is the +--- first element of list to be converted, so that we can skip +--- the begining of a list. +---------------------------------------------------------------- +function M:list(list, sep, start) + for i = start or 1, #list do + self:node(list[i]) + if list[i + 1] then + if not sep then + elseif type(sep) == "function" then + sep(self) + elseif type(sep) == "string" then + self:acc(sep) + else + error("Invalid list separator") + end + end + end +end + +---------------------------------------------------------------- +--- M:list() with line wrapping +--- @param list table +--- @param sep string|function +--- @param start integer? +--- @param split 'single'|'all'? +---------------------------------------------------------------- +function M:wrapped_list(list, sep, start, split) + local function prerender_list() + local s = '' + for i = start or 1, #list do + s = s .. self:prerender(list[i]) + if list[i + 1] then + if not sep then + --- TODO + -- elseif type(sep) == "function" then + -- sep(self) + elseif type(sep) == "string" then + s = s .. sep + else + error("Invalid list separator") + end + end + end + return s + end + local split_type = split or 'single' + local pre = prerender_list() + if self:fits(pre) then + self:list(list, sep, start) + return + end + if split_type == 'all' then + self:nlindent() + local newsep = function(self) + if type(sep) == "string" then + self:acc(string.normalize(sep)) + else + sep(self) + end + self:nl() + end + self:list(list, newsep, start) + self:nldedent() + else + self:list(list, sep, start) + --- TODO + -- local midpoint = math.ceil(#list / 2) + -- local starter = {} + -- for i = 1, midpoint do + -- table.insert(starter, list[i]) + -- end + -- print('m', midpoint) + -- self:wrapped_list(starter, sep) + -- self:nltempindent() + -- self:wrapped_list(list, sep, midpoint + 1) + end +end + +---------------------------------------------------------------- +--- +--- Tag methods. +--- ------------ +--- +--- Specific AST node dumping methods, associated to their node +--- kinds by their name, which is the corresponding AST tag. +--- synth:node() is in charge of delegating a node's treatment +--- to the appropriate tag method. +--- +--- Such tag methods are called with the AST node as 1st arg. +--- As a convenience, the n node's children are passed as +--- args #2 ... n+1. +--- +--- There are several things that could be refactored into +--- common subroutines here: statement blocks dumping, +--- function dumping... +--- However, given their small size and linear execution +--- (they basically perform series of :acc(), :node(), :list(), +--- :nl*() calls), it seems more readable +--- to avoid multiplication of such tiny functions. +--- +--- To make sense out of these, you need to know metalua's AST +--- syntax, as found in the reference manual or in +--- metalua/doc/ast.txt. +---------------------------------------------------------------- + +function M:Do(node) + self:acc("do") + self:nlindent() + self:list(node, self.nl) + self:nldedent() + self:acc("end") +end + +function M:Set(node) + local lhs = node[1] + local rhs = node[2] + if + --- LHS = { `Index{ lhs, `String{ method } } }, + --- { `Index{ lhs[1][1], `String{ lhs[1][2][1] } } }, + --- RHS = { `Function{ { `Id "self", ... } == + --- params, body } } } + --- { `Function{ { `Id "rhs[1][1][1][1]", ... } == + --- params, body } } } + lhs[1].tag == "Index" + and rhs[1].tag == "Function" + and rhs[1][1][1] and rhs[1][1][1][1] == "self" + and is_idx_stack(lhs[1][1]) + and is_ident(lhs[1][2][1]) + then + --- block 1 + --- `function foo:bar(...) ... end` --- + local method = lhs[1][2][1] + local params = rhs[1][1] + local body = rhs[1][2] + self:acc("function ") + self:node(lhs[1][1]) + self:acc(":") + self:acc(method) + self:acc("(") + self:wrapped_list(params, ", ", 2, 'all') + self:acc(")") + self:nlindent() + self:list(body, self.nl) + self:nldedent() + self:acc("end") + elseif rhs[1].tag == "Function" + and is_idx_stack(lhs[1]) + then + --- block 2 + --- `function foo(...) ... end` --- + local params = rhs[1][1] + local body = rhs[1][2] + self:acc("function ") + self:node(lhs[1]) + self:acc("(") + self:wrapped_list(params, ", ", nil, 'all') + self:acc(")") + self:nlindent() + self:node(body) + self:nldedent() + self:acc("end") + --- metalua extensions + --[[ elseif rhs[1].tag == "Id" + and not is_ident(lhs[1][2][1]) + then + --- block 3 + --- `foo, ... = ...` when foo is *not* a valid id. + --- In that case, the spliced 1st variable must get parentheses, + --- to be distinguished from a statement splice. + --- This cannot happen in a plain Lua AST. + self:list(lhs, ", ") + self:acc(" = ") + self:list(rhs, ", ") + elseif node[3] then + --- block 5 + --- `... = ...`, no syntax sugar, annotation --- + local annot = node[3] + local n = #lhs + for i = 1, n do + local ell, a = lhs[i], annot[i] + self:node(ell) + if a then + self:acc ' #' + self:node(a) + end + if i ~= n then self:acc ', ' end + end + self:acc " = " + self:list(rhs, ", ")--]] + else + --- block 4 + --- `... = ...`, no syntax sugar --- + self:wrapped_list(lhs, ", ") + self:acc(" = ") + self:wrapped_list(rhs, ", ") + end +end + +function M:While(_, cond, body) + self:acc("while ") + self:node(cond) + self:acc(" do") + self:nlindent() + self:list(body, self.nl) + self:nldedent() + self:acc("end") +end + +function M:Repeat(_, body, cond) + self:acc("repeat") + self:nlindent() + self:list(body, self.nl) + self:nldedent() + self:acc("until ") + self:node(cond) +end + +function M:If(node) + for i = 1, #node - 1, 2 do + --- for each ``if/then'' and ``elseif/then'' pair -- + local cond, body = node[i], node[i + 1] + self:acc(i == 1 and "if " or "elseif ") + local lc = self._lines + self:node(cond) + local ml = self._lines > lc + if ml then + self:nl() + self:acc("then") + else + self:acc(" then") + end + self:nlindent() + self:list(body, self.nl) + self:nldedent() + end + --- odd number of children --> last one is an `else' clause -- + if #node % 2 == 1 then + self:acc("else") + self:nlindent() + self:list(node[#node], self.nl) + self:nldedent() + end + self:acc("end") +end + +function M:Fornum(node, var, first, last) + local body = node[#node] + self:acc("for ") + self:node(var) + self:acc(" = ") + self:node(first) + self:acc(", ") + self:node(last) + if #node == 5 then --- 5 children --> + --- child #4 is a step increment. + self:acc(", ") + self:node(node[4]) + end + self:acc(" do") + self:nlindent() + self:list(body, self.nl) + self:nldedent() + self:acc("end") +end + +function M:Forin(_, vars, generators, body) + self:acc("for ") + self:list(vars, ", ") + self:acc(" in ") + self:list(generators, ", ") + self:acc(" do") + self:nlindent() + self:list(body, self.nl) + self:nldedent() + self:acc("end") +end + +function M:Local(node, lhs, rhs, annots) + if next(lhs) then + self:acc("local ") + if annots then + local n = #lhs + for i = 1, n do + self:node(lhs) + local a = annots[i] + if a then + self:acc(" #") + self:node(a) + end + if i ~= n then + self:acc(", ") + end + end + else + self:wrapped_list(lhs, ", ") + end + if rhs[1] then + self:acc(" = ") + self:wrapped_list(rhs, ", ") + end + else + --- Can't create a local statement with 0 variables in + --- plain Lua + self:acc(pp.tostring(node, "nohash")) + end +end + +function M:Localrec(_, lhs, rhs) + --- ``local function name() ... end'' -- + self:acc("local function ") + self:acc(lhs[1][1]) + self:acc("(") + self:wrapped_list(rhs[1][1], ", ", nil, 'all') + self:acc(")") + self:nlindent() + self:list(rhs[1][2], self.nl) + self:nldedent() + self:acc("end") + -- + -- | _ -> + -- -- Other localrec are unprintable ==> splice them -- + -- -- This cannot happen in a plain Lua AST. -- + -- self:acc "-{ " + -- self:acc (table.tostring (node, 'nohash', 80)) + -- self:acc " }" + -- end +end + +function M:Call(node, f) + self:node(f) + self:acc("(") + self:wrapped_list(node, ", ", 2, 'all') --- skip `f'. + self:acc(")") +end + +function M:Invoke(node, f, method) + if node[1].tag == 'String' then + self:acc("(") + self:node(f) + self:acc(")") + else + self:node(f) + end + self:acc(":") + self:acc(method[1]) + self:acc("(") + --- Skip args #1 and #2, object and method name. + self:wrapped_list(node, ", ", 3, 'all') + self:acc(")") +end + +function M:Return(node) + self:acc("return ") + self:wrapped_list(node, ", ") +end + +M.Break = "break" +M.Nil = "nil" +M.False = "false" +M.True = "true" +M.Dots = "..." + +function M:Number(_, n) + self:acc(tostring(n)) +end + +function M:String(_, str) + local fl = string.len('"" ..' .. self.indent_step) + local wl = self.wrap - fl + local multiline = string.ulen(str) > wl + local rendered = '' + --- format "%q" prints '\n' in an umpractical way IMO, + --- so this is fixed with the :gsub( ) call. + if multiline then + --- split the raw text + local split = string.lines(str) or {} + --- add newline placeholders + for i, v in ipairs(split) do + if i ~= #split then + split[i] = v .. '\\n' + end + end + --- wrap + local ls = string.wrap_array(split, wl) + for i, v in ipairs(ls) do + rendered = rendered .. "\n" .. self.indent_step + rendered = rendered .. + string.format("%q", v):gsub("\\\\", [[\]]) + if i ~= #ls then + rendered = rendered .. ' ..' + end + end + else + rendered = string.format("%q", str):gsub("\\\n", [[\n]]) + end + self:acc(rendered) +end + +function M:Function(_, params, body, annots) + self:acc("function(") + if annots then + local n = #params + for i = 1, n do + local p, a = params[i], annots[i] + self:node(p) + if annots then + self:acc(" #") + self:node(a) + end + if i ~= n then + self:acc(", ") + end + end + else + self:wrapped_list(params, ", ", nil, 'all') + end + self:acc(")") + self:nlindent() + self:list(body, self.nl) + self:nldedent() + self:acc("end") +end + +function M:Table(node) + if not node[1] then + self:acc("{ }") + else + self:acc("{") + local newline = + #node > 1 or + (node[1].tag == "Pair" and node[1][2].tag == "Function") + if newline then + self:nlindent() + else + self:acc(" ") + end + for i, elem in ipairs(node) do + if elem.tag == "Pair" then + if elem[1].tag == "String" and is_ident(elem[1][1]) then + --- `Pair{ `String{ key }, value } + self:acc(elem[1][1]) + self:acc(" = ") + self:node(elem[2]) + else + --- `Pair{ key, value } + self:acc("[") + self:node(elem[1]) + self:acc("] = ") + self:node(elem[2]) + end + else + self:node(elem) + end + if node[i + 1] then + self:acc(",") + self:nl() + end + end + if newline then + self:nldedent() + else + self:acc(" ") + end + self:acc("}") + end +end + +function M:Op(node, op, a, b) + --- Transform ``not (a == b)'' into ``a ~= b''. -- + if op == "not" then + if node[2][1] == "eq" then + op, a, b = "ne", node[2][2], node[2][3] + end + if (node[2].tag == "Paren" and node[2][1][1] == "eq") + then + op, a, b = "ne", node[2][1][2], node[2][1][3] + end + end + + if b then --- binary operator. + local left_paren, right_paren = false, false + if a.tag == "Op" + and op_prec[op] >= op_prec[a[1]] + and not op_comm[a[1]] + then + left_paren = true + end + if b.tag == "Op" + and op_prec[op] >= op_prec[b[1]] + and not op_infixr_assoc[b[1]] + then + right_paren = true + end + + self:acc(left_paren and "(" or '') + self:node(a) + self:acc(left_paren and ")" or '') + + if op_cond[op] then + local pre = op_symbol[op] .. self:prerender(b) + if not self:fits(pre) + or ( + type(a[2]) == "table" + and type(b[2]) == "table" + and (a[2].lineinfo and b[2].lineinfo + and a[2].lineinfo.first and b[2].lineinfo.first + and a[2].lineinfo.first.line < + b[2].lineinfo.first.line) + ) + then + self:nltempindent(2) + end + end + self:acc(op_symbol[op]) + + self:acc(right_paren and "(" or '') + self:node(b) + self:acc(right_paren and ")" or '') + else --- unary operator. + local paren = false + if a.tag == "Op" then + paren = op_prec[op] >= op_prec[a[1]] + end + if op == 'len' and is_call_chain(a) > 1 then + paren = true + end + self:acc(op_symbol[op]) + self:acc(paren and "(" or '') + self:node(a) + self:acc(paren and ")" or '') + end +end + +function M:Paren(_, content) + local pre = "(" .. self:prerender(content) .. ")" + if not self:fits(pre) then + self:nltempindent() + end + self:acc("(") + self:node(content) + self:acc(")") +end + +function M:Index(_, table, key) + local paren_table + if table.tag == "Op" + and op_prec[table[1][1]] < op_prec.index + then + paren_table = true + else + paren_table = false + end + + self:acc(paren_table and "(" or "") + self:node(table) + self:acc(paren_table and ")" or "") + + if key.tag == 'String' and is_ident(key[1]) then + --- ``table [key]'' + self:acc(".") + self:acc(key[1]) + else + --- ``table.key'' + self:acc("[") + self:node(key) + self:acc("]") + end +end + +function M:Id(node, name) + if is_ident(name) then + self:acc(name) + else + --- Unprintable identifier, fall back to splice repr. + --- This cannot happen in a plain Lua AST. + self:acc("-{`Id ") + self:String(node, name) + self:acc("}") + end +end + +-- M.TDyn = '*' +-- M.TDynbar = '**' +-- M.TPass = 'pass' +-- M.TField = 'field' +-- M.TIdbar = M.TId +-- M.TReturn = M.Return +-- +-- +-- function M:TId (node, name) self:acc(name) end +-- +-- +-- function M:TCatbar(node, te, tebar) +-- self:acc'(' +-- self:node(te) +-- self:acc'|' +-- self:tebar(tebar) +-- self:acc')' +-- end +-- +-- function M:TFunction(node, p, r) +-- self:tebar(p) +-- self:acc '->' +-- self:tebar(r) +-- end +-- +-- function M:TTable (node, default, pairs) +-- self:acc '[' +-- self:list (pairs, ', ') +-- if default.tag~='TField' then +-- self:acc '|' +-- self:node (default) +-- end +-- self:acc ']' +-- end +-- +-- function M:TPair (node, k, v) +-- self:node (k) +-- self:acc '=' +-- self:node (v) +-- end +-- +-- function M:TIdbar (node, name) +-- self :acc (name) +-- end +-- +-- function M:TCatbar (node, a, b) +-- self:node(a) +-- self:acc ' ++ ' +-- self:node(b) +-- end +-- +-- function M:tebar(node) +-- if node.tag then self:node(node) else +-- self:acc '(' +-- self:list(node, ', ') +-- self:acc ')' +-- end +-- end +-- +-- function M:TUnkbar(node, name) +-- self:acc '~~' +-- self:acc (name) +-- end +-- +-- function M:TUnk(node, name) +-- self:acc '~' +-- self:acc (name) +-- end +-- +-- for name, tag in pairs{ const='TConst', var='TVar', currently='TCurrently', just='TJust' } do +-- M[tag] = function(self, node, te) +-- self:acc (name..' ') +-- self:node(te) +-- end +-- end + +-- print(M.run(+{stat: local function add(a, b) +-- local c = a + b; return add(a,c) end})) + +M.oneshot = function(x, ...) + local a2s = M.new(...) + return a2s:run(x) +end +return M diff --git a/src/lib/metalua/metalua/compiler/parser.lua b/src/lib/metalua/metalua/compiler/parser.lua new file mode 100644 index 000000000..5b1ba8e0b --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser.lua @@ -0,0 +1,35 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +-- Export all public APIs from sub-modules, squashed into a flat spacename + +local MT = { __type='metalua.compiler.parser' } + +local MODULE_REL_NAMES = { "annot.grammar", "expr", "meta", "misc", + "stat", "table", "ext" } + +local function new() + local M = { + lexer = require "metalua.compiler.parser.lexer" (); + extensions = { } } + for _, rel_name in ipairs(MODULE_REL_NAMES) do + local abs_name = "metalua.compiler.parser."..rel_name + local extender = require (abs_name) + if not M.extensions[abs_name] then + if type (extender) == 'function' then extender(M) end + M.extensions[abs_name] = extender + end + end + return setmetatable(M, MT) +end + +return { new = new } diff --git a/src/lib/metalua/metalua/compiler/parser/annot/generator.lua b/src/lib/metalua/metalua/compiler/parser/annot/generator.lua new file mode 100644 index 000000000..b83fd3078 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/annot/generator.lua @@ -0,0 +1,41 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +require 'checks' +local gg = require 'metalua.grammar.generator' +local M = { } + +function M.opt(mlc, primary, a_type) + checks('table', 'table|function', 'string') + return gg.sequence{ + primary, + gg.onkeyword{ "#", function() return assert(mlc.annot[a_type]) end }, + builder = function(x) + local t, annot = unpack(x) + return annot and { tag='Annot', t, annot } or t + end } +end + +-- split a list of "foo" and "`Annot{foo, annot}" into a list of "foo" +-- and a list of "annot". +-- No annot list is returned if none of the elements were annotated. +function M.split(lst) + local x, a, some = { }, { }, false + for i, p in ipairs(lst) do + if p.tag=='Annot' then + some, x[i], a[i] = true, unpack(p) + else x[i] = p end + end + if some then return x, a else return lst end +end + +return M diff --git a/src/lib/metalua/metalua/compiler/parser/annot/grammar.lua b/src/lib/metalua/metalua/compiler/parser/annot/grammar.lua new file mode 100644 index 000000000..4fe8b7aaf --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/annot/grammar.lua @@ -0,0 +1,105 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +local gg = require 'metalua.grammar.generator' + +return function(M) + local _M = gg.future(M) + M.lexer :add '->' + local A = { } + local _A = gg.future(A) + M.annot = A + + -- Type identifier: Lua keywords such as `"nil"` allowed. + function M.annot.tid(lx) + local w = lx :next() + local t = w.tag + if t=='Keyword' and w[1] :match '^[%a_][%w_]*$' or w.tag=='Id' + then return {tag='TId'; lineinfo=w.lineinfo; w[1]} + else return gg.parse_error (lx, 'tid expected') end + end + + local field_types = { var='TVar'; const='TConst'; + currently='TCurrently'; field='TField' } + + -- TODO check lineinfo + function M.annot.tf(lx) + local tk = lx:next() + local w = tk[1] + local tag = field_types[w] + if not tag then error ('Invalid field type '..w) + elseif tag=='TField' then return {tag='TField'} else + local te = M.te(lx) + return {tag=tag; te} + end + end + + M.annot.tebar_content = gg.list{ + name = 'tebar content', + primary = _A.te, + separators = { ",", ";" }, + terminators = ")" } + + M.annot.tebar = gg.multisequence{ + name = 'annot.tebar', + --{ '*', builder = 'TDynbar' }, -- maybe not user-available + { '(', _A.tebar_content, ')', + builder = function(x) return x[1] end }, + { _A.te } + } + + M.annot.te = gg.multisequence{ + name = 'annot.te', + { _A.tid, builder=function(x) return x[1] end }, + { '*', builder = 'TDyn' }, + { "[", + gg.list{ + primary = gg.sequence{ + _M.expr, "=", _A.tf, + builder = 'TPair' + }, + separators = { ",", ";" }, + terminators = { "]", "|" } }, + gg.onkeyword{ "|", _A.tf }, + "]", + builder = function(x) + local fields, other = unpack(x) + return { tag='TTable', other or {tag='TField'}, fields } + end }, -- "[ ... ]" + { '(', _A.tebar_content, ')', '->', '(', _A.tebar_content, ')', + builder = function(x) + local p, r = unpack(x) + return {tag='TFunction', p, r } + end } } + + M.annot.ts = gg.multisequence{ + name = 'annot.ts', + { 'return', _A.tebar_content, builder='TReturn' }, + { _A.tid, builder = function(x) + if x[1][1]=='pass' then return {tag='TPass'} + else error "Bad statement type" end + end } } + +-- TODO: add parsers for statements: +-- #return tebar +-- #alias = te +-- #ell = tf +--[[ + M.annot.stat_annot = gg.sequence{ + gg.list{ primary=_A.tid, separators='.' }, + '=', + XXX??, + builder = 'Annot' } +--]] + + return M.annot +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/expr.lua b/src/lib/metalua/metalua/compiler/parser/expr.lua new file mode 100644 index 000000000..1a3192959 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/expr.lua @@ -0,0 +1,199 @@ +------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +-- +-- Exported API: +-- * [mlp.expr()] +-- * [mlp.expr_list()] +-- * [mlp.func_val()] +-- +------------------------------------------------------------------------------- + +local pp = require 'metalua.pprint' +local gg = require 'metalua.grammar.generator' +local annot = require 'metalua.compiler.parser.annot.generator' + +return function(M) + local _M = gg.future(M) + local _table = gg.future(M, 'table') + local _meta = gg.future(M, 'meta') -- TODO move to ext? + local _annot = gg.future(M, 'annot') -- TODO move to annot + + -------------------------------------------------------------------------------- + -- Non-empty expression list. Actually, this isn't used here, but that's + -- handy to give to users. + -------------------------------------------------------------------------------- + M.expr_list = gg.list{ primary=_M.expr, separators="," } + + -------------------------------------------------------------------------------- + -- Helpers for function applications / method applications + -------------------------------------------------------------------------------- + M.func_args_content = gg.list{ + name = "function arguments", + primary = _M.expr, + separators = ",", + terminators = ")" } + + -- Used to parse methods + M.method_args = gg.multisequence{ + name = "function argument(s)", + { "{", _table.content, "}" }, + { "(", _M.func_args_content, ")", builder = unpack }, + { "+{", _meta.quote_content, "}" }, + -- TODO lineinfo? + function(lx) local r = M.opt_string(lx); return r and {r} or { } end } + + -------------------------------------------------------------------------------- + -- [func_val] parses a function, from opening parameters parenthese to + -- "end" keyword included. Used for anonymous functions as well as + -- function declaration statements (both local and global). + -- + -- It's wrapped in a [_func_val] eta expansion, so that when expr + -- parser uses the latter, they will notice updates of [func_val] + -- definitions. + -------------------------------------------------------------------------------- + M.func_params_content = gg.list{ + name="function parameters", + gg.multisequence{ { "...", builder = "Dots" }, annot.opt(M, _M.id, 'te') }, + separators = ",", terminators = {")", "|"} } + + -- TODO move to annot + M.func_val = gg.sequence{ + name = "function body", + "(", _M.func_params_content, ")", _M.block, "end", + builder = function(x) + local params, body = unpack(x) + local annots, some = { }, false + for i, p in ipairs(params) do + if p.tag=='Annot' then + params[i], annots[i], some = p[1], p[2], true + else annots[i] = false end + end + if some then return { tag='Function', params, body, annots } + else return { tag='Function', params, body } end + end } + + local func_val = function(lx) return M.func_val(lx) end + + -------------------------------------------------------------------------------- + -- Default parser for primary expressions + -------------------------------------------------------------------------------- + function M.id_or_literal (lx) + local a = lx:next() + if a.tag~="Id" and a.tag~="String" and a.tag~="Number" then + local msg + if a.tag=='Eof' then + msg = "End of file reached when an expression was expected" + elseif a.tag=='Keyword' then + msg = "An expression was expected, and `"..a[1].. + "' can't start an expression" + else + msg = "Unexpected expr token " .. pp.tostring (a) + end + gg.parse_error (lx, msg) + end + return a + end + + + -------------------------------------------------------------------------------- + -- Builder generator for operators. Wouldn't be worth it if "|x|" notation + -- were allowed, but then lua 5.1 wouldn't compile it + -------------------------------------------------------------------------------- + + -- opf1 = |op| |_,a| `Op{ op, a } + local function opf1 (op) return + function (_,a) return { tag="Op", op, a } end end + + -- opf2 = |op| |a,_,b| `Op{ op, a, b } + local function opf2 (op) return + function (a,_,b) return { tag="Op", op, a, b } end end + + -- opf2r = |op| |a,_,b| `Op{ op, b, a } -- (args reversed) + local function opf2r (op) return + function (a,_,b) return { tag="Op", op, b, a } end end + + local function op_ne(a, _, b) + -- This version allows to remove the "ne" operator from the AST definition. + -- However, it doesn't always produce the exact same bytecode as Lua 5.1. + return { tag="Op", "not", + { tag="Op", "eq", a, b, lineinfo= { + first = a.lineinfo.first, last = b.lineinfo.last } } } + end + + + -------------------------------------------------------------------------------- + -- + -- complete expression + -- + -------------------------------------------------------------------------------- + + -- FIXME: set line number. In [expr] transformers probably + M.expr = gg.expr { + name = "expression", + primary = gg.multisequence{ + name = "expr primary", + { "(", _M.expr, ")", builder = "Paren" }, + { "function", _M.func_val, builder = unpack }, + { "-{", _meta.splice_content, "}", builder = unpack }, + { "+{", _meta.quote_content, "}", builder = unpack }, + { "nil", builder = "Nil" }, + { "true", builder = "True" }, + { "false", builder = "False" }, + { "...", builder = "Dots" }, + { "{", _table.content, "}", builder = unpack }, + _M.id_or_literal }, + + infix = { + name = "expr infix op", + { "+", prec = 60, builder = opf2 "add" }, + { "-", prec = 60, builder = opf2 "sub" }, + { "*", prec = 70, builder = opf2 "mul" }, + { "/", prec = 70, builder = opf2 "div" }, + { "%", prec = 70, builder = opf2 "mod" }, + { "^", prec = 90, builder = opf2 "pow", assoc = "right" }, + { "..", prec = 40, builder = opf2 "concat", assoc = "right" }, + { "==", prec = 30, builder = opf2 "eq" }, + { "~=", prec = 30, builder = op_ne }, + { "<", prec = 30, builder = opf2 "lt" }, + { "<=", prec = 30, builder = opf2 "le" }, + { ">", prec = 30, builder = opf2r "lt" }, + { ">=", prec = 30, builder = opf2r "le" }, + { "and",prec = 20, builder = opf2 "and" }, + { "or", prec = 10, builder = opf2 "or" } }, + + prefix = { + name = "expr prefix op", + { "not", prec = 80, builder = opf1 "not" }, + { "#", prec = 80, builder = opf1 "len" }, + { "-", prec = 80, builder = opf1 "unm" } }, + + suffix = { + name = "expr suffix op", + { "[", _M.expr, "]", builder = function (tab, idx) + return {tag="Index", tab, idx[1]} end}, + { ".", _M.id, builder = function (tab, field) + return {tag="Index", tab, _M.id2string(field[1])} end }, + { "(", _M.func_args_content, ")", builder = function(f, args) + return {tag="Call", f, unpack(args[1])} end }, + { "{", _table.content, "}", builder = function (f, arg) + return {tag="Call", f, arg[1]} end}, + { ":", _M.id, _M.method_args, builder = function (obj, post) + local m_name, args = unpack(post) + return {tag="Invoke", obj, _M.id2string(m_name), unpack(args)} end}, + { "+{", _meta.quote_content, "}", builder = function (f, arg) + return {tag="Call", f, arg[1] } end }, + default = { name="opt_string_arg", parse = _M.opt_string, builder = function(f, arg) + return {tag="Call", f, arg } end } } } + return M +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/ext.lua b/src/lib/metalua/metalua/compiler/parser/ext.lua new file mode 100644 index 000000000..d1d15f577 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/ext.lua @@ -0,0 +1,89 @@ +------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Non-Lua syntax extensions +-- +-------------------------------------------------------------------------------- + +local gg = require 'metalua.grammar.generator' + +return function(M) + + local _M = gg.future(M) + + --------------------------------------------------------------------------- + -- Algebraic Datatypes + ---------------------------------------------------------------------------- + local function adt (lx) + local node = _M.id (lx) + local tagval = node[1] + -- tagkey = `Pair{ `String "key", `String{ -{tagval} } } + local tagkey = { tag="Pair", {tag="String", "tag"}, {tag="String", tagval} } + if lx:peek().tag == "String" or lx:peek().tag == "Number" then + -- TODO support boolean litterals + return { tag="Table", tagkey, lx:next() } + elseif lx:is_keyword (lx:peek(), "{") then + local x = M.table.table (lx) + table.insert (x, 1, tagkey) + return x + else return { tag="Table", tagkey } end + end + + M.adt = gg.sequence{ "`", adt, builder = unpack } + + M.expr.primary :add(M.adt) + + ---------------------------------------------------------------------------- + -- Anonymous lambda + ---------------------------------------------------------------------------- + M.lambda_expr = gg.sequence{ + "|", _M.func_params_content, "|", _M.expr, + builder = function (x) + local li = x[2].lineinfo + return { tag="Function", x[1], + { {tag="Return", x[2], lineinfo=li }, lineinfo=li } } + end } + + M.expr.primary :add (M.lambda_expr) + + -------------------------------------------------------------------------------- + -- Allows to write "a `f` b" instead of "f(a, b)". Taken from Haskell. + -------------------------------------------------------------------------------- + function M.expr_in_backquotes (lx) return M.expr(lx, 35) end -- 35=limited precedence + M.expr.infix :add{ name = "infix function", + "`", _M.expr_in_backquotes, "`", prec = 35, assoc="left", + builder = function(a, op, b) return {tag="Call", op[1], a, b} end } + + -------------------------------------------------------------------------------- + -- C-style op+assignments + -- TODO: no protection against side-effects in LHS vars. + -------------------------------------------------------------------------------- + local function op_assign(kw, op) + local function rhs(a, b) return { tag="Op", op, a, b } end + local function f(a,b) + if #a ~= #b then gg.parse_error "assymetric operator+assignment" end + local right = { } + local r = { tag="Set", a, right } + for i=1, #a do right[i] = { tag="Op", op, a[i], b[i] } end + return r + end + M.lexer :add (kw) + M.assignments[kw] = f + end + + local ops = { add='+='; sub='-='; mul='*='; div='/=' } + for ast_op_name, keyword in pairs(ops) do op_assign(keyword, ast_op_name) end + + return M +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/lexer.lua b/src/lib/metalua/metalua/compiler/parser/lexer.lua new file mode 100644 index 000000000..a577cf4e7 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/lexer.lua @@ -0,0 +1,36 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2014 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +---------------------------------------------------------------------- +-- Generate a new lua-specific lexer, derived from the generic lexer. +---------------------------------------------------------------------- + +local generic_lexer = require 'metalua.grammar.lexer' + +return function() + local lexer = generic_lexer.lexer :clone() + + local keywords = { + "and", "break", "do", "else", "elseif", + "end", "false", "for", "function", + "goto", -- Lua5.2 + "if", + "in", "local", "nil", "not", "or", "repeat", + "return", "then", "true", "until", "while", + "...", "..", "==", ">=", "<=", "~=", + "::", -- Lua5,2 + "+{", "-{" } -- Metalua + + for _, w in ipairs(keywords) do lexer :add (w) end + + return lexer +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/meta.lua b/src/lib/metalua/metalua/compiler/parser/meta.lua new file mode 100644 index 000000000..e4897c31c --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/meta.lua @@ -0,0 +1,131 @@ +------------------------------------------------------------------------------- +-- Copyright (c) 2006-2014 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +------------------------------------------------------------------------------- + +-- Compile-time metaprogramming features: splicing ASTs generated during compilation, +-- AST quasi-quoting helpers. + +local gg = require 'metalua.grammar.generator' + +return function(M) + local _M = gg.future(M) + M.meta={ } + local _MM = gg.future(M.meta) + + -------------------------------------------------------------------------------- + -- External splicing: compile an AST into a chunk, load and evaluate + -- that chunk, and replace the chunk by its result (which must also be + -- an AST). + -------------------------------------------------------------------------------- + + -- TODO: that's not part of the parser + function M.meta.eval (ast) + -- TODO: should there be one mlc per splice, or per parser instance? + local mlc = require 'metalua.compiler'.new() + local f = mlc :ast_to_function (ast, '=splice') + local result=f(M) -- splices act on the current parser + return result + end + + ---------------------------------------------------------------------------- + -- Going from an AST to an AST representing that AST + -- the only hash-part key being lifted is `"tag"`. + -- Doesn't lift subtrees protected inside a `Splice{ ... }. + -- e.g. change `Foo{ 123 } into + -- `Table{ `Pair{ `String "tag", `String "foo" }, `Number 123 } + ---------------------------------------------------------------------------- + local function lift (t) + --print("QUOTING:", table.tostring(t, 60,'nohash')) + local cases = { } + function cases.table (t) + local mt = { tag = "Table" } + --table.insert (mt, { tag = "Pair", quote "quote", { tag = "True" } }) + if t.tag == "Splice" then + assert (#t==1, "Invalid splice") + local sp = t[1] + return sp + elseif t.tag then + table.insert (mt, { tag="Pair", lift "tag", lift(t.tag) }) + end + for _, v in ipairs (t) do + table.insert (mt, lift(v)) + end + return mt + end + function cases.number (t) return { tag = "Number", t, quote = true } end + function cases.string (t) return { tag = "String", t, quote = true } end + function cases.boolean (t) return { tag = t and "True" or "False", t, quote = true } end + local f = cases [type(t)] + if f then return f(t) else error ("Cannot quote an AST containing "..tostring(t)) end + end + M.meta.lift = lift + + -------------------------------------------------------------------------------- + -- when this variable is false, code inside [-{...}] is compiled and + -- avaluated immediately. When it's true (supposedly when we're + -- parsing data inside a quasiquote), [-{foo}] is replaced by + -- [`Splice{foo}], which will be unpacked by [quote()]. + -------------------------------------------------------------------------------- + local in_a_quote = false + + -------------------------------------------------------------------------------- + -- Parse the inside of a "-{ ... }" + -------------------------------------------------------------------------------- + function M.meta.splice_content (lx) + local parser_name = "expr" + if lx:is_keyword (lx:peek(2), ":") then + local a = lx:next() + lx:next() -- skip ":" + assert (a.tag=="Id", "Invalid splice parser name") + parser_name = a[1] + end + -- TODO FIXME running a new parser with the old lexer?! + local parser = require 'metalua.compiler.parser'.new() + local ast = parser [parser_name](lx) + if in_a_quote then -- only prevent quotation in this subtree + --printf("SPLICE_IN_QUOTE:\n%s", _G.table.tostring(ast, "nohash", 60)) + return { tag="Splice", ast } + else -- convert in a block, eval, replace with result + if parser_name == "expr" then ast = { { tag="Return", ast } } + elseif parser_name == "stat" then ast = { ast } + elseif parser_name ~= "block" then + error ("splice content must be an expr, stat or block") end + --printf("EXEC THIS SPLICE:\n%s", _G.table.tostring(ast, "nohash", 60)) + return M.meta.eval (ast) + end + end + + M.meta.splice = gg.sequence{ "-{", _MM.splice_content, "}", builder=unpack } + + -------------------------------------------------------------------------------- + -- Parse the inside of a "+{ ... }" + -------------------------------------------------------------------------------- + function M.meta.quote_content (lx) + local parser + if lx:is_keyword (lx:peek(2), ":") then -- +{parser: content } + local parser_name = M.id(lx)[1] + parser = M[parser_name] + lx:next() -- skip ":" + else -- +{ content } + parser = M.expr + end + + local prev_iq = in_a_quote + in_a_quote = true + --print("IN_A_QUOTE") + local content = parser (lx) + local q_content = M.meta.lift (content) + in_a_quote = prev_iq + return q_content + end + + return M +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/misc.lua b/src/lib/metalua/metalua/compiler/parser/misc.lua new file mode 100644 index 000000000..07b3eceed --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/misc.lua @@ -0,0 +1,168 @@ +------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +-- +-- Summary: metalua parser, miscellaneous utility functions. +-- +------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Exported API: +-- * [mlp.fget()] +-- * [mlp.id()] +-- * [mlp.opt_id()] +-- * [mlp.id_list()] +-- * [mlp.string()] +-- * [mlp.opt_string()] +-- * [mlp.id2string()] +-- +-------------------------------------------------------------------------------- + +local gg = require 'metalua.grammar.generator' + +-- TODO: replace splice-aware versions with naive ones, move etensions in ./meta + +return function(M) + local _M = gg.future(M) + +--[[ metaprog-free versions: + function M.id(lx) + if lx:peek().tag~='Id' then gg.parse_error(lx, "Identifier expected") + else return lx:next() end + end + + function M.opt_id(lx) + if lx:peek().tag~='Id' then return lx:next() else return false end + end + + function M.string(lx) + if lx:peek().tag~='String' then gg.parse_error(lx, "String expected") + else return lx:next() end + end + + function M.opt_string(lx) + if lx:peek().tag~='String' then return lx:next() else return false end + end + + -------------------------------------------------------------------------------- + -- Converts an identifier into a string. Hopefully one day it'll handle + -- splices gracefully, but that proves quite tricky. + -------------------------------------------------------------------------------- + function M.id2string (id) + if id.tag == "Id" then id.tag = "String"; return id + else error ("Identifier expected: "..table.tostring(id, 'nohash')) end + end +--]] + + -------------------------------------------------------------------------------- + -- Try to read an identifier (possibly as a splice), or return [false] if no + -- id is found. + -------------------------------------------------------------------------------- + function M.opt_id (lx) + local a = lx:peek(); + if lx:is_keyword (a, "-{") then + local v = M.meta.splice(lx) + if v.tag ~= "Id" and v.tag ~= "Splice" then + gg.parse_error(lx, "Bad id splice") + end + return v + elseif a.tag == "Id" then return lx:next() + else return false end + end + + -------------------------------------------------------------------------------- + -- Mandatory reading of an id: causes an error if it can't read one. + -------------------------------------------------------------------------------- + function M.id (lx) + return M.opt_id (lx) or gg.parse_error(lx,"Identifier expected") + end + + -------------------------------------------------------------------------------- + -- Common helper function + -------------------------------------------------------------------------------- + M.id_list = gg.list { primary = _M.id, separators = "," } + + -------------------------------------------------------------------------------- + -- Converts an identifier into a string. Hopefully one day it'll handle + -- splices gracefully, but that proves quite tricky. + -------------------------------------------------------------------------------- + function M.id2string (id) + --print("id2string:", disp.ast(id)) + if id.tag == "Id" then id.tag = "String"; return id + elseif id.tag == "Splice" then + error ("id2string on splice not implemented") + -- Evaluating id[1] will produce `Id{ xxx }, + -- and we want it to produce `String{ xxx }. + -- The following is the plain notation of: + -- +{ `String{ `Index{ `Splice{ -{id[1]} }, `Number 1 } } } + return { tag="String", { tag="Index", { tag="Splice", id[1] }, + { tag="Number", 1 } } } + else error ("Identifier expected: "..table.tostring(id, 'nohash')) end + end + + -------------------------------------------------------------------------------- + -- Read a string, possibly spliced, or return an error if it can't + -------------------------------------------------------------------------------- + function M.string (lx) + local a = lx:peek() + if lx:is_keyword (a, "-{") then + local v = M.meta.splice(lx) + if v.tag ~= "String" and v.tag ~= "Splice" then + gg.parse_error(lx,"Bad string splice") + end + return v + elseif a.tag == "String" then return lx:next() + else error "String expected" end + end + + -------------------------------------------------------------------------------- + -- Try to read a string, or return false if it can't. No splice allowed. + -------------------------------------------------------------------------------- + function M.opt_string (lx) + return lx:peek().tag == "String" and lx:next() + end + + -------------------------------------------------------------------------------- + -- Chunk reader: block + Eof + -------------------------------------------------------------------------------- + function M.skip_initial_sharp_comment (lx) + -- Dirty hack: I'm happily fondling lexer's private parts + -- FIXME: redundant with lexer:newstream() + lx :sync() + local i = lx.src:match ("^#.-\n()", lx.i) + if i then + lx.i = i + lx.column_offset = i + lx.line = lx.line and lx.line + 1 or 1 + end + end + + local function chunk (lx) + if lx:peek().tag == 'Eof' then + return { } -- handle empty files + else + M.skip_initial_sharp_comment (lx) + local chunk = M.block (lx) + if lx:peek().tag ~= "Eof" then + gg.parse_error(lx, "End-of-file expected") + end + return chunk + end + end + + -- chunk is wrapped in a sequence so that it has a "transformer" field. + M.chunk = gg.sequence { chunk, builder = unpack } + + return M +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/stat.lua b/src/lib/metalua/metalua/compiler/parser/stat.lua new file mode 100644 index 000000000..53960b442 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/stat.lua @@ -0,0 +1,272 @@ +------------------------------------------------------------------------------ +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +-- +-- Summary: metalua parser, statement/block parser. This is part of the +-- definition of module [mlp]. +-- +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +-- +-- Exports API: +-- * [mlp.stat()] +-- * [mlp.block()] +-- * [mlp.for_header()] +-- +------------------------------------------------------------------------------- + +local lexer = require 'metalua.grammar.lexer' +local gg = require 'metalua.grammar.generator' + +local annot = require 'metalua.compiler.parser.annot.generator' + +-------------------------------------------------------------------------------- +-- List of all keywords that indicate the end of a statement block. Users are +-- likely to extend this list when designing extensions. +-------------------------------------------------------------------------------- + + +return function(M) + local _M = gg.future(M) + + M.block_terminators = { "else", "elseif", "end", "until", ")", "}", "]" } + + -- FIXME: this must be handled from within GG!!! + -- FIXME: there's no :add method in the list anyway. Added by gg.list?! + function M.block_terminators :add(x) + if type (x) == "table" then for _, y in ipairs(x) do self :add (y) end + else table.insert (self, x) end + end + + ---------------------------------------------------------------------------- + -- list of statements, possibly followed by semicolons + ---------------------------------------------------------------------------- + M.block = gg.list { + name = "statements block", + terminators = M.block_terminators, + primary = function (lx) + -- FIXME use gg.optkeyword() + local x = M.stat (lx) + if lx:is_keyword (lx:peek(), ";") then lx:next() end + return x + end } + + ---------------------------------------------------------------------------- + -- Helper function for "return " parsing. + -- Called when parsing return statements. + -- The specific test for initial ";" is because it's not a block terminator, + -- so without it gg.list would choke on "return ;" statements. + -- We don't make a modified copy of block_terminators because this list + -- is sometimes modified at runtime, and the return parser would get out of + -- sync if it was relying on a copy. + ---------------------------------------------------------------------------- + local return_expr_list_parser = gg.multisequence{ + { ";" , builder = function() return { } end }, + default = gg.list { + _M.expr, separators = ",", terminators = M.block_terminators } } + + + local for_vars_list = gg.list{ + name = "for variables list", + primary = _M.id, + separators = ",", + terminators = "in" } + + ---------------------------------------------------------------------------- + -- for header, between [for] and [do] (exclusive). + -- Return the `Forxxx{...} AST, without the body element (the last one). + ---------------------------------------------------------------------------- + function M.for_header (lx) + local vars = M.id_list(lx) + if lx :is_keyword (lx:peek(), "=") then + if #vars ~= 1 then + gg.parse_error (lx, "numeric for only accepts one variable") + end + lx:next() -- skip "=" + local exprs = M.expr_list (lx) + if #exprs < 2 or #exprs > 3 then + gg.parse_error (lx, "numeric for requires 2 or 3 boundaries") + end + return { tag="Fornum", vars[1], unpack (exprs) } + else + if not lx :is_keyword (lx :next(), "in") then + gg.parse_error (lx, '"=" or "in" expected in for loop') + end + local exprs = M.expr_list (lx) + return { tag="Forin", vars, exprs } + end + end + + ---------------------------------------------------------------------------- + -- Function def parser helper: id ( . id ) * + ---------------------------------------------------------------------------- + local function fn_builder (list) + local acc = list[1] + local first = acc.lineinfo.first + for i = 2, #list do + local index = M.id2string(list[i]) + local li = lexer.new_lineinfo(first, index.lineinfo.last) + acc = { tag="Index", acc, index, lineinfo=li } + end + return acc + end + local func_name = gg.list{ _M.id, separators = ".", builder = fn_builder } + + ---------------------------------------------------------------------------- + -- Function def parser helper: ( : id )? + ---------------------------------------------------------------------------- + local method_name = gg.onkeyword{ name = "method invocation", ":", _M.id, + transformers = { function(x) return x and x.tag=='Id' and M.id2string(x) end } } + + ---------------------------------------------------------------------------- + -- Function def builder + ---------------------------------------------------------------------------- + local function funcdef_builder(x) + local name, method, func = unpack(x) + if method then + name = { tag="Index", name, method, + lineinfo = { + first = name.lineinfo.first, + last = method.lineinfo.last } } + table.insert (func[1], 1, {tag="Id", "self"}) + end + local r = { tag="Set", {name}, {func} } + r[1].lineinfo = name.lineinfo + r[2].lineinfo = func.lineinfo + return r + end + + + ---------------------------------------------------------------------------- + -- if statement builder + ---------------------------------------------------------------------------- + local function if_builder (x) + local cond_block_pairs, else_block, r = x[1], x[2], {tag="If"} + local n_pairs = #cond_block_pairs + for i = 1, n_pairs do + local cond, block = unpack(cond_block_pairs[i]) + r[2*i-1], r[2*i] = cond, block + end + if else_block then table.insert(r, #r+1, else_block) end + return r + end + + -------------------------------------------------------------------------------- + -- produce a list of (expr,block) pairs + -------------------------------------------------------------------------------- + local elseifs_parser = gg.list { + gg.sequence { _M.expr, "then", _M.block , name='elseif parser' }, + separators = "elseif", + terminators = { "else", "end" } + } + + local annot_expr = gg.sequence { + _M.expr, + gg.onkeyword{ "#", gg.future(M, 'annot').tf }, + builder = function(x) + local e, a = unpack(x) + if a then return { tag='Annot', e, a } + else return e end + end } + + local annot_expr_list = gg.list { + primary = annot.opt(M, _M.expr, 'tf'), separators = ',' } + + ------------------------------------------------------------------------ + -- assignments and calls: statements that don't start with a keyword + ------------------------------------------------------------------------ + local function assign_or_call_stat_parser (lx) + local e = annot_expr_list (lx) + local a = lx:is_keyword(lx:peek()) + local op = a and M.assignments[a] + -- TODO: refactor annotations + if op then + --FIXME: check that [e] is a LHS + lx :next() + local annots + e, annots = annot.split(e) + local v = M.expr_list (lx) + if type(op)=="string" then return { tag=op, e, v, annots } + else return op (e, v) end + else + assert (#e > 0) + if #e > 1 then + gg.parse_error (lx, + "comma is not a valid statement separator; statement can be ".. + "separated by semicolons, or not separated at all") + elseif e[1].tag ~= "Call" and e[1].tag ~= "Invoke" then + local typename + if e[1].tag == 'Id' then + typename = '("'..e[1][1]..'") is an identifier' + elseif e[1].tag == 'Op' then + typename = "is an arithmetic operation" + else typename = "is of type '"..(e[1].tag or "").."'" end + gg.parse_error (lx, + "This expression %s; ".. + "a statement was expected, and only function and method call ".. + "expressions can be used as statements", typename); + end + return e[1] + end + end + + M.local_stat_parser = gg.multisequence{ + -- local function + { "function", _M.id, _M.func_val, builder = + function(x) + local vars = { x[1], lineinfo = x[1].lineinfo } + local vals = { x[2], lineinfo = x[2].lineinfo } + return { tag="Localrec", vars, vals } + end }, + -- local ( = )? + default = gg.sequence{ + gg.list{ + primary = annot.opt(M, _M.id, 'tf'), + separators = ',' }, + gg.onkeyword{ "=", _M.expr_list }, + builder = function(x) + local annotated_left, right = unpack(x) + local left, annotations = annot.split(annotated_left) + return {tag="Local", left, right or { }, annotations } + end } } + + ------------------------------------------------------------------------ + -- statement + ------------------------------------------------------------------------ + M.stat = gg.multisequence { + name = "statement", + { "do", _M.block, "end", builder = + function (x) return { tag="Do", unpack (x[1]) } end }, + { "for", _M.for_header, "do", _M.block, "end", builder = + function (x) x[1][#x[1]+1] = x[2]; return x[1] end }, + { "function", func_name, method_name, _M.func_val, builder=funcdef_builder }, + { "while", _M.expr, "do", _M.block, "end", builder = "While" }, + { "repeat", _M.block, "until", _M.expr, builder = "Repeat" }, + { "local", _M.local_stat_parser, builder = unpack }, + { "return", return_expr_list_parser, builder = + function(x) x[1].tag='Return'; return x[1] end }, + { "break", builder = function() return { tag="Break" } end }, + { "-{", gg.future(M, 'meta').splice_content, "}", builder = unpack }, + { "if", gg.nonempty(elseifs_parser), gg.onkeyword{ "else", M.block }, "end", + builder = if_builder }, + default = assign_or_call_stat_parser } + + M.assignments = { + ["="] = "Set" + } + + function M.assignments:add(k, v) self[k] = v end + + return M +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/compiler/parser/table.lua b/src/lib/metalua/metalua/compiler/parser/table.lua new file mode 100644 index 000000000..719690d78 --- /dev/null +++ b/src/lib/metalua/metalua/compiler/parser/table.lua @@ -0,0 +1,70 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Exported API: +-- * [M.table_bracket_field()] +-- * [M.table_field()] +-- * [M.table_content()] +-- * [M.table()] +-- +-- KNOWN BUG: doesn't handle final ";" or "," before final "}" +-- +-------------------------------------------------------------------------------- + +local gg = require 'metalua.grammar.generator' + +return function(M) + + M.table = { } + local _table = gg.future(M.table) + local _expr = gg.future(M).expr + + -------------------------------------------------------------------------------- + -- `[key] = value` table field definition + -------------------------------------------------------------------------------- + M.table.bracket_pair = gg.sequence{ "[", _expr, "]", "=", _expr, builder = "Pair" } + + -------------------------------------------------------------------------------- + -- table element parser: list value, `id = value` pair or `[value] = value` pair. + -------------------------------------------------------------------------------- + function M.table.element (lx) + if lx :is_keyword (lx :peek(), "[") then return M.table.bracket_pair(lx) end + local e = M.expr (lx) + if not lx :is_keyword (lx :peek(), "=") then return e end + lx :next(); -- skip the "=" + local key = M.id2string(e) -- will fail on non-identifiers + local val = M.expr(lx) + local r = { tag="Pair", key, val } + r.lineinfo = { first = key.lineinfo.first, last = val.lineinfo.last } + return r + end + + ----------------------------------------------------------------------------- + -- table constructor, without enclosing braces; returns a full table object + ----------------------------------------------------------------------------- + M.table.content = gg.list { + -- eta expansion to allow patching the element definition + primary = _table.element, + separators = { ",", ";" }, + terminators = "}", + builder = "Table" } + + -------------------------------------------------------------------------------- + -- complete table constructor including [{...}] + -------------------------------------------------------------------------------- + -- TODO beware, stat and expr use only table.content, this can't be patched. + M.table.table = gg.sequence{ "{", _table.content, "}", builder = unpack } + + return M +end \ No newline at end of file diff --git a/src/lib/metalua/metalua/grammar/generator.lua b/src/lib/metalua/metalua/grammar/generator.lua new file mode 100644 index 000000000..b73e87b74 --- /dev/null +++ b/src/lib/metalua/metalua/grammar/generator.lua @@ -0,0 +1,854 @@ +-------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +-------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Summary: parser generator. Collection of higher order functors, +-- which allow to build and combine parsers. Relies on a lexer +-- that supports the same API as the one exposed in mll.lua. +-- +-------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Exported API: +-- +-- Parser generators: +-- * [gg.sequence()] +-- * [gg.multisequence()] +-- * [gg.expr()] +-- * [gg.list()] +-- * [gg.onkeyword()] +-- * [gg.optkeyword()] +-- +-- Other functions: +-- * [gg.parse_error()] +-- * [gg.make_parser()] +-- * [gg.is_parser()] +-- +-------------------------------------------------------------------------------- + +local M = { } + +local lexer = require 'metalua.grammar.lexer' + +-------------------------------------------------------------------------------- +-- Symbol generator: [gensym()] returns a guaranteed-to-be-unique identifier. +-- The main purpose is to avoid variable capture in macros. +-- +-- If a string is passed as an argument, theis string will be part of the +-- id name (helpful for macro debugging) +-------------------------------------------------------------------------------- +local gensymidx = 0 + +function M.gensym (arg) + gensymidx = gensymidx + 1 + return { tag="Id", string.format(".%i.%s", gensymidx, arg or "")} +end + + +------------------------------------------------------------------------------- +-- parser metatable, which maps __call to method parse, and adds some +-- error tracing boilerplate. +------------------------------------------------------------------------------- +local parser_metatable = { } + +function parser_metatable :__call (lx, ...) + return self :parse (lx, ...) +end + +------------------------------------------------------------------------------- +-- Turn a table into a parser, mainly by setting the metatable. +------------------------------------------------------------------------------- +function M.make_parser(kind, p) + p.kind = kind + if not p.transformers then p.transformers = { } end + function p.transformers:add (x) + table.insert (self, x) + end + setmetatable (p, parser_metatable) + return p +end + +------------------------------------------------------------------------------- +-- Return true iff [x] is a parser. +-- If it's a gg-generated parser, return the name of its kind. +------------------------------------------------------------------------------- +function M.is_parser (x) + return type(x)=="function" or getmetatable(x)==parser_metatable and x.kind +end + +------------------------------------------------------------------------------- +-- Parse a sequence, without applying builder nor transformers. +------------------------------------------------------------------------------- +local function raw_parse_sequence (lx, p) + local r = { } + for i=1, #p do + local e=p[i] + if type(e) == "string" then + local kw = lx :next() + if not lx :is_keyword (kw, e) then + M.parse_error( + lx, "A keyword was expected, probably `%s'.", e) + end + elseif M.is_parser (e) then + table.insert (r, e(lx)) + else -- Invalid parser definition, this is *not* a parsing error + error(string.format( + "Sequence `%s': element #%i is neither a string nor a parser: %s", + p.name, i, table.tostring(e))) + end + end + return r +end + +------------------------------------------------------------------------------- +-- Parse a multisequence, without applying multisequence transformers. +-- The sequences are completely parsed. +------------------------------------------------------------------------------- +local function raw_parse_multisequence (lx, sequence_table, default) + local seq_parser = sequence_table[lx:is_keyword(lx:peek())] + if seq_parser then return seq_parser (lx) + elseif default then return default (lx) + else return false end +end + +------------------------------------------------------------------------------- +-- Applies all transformers listed in parser on ast. +------------------------------------------------------------------------------- +local function transform (ast, parser, fli, lli) + if parser.transformers then + for _, t in ipairs (parser.transformers) do ast = t(ast) or ast end + end + if type(ast) == 'table' then + local ali = ast.lineinfo + if not ali or ali.first~=fli or ali.last~=lli then + ast.lineinfo = lexer.new_lineinfo(fli, lli) + end + end + return ast +end + +------------------------------------------------------------------------------- +-- Generate a tracable parsing error (not implemented yet) +------------------------------------------------------------------------------- +function M.parse_error(lx, fmt, ...) + local inf = lx:lineinfo_error() + local file, line, column, offset, positions + local p_line, p_column + if inf then + local li = inf.last + file, line, column, offset = li.source, li.line, li.column, li.offset + local first = inf.first + positions = { first = first, last = li } + + local facing = inf.first.facing + local no_invalid_comment = function(comments) + for _, c in ipairs(comments) do + if c[1]:match("%[%[") then + return false + end + end + return true + end + -- in case of a parse error, the consumed token usually facing the one + -- that caused the problem is the last valid one, allowing us to point + -- to the exact location where the invalid token starts + -- there are exceptions to this, identified so far: + -- * there's just one token yet + -- * there are comment(s) after the last valid token + if (facing.line >= first.line and facing.column >= first.column) or + (facing.comments and no_invalid_comment(facing.comments)) + then + p_line, p_column = first.line, first.column + else + p_line, p_column = facing.line, facing.column + end + else + line, column, offset, p_line, p_column = -1, -1, -1, -1, -1 + end + + local msg = string.format("line %i, char %i: "..fmt, line, column, ...) + if file and file~='?' then msg = "file "..file..", "..msg end + local prev_token = string.format("facing: line %i, char %i", p_line, p_column) + msg = string.format("%s\n%s", msg, prev_token) + + + local src = lx.src + if offset>0 and src then + local i, j = offset, offset + while src:sub(i,i) ~= '\n' and i>=0 do i=i-1 end + while src:sub(j,j) ~= '\n' and j<=#src do j=j+1 end + local srcline = src:sub (i+1, j-1) + local idx = string.rep (" ", column).."^" + msg = string.format("%s\n>>> %s\n>>> %s", msg, srcline, idx) + end + --lx :kill() + error(msg) +end + +------------------------------------------------------------------------------- +-- +-- Sequence parser generator +-- +------------------------------------------------------------------------------- +-- Input fields: +-- +-- * [builder]: how to build an AST out of sequence parts. let [x] be the list +-- of subparser results (keywords are simply omitted). [builder] can be: +-- - [nil], in which case the result of parsing is simply [x] +-- - a string, which is then put as a tag on [x] +-- - a function, which takes [x] as a parameter and returns an AST. +-- +-- * [name]: the name of the parser. Used for debug messages +-- +-- * [transformers]: a list of AST->AST functions, applied in order on ASTs +-- returned by the parser. +-- +-- * Table-part entries corresponds to keywords (strings) and subparsers +-- (function and callable objects). +-- +-- After creation, the following fields are added: +-- * [parse] the parsing function lexer->AST +-- * [kind] == "sequence" +-- * [name] is set, if it wasn't in the input. +-- +------------------------------------------------------------------------------- +function M.sequence (p) + M.make_parser ("sequence", p) + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p:parse (lx) + + -- Raw parsing: + local fli = lx:lineinfo_right() + local seq = raw_parse_sequence (lx, self) + local lli = lx:lineinfo_left() + + -- Builder application: + local builder, tb = self.builder, type (self.builder) + if tb == "string" then seq.tag = builder + elseif tb == "function" or builder and builder.__call then seq = builder(seq) + elseif builder == nil then -- nothing + else error ("Invalid builder of type "..tb.." in sequence") end + seq = transform (seq, self, fli, lli) + assert (not seq or seq.lineinfo) + return seq + end + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + -- Try to build a proper name + if p.name then + -- don't touch existing name + elseif type(p[1])=="string" then -- find name based on 1st keyword + if #p==1 then p.name=p[1] + elseif type(p[#p])=="string" then + p.name = p[1] .. " ... " .. p[#p] + else p.name = p[1] .. " ..." end + else -- can't find a decent name + p.name = "unnamed_sequence" + end + + return p +end -- + + +------------------------------------------------------------------------------- +-- +-- Multiple, keyword-driven, sequence parser generator +-- +------------------------------------------------------------------------------- +-- in [p], useful fields are: +-- +-- * [transformers]: as usual +-- +-- * [name]: as usual +-- +-- * Table-part entries must be sequence parsers, or tables which can +-- be turned into a sequence parser by [gg.sequence]. These +-- sequences must start with a keyword, and this initial keyword +-- must be different for each sequence. The table-part entries will +-- be removed after [gg.multisequence] returns. +-- +-- * [default]: the parser to run if the next keyword in the lexer is +-- none of the registered initial keywords. If there's no default +-- parser and no suitable initial keyword, the multisequence parser +-- simply returns [false]. +-- +-- After creation, the following fields are added: +-- +-- * [parse] the parsing function lexer->AST +-- +-- * [sequences] the table of sequences, indexed by initial keywords. +-- +-- * [add] method takes a sequence parser or a config table for +-- [gg.sequence], and adds/replaces the corresponding sequence +-- parser. If the keyword was already used, the former sequence is +-- removed and a warning is issued. +-- +-- * [get] method returns a sequence by its initial keyword +-- +-- * [kind] == "multisequence" +-- +------------------------------------------------------------------------------- +function M.multisequence (p) + M.make_parser ("multisequence", p) + + ------------------------------------------------------------------- + -- Add a sequence (might be just a config table for [gg.sequence]) + ------------------------------------------------------------------- + function p :add (s) + -- compile if necessary: + local keyword = type(s)=='table' and s[1] + if type(s)=='table' and not M.is_parser(s) then M.sequence(s) end + if M.is_parser(s)~='sequence' or type(keyword)~='string' then + if self.default then -- two defaults + error ("In a multisequence parser, all but one sequences ".. + "must start with a keyword") + else self.default = s end -- first default + else + if self.sequences[keyword] then -- duplicate keyword + -- TODO: warn that initial keyword `keyword` is overloaded in multiseq + end + self.sequences[keyword] = s + end + end -- + + ------------------------------------------------------------------- + -- Get the sequence starting with this keyword. [kw :: string] + ------------------------------------------------------------------- + function p :get (kw) return self.sequences [kw] end + + ------------------------------------------------------------------- + -- Remove the sequence starting with keyword [kw :: string] + ------------------------------------------------------------------- + function p :del (kw) + if not self.sequences[kw] then + -- TODO: warn that we try to delete a non-existent entry + end + local removed = self.sequences[kw] + self.sequences[kw] = nil + return removed + end + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p :parse (lx) + local fli = lx:lineinfo_right() + local x = raw_parse_multisequence (lx, self.sequences, self.default) + local lli = lx:lineinfo_left() + return transform (x, self, fli, lli) + end + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + -- Register the sequences passed to the constructor. They're going + -- from the array part of the parser to the hash part of field + -- [sequences] + p.sequences = { } + for i=1, #p do p :add (p[i]); p[i] = nil end + + -- FIXME: why is this commented out? + --if p.default and not is_parser(p.default) then sequence(p.default) end + return p +end -- + + +------------------------------------------------------------------------------- +-- +-- Expression parser generator +-- +------------------------------------------------------------------------------- +-- +-- Expression configuration relies on three tables: [prefix], [infix] +-- and [suffix]. Moreover, the primary parser can be replaced by a +-- table: in this case the [primary] table will be passed to +-- [gg.multisequence] to create a parser. +-- +-- Each of these tables is a modified multisequence parser: the +-- differences with respect to regular multisequence config tables are: +-- +-- * the builder takes specific parameters: +-- - for [prefix], it takes the result of the prefix sequence parser, +-- and the prefixed expression +-- - for [infix], it takes the left-hand-side expression, the results +-- of the infix sequence parser, and the right-hand-side expression. +-- - for [suffix], it takes the suffixed expression, and the result +-- of the suffix sequence parser. +-- +-- * the default field is a list, with parameters: +-- - [parser] the raw parsing function +-- - [transformers], as usual +-- - [prec], the operator's precedence +-- - [assoc] for [infix] table, the operator's associativity, which +-- can be "left", "right" or "flat" (default to left) +-- +-- In [p], useful fields are: +-- * [transformers]: as usual +-- * [name]: as usual +-- * [primary]: the atomic expression parser, or a multisequence config +-- table (mandatory) +-- * [prefix]: prefix operators config table, see above. +-- * [infix]: infix operators config table, see above. +-- * [suffix]: suffix operators config table, see above. +-- +-- After creation, these fields are added: +-- * [kind] == "expr" +-- * [parse] as usual +-- * each table is turned into a multisequence, and therefore has an +-- [add] method +-- +------------------------------------------------------------------------------- +function M.expr (p) + M.make_parser ("expr", p) + + ------------------------------------------------------------------- + -- parser method. + -- In addition to the lexer, it takes an optional precedence: + -- it won't read expressions whose precedence is lower or equal + -- to [prec]. + ------------------------------------------------------------------- + function p :parse (lx, prec) + prec = prec or 0 + + ------------------------------------------------------ + -- Extract the right parser and the corresponding + -- options table, for (pre|in|suff)fix operators. + -- Options include prec, assoc, transformers. + ------------------------------------------------------ + local function get_parser_info (tab) + local p2 = tab :get (lx :is_keyword (lx :peek())) + if p2 then -- keyword-based sequence found + local function parser(lx) return raw_parse_sequence(lx, p2) end + return parser, p2 + else -- Got to use the default parser + local d = tab.default + if d then return d.parse or d.parser, d + else return false, false end + end + end + + ------------------------------------------------------ + -- Look for a prefix sequence. Multiple prefixes are + -- handled through the recursive [p.parse] call. + -- Notice the double-transform: one for the primary + -- expr, and one for the one with the prefix op. + ------------------------------------------------------ + local function handle_prefix () + local fli = lx :lineinfo_right() + local p2_func, p2 = get_parser_info (self.prefix) + local op = p2_func and p2_func (lx) + if op then -- Keyword-based sequence found + local ili = lx :lineinfo_right() -- Intermediate LineInfo + local e = p2.builder (op, self :parse (lx, p2.prec)) + local lli = lx :lineinfo_left() + return transform (transform (e, p2, ili, lli), self, fli, lli) + else -- No prefix found, get a primary expression + local e = self.primary(lx) + local lli = lx :lineinfo_left() + return transform (e, self, fli, lli) + end + end -- + + ------------------------------------------------------ + -- Look for an infix sequence+right-hand-side operand. + -- Return the whole binary expression result, + -- or false if no operator was found. + ------------------------------------------------------ + local function handle_infix (e) + local p2_func, p2 = get_parser_info (self.infix) + if not p2 then return false end + + ----------------------------------------- + -- Handle flattening operators: gather all operands + -- of the series in [list]; when a different operator + -- is found, stop, build from [list], [transform] and + -- return. + ----------------------------------------- + if (not p2.prec or p2.prec>prec) and p2.assoc=="flat" then + local fli = lx:lineinfo_right() + local pflat, list = p2, { e } + repeat + local op = p2_func(lx) + if not op then break end + table.insert (list, self:parse (lx, p2.prec)) + local _ -- We only care about checking that p2==pflat + _, p2 = get_parser_info (self.infix) + until p2 ~= pflat + local e2 = pflat.builder (list) + local lli = lx:lineinfo_left() + return transform (transform (e2, pflat, fli, lli), self, fli, lli) + + ----------------------------------------- + -- Handle regular infix operators: [e] the LHS is known, + -- just gather the operator and [e2] the RHS. + -- Result goes in [e3]. + ----------------------------------------- + elseif p2.prec and p2.prec>prec or + p2.prec==prec and p2.assoc=="right" then + local fli = e.lineinfo.first -- lx:lineinfo_right() + local op = p2_func(lx) + if not op then return false end + local e2 = self:parse (lx, p2.prec) + local e3 = p2.builder (e, op, e2) + local lli = lx:lineinfo_left() + return transform (transform (e3, p2, fli, lli), self, fli, lli) + + ----------------------------------------- + -- Check for non-associative operators, and complain if applicable. + ----------------------------------------- + elseif p2.assoc=="none" and p2.prec==prec then + M.parse_error (lx, "non-associative operator!") + + ----------------------------------------- + -- No infix operator suitable at that precedence + ----------------------------------------- + else return false end + + end -- + + ------------------------------------------------------ + -- Look for a suffix sequence. + -- Return the result of suffix operator on [e], + -- or false if no operator was found. + ------------------------------------------------------ + local function handle_suffix (e) + -- FIXME bad fli, must take e.lineinfo.first + local p2_func, p2 = get_parser_info (self.suffix) + if not p2 then return false end + if not p2.prec or p2.prec>=prec then + --local fli = lx:lineinfo_right() + local fli = e.lineinfo.first + local op = p2_func(lx) + if not op then return false end + local lli = lx:lineinfo_left() + e = p2.builder (e, op) + e = transform (transform (e, p2, fli, lli), self, fli, lli) + return e + end + return false + end -- + + ------------------------------------------------------ + -- Parser body: read suffix and (infix+operand) + -- extensions as long as we're able to fetch more at + -- this precedence level. + ------------------------------------------------------ + local e = handle_prefix() + repeat + local x = handle_suffix (e); e = x or e + local y = handle_infix (e); e = y or e + until not (x or y) + + -- No transform: it already happened in operators handling + return e + end -- + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + if not p.primary then p.primary=p[1]; p[1]=nil end + for _, t in ipairs{ "primary", "prefix", "infix", "suffix" } do + if not p[t] then p[t] = { } end + if not M.is_parser(p[t]) then M.multisequence(p[t]) end + end + function p:add(...) return self.primary:add(...) end + return p +end -- + + +------------------------------------------------------------------------------- +-- +-- List parser generator +-- +------------------------------------------------------------------------------- +-- In [p], the following fields can be provided in input: +-- +-- * [builder]: takes list of subparser results, returns AST +-- * [transformers]: as usual +-- * [name]: as usual +-- +-- * [terminators]: list of strings representing the keywords which +-- might mark the end of the list. When non-empty, the list is +-- allowed to be empty. A string is treated as a single-element +-- table, whose element is that string, e.g. ["do"] is the same as +-- [{"do"}]. +-- +-- * [separators]: list of strings representing the keywords which can +-- separate elements of the list. When non-empty, one of these +-- keyword has to be found between each element. Lack of a separator +-- indicates the end of the list. A string is treated as a +-- single-element table, whose element is that string, e.g. ["do"] +-- is the same as [{"do"}]. If [terminators] is empty/nil, then +-- [separators] has to be non-empty. +-- +-- After creation, the following fields are added: +-- * [parse] the parsing function lexer->AST +-- * [kind] == "list" +-- +------------------------------------------------------------------------------- +function M.list (p) + M.make_parser ("list", p) + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p :parse (lx) + + ------------------------------------------------------ + -- Used to quickly check whether there's a terminator + -- or a separator immediately ahead + ------------------------------------------------------ + local function peek_is_in (keywords) + return keywords and lx:is_keyword(lx:peek(), unpack(keywords)) end + + local x = { } + local fli = lx :lineinfo_right() + + -- if there's a terminator to start with, don't bother trying + local is_empty_list = self.terminators and (peek_is_in (self.terminators) or lx:peek().tag=="Eof") + if not is_empty_list then + repeat + local item = self.primary(lx) + table.insert (x, item) -- read one element + until + -- There's a separator list specified, and next token isn't in it. + -- Otherwise, consume it with [lx:next()] + self.separators and not(peek_is_in (self.separators) and lx:next()) or + -- Terminator token ahead + peek_is_in (self.terminators) or + -- Last reason: end of file reached + lx:peek().tag=="Eof" + end + + local lli = lx:lineinfo_left() + + -- Apply the builder. It can be a string, or a callable value, + -- or simply nothing. + local b = self.builder + if b then + if type(b)=="string" then x.tag = b -- b is a string, use it as a tag + elseif type(b)=="function" then x=b(x) + else + local bmt = getmetatable(b) + if bmt and bmt.__call then x=b(x) end + end + end + return transform (x, self, fli, lli) + end -- + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + if not p.primary then p.primary = p[1]; p[1] = nil end + if type(p.terminators) == "string" then p.terminators = { p.terminators } + elseif p.terminators and #p.terminators == 0 then p.terminators = nil end + if type(p.separators) == "string" then p.separators = { p.separators } + elseif p.separators and #p.separators == 0 then p.separators = nil end + + return p +end -- + + +------------------------------------------------------------------------------- +-- +-- Keyword-conditioned parser generator +-- +------------------------------------------------------------------------------- +-- +-- Only apply a parser if a given keyword is found. The result of +-- [gg.onkeyword] parser is the result of the subparser (modulo +-- [transformers] applications). +-- +-- lineinfo: the keyword is *not* included in the boundaries of the +-- resulting lineinfo. A review of all usages of gg.onkeyword() in the +-- implementation of metalua has shown that it was the appropriate choice +-- in every case. +-- +-- Input fields: +-- +-- * [name]: as usual +-- +-- * [transformers]: as usual +-- +-- * [peek]: if non-nil, the conditioning keyword is left in the lexeme +-- stream instead of being consumed. +-- +-- * [primary]: the subparser. +-- +-- * [keywords]: list of strings representing triggering keywords. +-- +-- * Table-part entries can contain strings, and/or exactly one parser. +-- Strings are put in [keywords], and the parser is put in [primary]. +-- +-- After the call, the following fields will be set: +-- +-- * [parse] the parsing method +-- * [kind] == "onkeyword" +-- * [primary] +-- * [keywords] +-- +------------------------------------------------------------------------------- +function M.onkeyword (p) + M.make_parser ("onkeyword", p) + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p :parse (lx) + if lx :is_keyword (lx:peek(), unpack(self.keywords)) then + local fli = lx:lineinfo_right() + if not self.peek then lx:next() end + local content = self.primary (lx) + local lli = lx:lineinfo_left() + local li = content.lineinfo or { } + fli, lli = li.first or fli, li.last or lli + return transform (content, p, fli, lli) + else return false end + end + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + if not p.keywords then p.keywords = { } end + for _, x in ipairs(p) do + if type(x)=="string" then table.insert (p.keywords, x) + else assert (not p.primary and M.is_parser (x)); p.primary = x end + end + assert (next (p.keywords), "Missing trigger keyword in gg.onkeyword") + assert (p.primary, 'no primary parser in gg.onkeyword') + return p +end -- + + +------------------------------------------------------------------------------- +-- +-- Optional keyword consummer pseudo-parser generator +-- +------------------------------------------------------------------------------- +-- +-- This doesn't return a real parser, just a function. That function parses +-- one of the keywords passed as parameters, and returns it. It returns +-- [false] if no matching keyword is found. +-- +-- Notice that tokens returned by lexer already carry lineinfo, therefore +-- there's no need to add them, as done usually through transform() calls. +------------------------------------------------------------------------------- +function M.optkeyword (...) + local args = {...} + if type (args[1]) == "table" then + assert (#args == 1) + args = args[1] + end + for _, v in ipairs(args) do assert (type(v)=="string") end + return function (lx) + local x = lx:is_keyword (lx:peek(), unpack (args)) + if x then lx:next(); return x + else return false end + end +end + + +------------------------------------------------------------------------------- +-- +-- Run a parser with a special lexer +-- +------------------------------------------------------------------------------- +-- +-- This doesn't return a real parser, just a function. +-- First argument is the lexer class to be used with the parser, +-- 2nd is the parser itself. +-- The resulting parser returns whatever the argument parser does. +-- +------------------------------------------------------------------------------- +function M.with_lexer(new_lexer, parser) + + ------------------------------------------------------------------- + -- Most gg functions take their parameters in a table, so it's + -- better to silently accept when with_lexer{ } is called with + -- its arguments in a list: + ------------------------------------------------------------------- + if not parser and #new_lexer==2 and type(new_lexer[1])=='table' then + return M.with_lexer(unpack(new_lexer)) + end + + ------------------------------------------------------------------- + -- Save the current lexer, switch it for the new one, run the parser, + -- restore the previous lexer, even if the parser caused an error. + ------------------------------------------------------------------- + return function (lx) + local old_lexer = getmetatable(lx) + lx:sync() + setmetatable(lx, new_lexer) + local status, result = pcall(parser, lx) + lx:sync() + setmetatable(lx, old_lexer) + if status then return result else error(result) end + end +end + +-------------------------------------------------------------------------------- +-- +-- Make sure a parser is used and returns successfully. +-- +-------------------------------------------------------------------------------- +function M.nonempty(primary) + local p = M.make_parser('non-empty list', { primary = primary, name=primary.name }) + function p :parse (lx) + local fli = lx:lineinfo_right() + local content = self.primary (lx) + local lli = lx:lineinfo_left() + local li = content.lineinfo or { } + fli, lli = li.first or fli, li.last or lli + if #content == 0 then + M.parse_error (lx, "`%s' must not be empty.", self.name or "list") + else + return transform (content, self, fli, lli) + end + end + return p +end + +local FUTURE_MT = { } +function FUTURE_MT:__tostring() return "" end +function FUTURE_MT:__newindex(key, value) error "don't write in futures" end +function FUTURE_MT :__index (parser_name) + return function(...) + local p, m = rawget(self, '__path'), self.__module + if p then for _, name in ipairs(p) do + m=rawget(m, name) + if not m then error ("Submodule '"..name.."' undefined") end + end end + local f = rawget(m, parser_name) + if not f then error ("Parser '"..parser_name.."' undefined") end + return f(...) + end +end + +function M.future(module, ...) + checks('table') + local path = ... and {...} + if path then for _, x in ipairs(path) do + assert(type(x)=='string', "Bad future arg") + end end + local self = { __module = module, + __path = path } + return setmetatable(self, FUTURE_MT) +end + +return M diff --git a/src/lib/metalua/metalua/grammar/lexer.lua b/src/lib/metalua/metalua/grammar/lexer.lua new file mode 100644 index 000000000..5102ae047 --- /dev/null +++ b/src/lib/metalua/metalua/grammar/lexer.lua @@ -0,0 +1,773 @@ +------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +------------------------------------------------------------------------------- +local M = {} + +---@class lexer +---@field alpha table +---@field sym table> +---@field src string +---@field posfact position_factory +---@field newstream fun(lx: lexer, src_or_stream: string | table, src_name: string?): lexer? +---@field is_keyword fun(lx: lexer, tk: token, ...: string): string | false +---@field extract_short_comment fun(lx: lexer): string?, number?, number? +---@field extract_long_comment fun(lx: lexer): string?, number?, number? +---@field extract_short_string fun(lx: lexer): string?, string? +---@field extract_word fun(lx: lexer): string?, number? +---@field extract_number fun(lx: lexer): string?, number? +---@field extract_long_string fun(lx: lexer): string?, string? | nil +---@field extract_symbol fun(lx: lexer): string, string | nil +---@field peek fun(lx: lexer, n?: number): token +---@field next fun(lx: lexer, n?: number): token +---@field extract fun(lx: lexer): token +---@field kill fun(lx: lexer): nil +---@field clone fun(lx: lexer): lexer +---@field check fun(lx: lexer, ...): string | nil +---@field add fun(lx: lexer, keyword: string | string[]): nil +---@field sync fun(lx: lexer): nil +local lexer = { alpha = {}, sym = {} } +lexer.__index = lexer +lexer.__type = "lexer.stream" +M.lexer = lexer + +-- HACK: wtf is locale? + +-- Some locale settings produce bad results, e.g. French locale +-- expect float numbers to use commas instead of periods. +-- TODO: change number parser into something loclae-independent, +-- locales are nasty. +---------------------------------------------------------------------- +os.setlocale("C") + +local MT = {} + +M.metatables = MT + +---Create a new metatable, for a new class of objects. +---@param name string +local function new_metatable(name) + local mt = { __type = "lexer." .. name } + mt.__index = mt + MT[name] = mt +end + +new_metatable("position") + +local position_idx = 1 + +---@class position : lexer +---represent a point in a source file. +---@field line number +---@field column number +---@field offset number +---@field source string +---@field comments comment +---@field facing position +---@field id integer? +function M.new_position(line, column, offset, source) + local id = position_idx + position_idx = position_idx + 1 + return setmetatable({ line = line, column = column, offset = offset, source = source, id = id }, MT.position) +end + +function MT.position:__tostring() + return string.format( + "<%s%s|L%d|C%d|K%d>", + self.comments and "C|" or "", + self.source, + self.line, + self.column, + self.offset + ) +end + +---@class position_factory: lexer +---@field src_name string +---@field line2offset number[] +---@field max number +---@field get_position fun(self: position_factory, offset: number): position +new_metatable("position_factory") + +---convert offsets into line/column/offset positions. +---@param src string +---@param src_name string +---@return position_factory +function M.new_position_factory(src, src_name) + local lines = { 1 } + for offset in src:gmatch("\n()") do + table.insert(lines, offset) + end + local max = #src + 1 + table.insert(lines, max + 1) -- +1 includes Eof + return setmetatable({ src_name = src_name, line2offset = lines, max = max }, MT.position_factory) +end + +---get the position of a given offset. +---@param offset number +---@return position +function MT.position_factory:get_position(offset) + assert(offset <= self.max) + local line2offset = self.line2offset + local left = self.last_left or 1 + if offset < line2offset[left] then + left = 1 + end + local right = left + 1 + if line2offset[right] <= offset then + right = right + 1 + end + if line2offset[right] <= offset then + right = #line2offset + end + while true do + if left + 1 == right then + break + end + local middle = math.floor((left + right) / 2) + if line2offset[middle] <= offset then + left = middle + else + right = middle + end + end + local line = left + local column = offset - line2offset[line] + 1 + self.last_left = left + return M.new_position(line, column, offset, self.src_name) +end + +---@class lineinfo +---@field first position +---@field last position +---Lineinfo: represent a node's range in a source file; +---embed information about prefix and suffix comments. +new_metatable("lineinfo") + +---@param first position +---@param last position +---@return lineinfo +function M.new_lineinfo(first, last) + return setmetatable({ first = first, last = last }, MT.lineinfo) +end + +function MT.lineinfo:__tostring() + local fli, lli = self.first, self.last + local line = fli.line + if line ~= lli.line then + line = line .. "-" .. lli.line + end + local column = fli.column + if column ~= lli.column then + column = column .. "-" .. lli.column + end + local offset = fli.offset + if offset ~= lli.offset then + offset = offset .. "-" .. lli.offset + end + return string.format( + "<%s%s|L%s|C%s|K%s%s>", + fli.comments and "C|" or "", + fli.source, + line, + column, + offset, + lli.comments and "|C" or "" + ) +end + +---@class token +---Token: atomic Lua language element, with a tag, a content, +---and some lineinfo relating it to its original source. +---@field tag string +---@field content string +---@field lineinfo lineinfo +---@field next token +new_metatable("token") + +---@param tag string +---@param content string +---@param lineinfo lineinfo +---@return token +function M.new_token(tag, content, lineinfo) + --printf("TOKEN `%s{ %q, lineinfo = %s} boundaries %d, %d", + -- tag, content, tostring(lineinfo), lineinfo.first.id, lineinfo.last.id) + return setmetatable({ tag = tag, lineinfo = lineinfo, content }, MT.token) +end + +function MT.token:__tostring() + --return string.format("`%s{ %q, %s }", self.tag, self[1], tostring(self.lineinfo)) + return string.format("`%s %q", self.tag, self[1]) +end + +---@class comment +---Comment: series of comment blocks with associated lineinfo. +---To be attached to the tokens just before and just after them. +---@field lineinfo lineinfo +---@field text fun(): string +new_metatable("comment") + +---@param lines token[] +---@return comment +function M.new_comment(lines) + local first = lines[1].lineinfo.first + local last = lines[#lines].lineinfo.last + local lineinfo = M.new_lineinfo(first, last) + return setmetatable({ lineinfo = lineinfo, unpack(lines) }, MT.comment) +end + +---Return the text of the comment, as a string. +---@return string +function MT.comment:text() + local last_line = self[1].lineinfo.last.line + local acc = {} + for _, line in ipairs(self) do + local nreturns = line.lineinfo.first.line - last_line + table.insert(acc, ("\n"):rep(nreturns)) + table.insert(acc, line[1]) + end + return table.concat(acc) +end + +---returns new_comment_line +---@param text string +---@param lineinfo lineinfo +---@param nequals? number +---@return table +function M.new_comment_line(text, lineinfo, nequals) + return { lineinfo = lineinfo, text, nequals } +end + +---@table Patterns used by [lexer :extract] to decompose the raw string into correctly tagged tokens. +lexer.patterns = { + spaces = "^[ \r\n\t]*()", + short_comment = "^%-%-([^\n]*)\n?()", + --final_short_comment = "^%-%-([^\n]*)()$", + long_comment = "^%-%-%[(=*)%[\n?(.-)%]%1%]()", + long_string = "^%[(=*)%[\n?(.-)%]%1%]()", + number_mantissa = { "^%d+%.?%d*()", "^%d*%.%d+()" }, + number_mantissa_hex = { "^%x+%.?%x*()", "^%x*%.%x+()" }, --Lua5.1 and Lua5.2 + number_exponant = "^[eE][%+%-]?%d+()", + number_exponant_hex = "^[pP][%+%-]?%d+()", --Lua5.2 + number_hex = "^0[xX]()", + word = "^([%a_][%w_]*)()", +} + +-- unescape a whole string, applying [unesc_digits] and +-- [unesc_letter] as many times as required. +local function unescape_string(s) + -- Turn the digits of an escape sequence into the corresponding + -- character, e.g. [unesc_digits("123") == string.char(123)]. + local function unesc_digits(backslashes, digits) + if #backslashes % 2 == 0 then + -- Even number of backslashes, they escape each other, not the digits. + -- Return them so that unesc_letter() can treat them + return backslashes .. digits + else + -- Remove the odd backslash, which escapes the number sequence. + -- The rest will be returned and parsed by unesc_letter() + backslashes = backslashes:sub(1, -2) + end + local k, j, i = digits:reverse():byte(1, 3) + local z = string.byte("0") + local code = (k or z) + 10 * (j or z) + 100 * (i or z) - 111 * z + if code > 255 then + error("Illegal escape sequence '\\" .. digits .. "' in string: ASCII codes must be in [0..255]") + end + local c = string.char(code) + if c == "\\" then + c = "\\\\" + end -- parsed by unesc_letter (test: "\092b" --> "\\b") + return backslashes .. c + end + + -- Turn hex digits of escape sequence into char. + local function unesc_hex(backslashes, digits) + if #backslashes % 2 == 0 then + return backslashes .. "x" .. digits + else + backslashes = backslashes:sub(1, -2) + end + local c = string.char(tonumber(digits, 16)) + if c == "\\" then + c = "\\\\" + end -- parsed by unesc_letter (test: "\x5cb" --> "\\b") + return backslashes .. c + end + + -- Handle Lua 5.2 \z sequences + local function unesc_z(backslashes, more) + if #backslashes % 2 == 0 then + return backslashes .. more + else + return backslashes:sub(1, -2) + end + end + + -- Take a letter [x], and returns the character represented by the + -- sequence ['\\'..x], e.g. [unesc_letter "n" == "\n"]. + local function unesc_letter(x) + local t = { + a = "\a", + b = "\b", + f = "\f", + n = "\n", + r = "\r", + t = "\t", + v = "\v", + ["\\"] = "\\", + ["'"] = "'", + ['"'] = '"', + ["\n"] = "\n", + } + return t[x] or x + end + + s = s:gsub("(\\+)(z%s*)", unesc_z) -- Lua 5.2 + s = s:gsub("(\\+)([0-9][0-9]?[0-9]?)", unesc_digits) + s = s:gsub("(\\+)x([0-9a-fA-F][0-9a-fA-F])", unesc_hex) -- Lua 5.2 + s = s:gsub("\\(%D)", unesc_letter) + return s +end + +lexer.extractors = { + "extract_long_comment", + "extract_short_comment", + "extract_short_string", + "extract_word", + "extract_number", + "extract_long_string", + "extract_symbol", +} + +---Really extract next token from the raw string (and update the index). +---loc: offset of the position just after spaces and comments +---previous_i: offset in src before extraction began +---@return token +function lexer:extract() + local attached_comments = {} + local function gen_token(...) + local token = M.new_token(...) + if #attached_comments > 0 then -- attach previous comments to token + local comments = M.new_comment(attached_comments) + token.lineinfo.first.comments = comments + if self.lineinfo_last_extracted then + self.lineinfo_last_extracted.comments = comments + end + attached_comments = {} + end + token.lineinfo.first.facing = self.lineinfo_last_extracted + self.lineinfo_last_extracted.facing = assert(token.lineinfo.first) + self.lineinfo_last_extracted = assert(token.lineinfo.last) + return token + end + while true do -- loop until a non-comment token is found + -- skip whitespaces + self.i = self.src:match(self.patterns.spaces, self.i) + if self.i > #self.src then + local fli = self.posfact:get_position(#self.src + 1) + local lli = self.posfact:get_position(#self.src + 1) -- ok? + local tok = gen_token("Eof", "eof", M.new_lineinfo(fli, lli)) + tok.lineinfo.last.facing = lli + return tok + end + local i_first = self.i -- loc = position after whitespaces + + -- try every extractor until a token is found + for _, extractor in ipairs(self.extractors) do + local tag, content, xtra = self[extractor](self) + if tag then + local fli = self.posfact:get_position(i_first) + local lli = self.posfact:get_position(self.i - 1) + local lineinfo = M.new_lineinfo(fli, lli) + if tag == "Comment" then + local prev_comment = attached_comments[#attached_comments] + if + not xtra -- new comment is short + and prev_comment + and not prev_comment[2] -- prev comment is short + and prev_comment.lineinfo.last.line + 1 == fli.line + then -- adjascent lines + -- concat with previous comment + prev_comment[1] = prev_comment[1] .. "\n" .. content -- TODO quadratic, BAD! + prev_comment.lineinfo.last = lli + else -- accumulate comment + local comment = M.new_comment_line(content, lineinfo, xtra) + table.insert(attached_comments, comment) + end + break -- back to skipping spaces + else -- not a comment: real token, then + return gen_token(tag, content, lineinfo) + end -- if token is a comment + end -- if token found + end -- for each extractor + end -- while token is a comment +end -- :extract() + +-- Extract a short comment. +function lexer:extract_short_comment() + -- TODO: handle final_short_comment + local content, j = self.src:match(self.patterns.short_comment, self.i) + if content then + self.i = j + return "Comment", content, nil + end +end + +-- Extract a long comment. +function lexer:extract_long_comment() + local equals, content, j = self.src:match(self.patterns.long_comment, self.i) + if j then + self.i = j + return "Comment", content, #equals + end +end + +-- Extract a '...' or "..." short string. +function lexer:extract_short_string() + local k = self.src:sub(self.i, self.i) -- first char + if k ~= [[']] and k ~= [["]] then + return + end -- no match' + local i = self.i + 1 + local j = i + while true do + local x, y + x, j, y = self.src:match("([\\\r\n" .. k .. "])()(.?)", j) -- next interesting char + if x == "\\" then + if y == "z" then -- Lua 5.2 \z + j = self.src:match("^%s*()", j + 1) + else + j = j + 1 -- escaped char + end + elseif x == k then + break -- end of string + else + assert(not x or x == "\r" or x == "\n") + return nil, "Unterminated string" + end + end + self.i = j + + return "String", unescape_string(self.src:sub(i, j - 2)) +end + +-- Extract Id or Keyword. +function lexer:extract_word() + local word, j = self.src:match(self.patterns.word, self.i) + if word then + self.i = j + return (self.alpha[word] and "Keyword" or "Id"), word + end +end + +---Extract Number. +---@return string?, number? +function lexer:extract_number() + local j = self.src:match(self.patterns.number_hex, self.i) + if j then + j = self.src:match(self.patterns.number_mantissa_hex[1], j) + or self.src:match(self.patterns.number_mantissa_hex[2], j) + if j then + j = self.src:match(self.patterns.number_exponant_hex, j) or j + end + else + j = self.src:match(self.patterns.number_mantissa[1], self.i) + or self.src:match(self.patterns.number_mantissa[2], self.i) + if j then + j = self.src:match(self.patterns.number_exponant, j) or j + end + end + if not j then + return + end + -- Number found, interpret with tonumber() and return it + local str = self.src:sub(self.i, j - 1) + -- :TODO: tonumber on Lua5.2 floating hex may or may not work on Lua5.1 + local n = tonumber(str) + if not n then + error(str .. " is not a valid number according to tonumber()") + end + self.i = j + return "Number", n +end + +---Extract long string. +---@return string?, string? +function lexer:extract_long_string() + local _, content, j = self.src:match(self.patterns.long_string, self.i) + if j then + self.i = j + return "String", content + end +end + +---Extract symbol. +---@return string, string? +function lexer:extract_symbol() + local k = self.src:sub(self.i, self.i) + local symk = self.sym[k] -- symbols starting with `k` + if not symk then + self.i = self.i + 1 + return "Keyword", k + end + for _, sym in pairs(symk) do + if sym == self.src:sub(self.i, self.i + #sym - 1) then + self.i = self.i + #sym + return "Keyword", sym + end + end + self.i = self.i + 1 + return "Keyword", k +end + +---Add a keyword to the list of keywords recognized by the lexer. +---@param w string | string[] +function lexer:add(w) + if type(w) == "table" then + for _, x in ipairs(w) do + self:add(x) + end + else + if w:match(self.patterns.word .. "$") then + self.alpha[w] = true + elseif w:match("^%p%p+$") then + local k = w:sub(1, 1) + local list = self.sym[k] + if not list then + list = {} + self.sym[k] = list + end + table.insert(list, w) + elseif w:match("^%p$") then + return + else + error("Invalid keyword") + end + end +end + +---Return the [n]th next token, without consuming it. +---[n] defaults to 1. If it goes pass the end of the stream, +---an EOF token is returned. +---@param n? number +---@return token +function lexer:peek(n) + if not n then + n = 1 + end + if n > #self.peeked then + for i = #self.peeked + 1, n do + self.peeked[i] = self:extract() + end + end + return self.peeked[n] +end + +---Return the [n]th next token, removing it as well as the 0..n-1 +---previous tokens. [n] defaults to 1. If it goes pass the end of the +---stream, an EOF token is returned. +---@param n number? +---@return token +function lexer:next(n) + n = n or 1 + self:peek(n) + local a + for _ = 1, n do + a = table.remove(self.peeked, 1) + -- TODO: is this used anywhere? I think not. a.lineinfo.last may be nil. + --self.lastline = a.lineinfo.last.line + end + self.lineinfo_consumed = a.lineinfo + self.lineinfo_last_consumed = a.lineinfo.last + return a +end + +---Returns an object which saves the stream's current state. +---FIXME there are more fields than that to save +---@alias lexer_state { i: number, peeked: token[] } +---@return lexer_state +function lexer:save() + return { self.i, { unpack(self.peeked) } } +end + +---Restore the stream's state, as saved by method [save]. +---FIXME there are more fields than that to restore +---@param s lexer_state +function lexer:restore(s) + self.i = s[1] + self.peeked = s[2] +end + +---Resynchronize: cancel any token in self.peeked, by emptying the +---list and resetting the indexes +---@return nil +function lexer:sync() + local p1 = self.peeked[1] + if p1 then + local li_first = p1.lineinfo.first + if li_first.comments then + li_first = li_first.comments.lineinfo.first + end + self.i = li_first.offset + self.column_offset = self.i - li_first.column + self.peeked = {} + self.attached_comments = p1.lineinfo.first.comments or {} + end +end + +---Take the source and offset of an old lexer. +---@param old lexer +---@return lexer +function lexer:takeover(old) + self:sync() + old:sync() + for _, field in ipairs({ "i", "src", "attached_comments", "posfact" }) do + self[field] = old[field] + end + return self +end + +---------------------------------------------------------------------- +-- Return the current position in the sources. This position is between +-- two tokens, and can be within a space / comment area, and therefore +-- have a non-null width. :lineinfo_left() returns the beginning of the +-- separation area, :lineinfo_right() returns the end of that area. +-- +-- ____ last consummed token ____ first unconsummed token +-- / / +-- XXXXX YYYYY +-- \____ \____ +-- :lineinfo_left() :lineinfo_right() +---------------------------------------------------------------------- +---return the position of the right boundary of the current gap +---@return position +function lexer:lineinfo_right() + return self:peek(1).lineinfo.first +end + +---return the position of the left boundary of the current gap +---@return position +function lexer:lineinfo_left() + return self.lineinfo_last_consumed +end + +function lexer:lineinfo_error() + return self.lineinfo_consumed +end + +--- Create a new lexstream. +--- @param src_or_stream string | table +--- @param name? string +--- @return lexer +function lexer:newstream(src_or_stream, name) + name = name or "?" + if type(src_or_stream) == "table" then -- it's a stream + return setmetatable({}, self):takeover(src_or_stream) + elseif type(src_or_stream) == "string" then -- it's a source string + local src = src_or_stream + local pos1 = M.new_position(1, 1, 1, name) + local stream = { + src_name = name, -- Name of the file + src = src, -- The source, as a single string + peeked = {}, -- Already peeked, but not discarded yet, tokens + i = 1, -- Character offset in src + attached_comments = {}, -- comments accumulator + lineinfo_last_extracted = pos1, + lineinfo_last_consumed = pos1, + posfact = M.new_position_factory(src_or_stream, name), + } + setmetatable(stream, self) + + -- Skip initial sharp-bang for Unix scripts + -- FIXME: redundant with mlp.chunk() + if src and src:match("^#!") then + local endofline = src:find("\n") + stream.i = endofline and (endofline + 1) or #src + end + return stream + else + assert(false, ":newstream() takes a source string or a stream, not a " .. type(src_or_stream)) + end +end + +---If there's no ... args, return the token a (whose truth value is +---true) if it's a `Keyword{ }, or nil. If there are ... args, they +---have to be strings. if the token a is a keyword, and it's content +---is one of the ... args, then returns it (it's truth value is +---true). If no a keyword or not in ..., return nil. +---@param a table +---@param ... string[] +---@return boolean | string[] +function lexer:is_keyword(a, ...) + if not a or a.tag ~= "Keyword" then + return false + end + local words = { ... } + if #words == 0 then + return a[1] + end + for _, w in ipairs(words) do + if w == a[1] then + return w + end + end + return false +end + +---Cause an error if the next token isn't a keyword whose content is listed among ... args (which have to be strings). +---@param ... string[] +---@return string[]? +function lexer:check(...) + local words = { ... } + local a = self:next() + local function err() + error("Got " .. tostring(a) .. ", expected one of these keywords : '" .. table.concat(words, "', '") .. "'") + end + if not a or a.tag ~= "Keyword" then + err() + end + if #words == 0 then + return a[1] + end + for _, w in ipairs(words) do + if w == a[1] then + return w + end + end + err() +end + +---clone a lexer +---@return lexer +function lexer:clone() + local alpha_clone, sym_clone = {}, {} + for word in pairs(self.alpha) do + alpha_clone[word] = true + end + for letter, list in pairs(self.sym) do + sym_clone[letter] = { unpack(list) } + end + local clone = { alpha = alpha_clone, sym = sym_clone } + setmetatable(clone, self) + clone.__index = clone + return clone +end + +---Cancel everything left in a lexer, all subsequent attempts at `:peek()` or `:next()` will return `Eof`. +---@return nil +function lexer:kill() + self.i = #self.src + 1 + self.peeked = {} + self.attached_comments = {} + self.lineinfo_last = self.posfact:get_position(#self.src + 1) +end + +return M diff --git a/src/lib/metalua/metalua/pprint.lua b/src/lib/metalua/metalua/pprint.lua new file mode 100644 index 000000000..fe8f9663a --- /dev/null +++ b/src/lib/metalua/metalua/pprint.lua @@ -0,0 +1,288 @@ +------------------------------------------------------------------------------- +-- Copyright (c) 2006-2013 Fabien Fleutot and others. +-- +-- Made available under the terms of the MIT public license, which +-- accompanies this distribution in LICENSE and is available at +-- http://www.lua.org/license.html +-- +-- Contributors: +-- Fabien Fleutot - API and implementation +-- +---------------------------------------------------------------------- + +---------------------------------------------------------------------- +---------------------------------------------------------------------- +-- +-- Lua objects pretty-printer +-- +---------------------------------------------------------------------- +---------------------------------------------------------------------- + +local M = { } + +M.DEFAULT_CFG = { + hide_hash = false; -- Print the non-array part of tables? + metalua_tag = true; -- Use Metalua's backtick syntax sugar? + fix_indent = nil; -- If a number, number of indentation spaces; + -- If false, indent to the previous brace. + line_max = nil; -- If a number, tries to avoid making lines with + -- more than this number of chars. + initial_indent = 0; -- If a number, starts at this level of indentation + keywords = { }; -- Set of keywords which must not use Lua's field + -- shortcuts {["foo"]=...} -> {foo=...} +} + +local function valid_id(cfg, x) + if type(x) ~= "string" then return false end + if not x:match "^[a-zA-Z_][a-zA-Z0-9_]*$" then return false end + if cfg.keywords and cfg.keywords[x] then return false end + return true +end + +local __tostring_cache = setmetatable({ }, {__mode='k'}) + +-- Retrieve the string produced by `__tostring` metamethod if present, +-- return `false` otherwise. Cached in `__tostring_cache`. +local function __tostring(x) + local the_string = __tostring_cache[x] + if the_string~=nil then return the_string end + local mt = getmetatable(x) + if mt then + local __tostring = mt.__tostring + if __tostring then + the_string = __tostring(x) + __tostring_cache[x] = the_string + return the_string + end + end + if x~=nil then __tostring_cache[x] = false end -- nil is an illegal key + return false +end + +local xlen -- mutually recursive with `xlen_type` + +local xlen_cache = setmetatable({ }, {__mode='k'}) + +-- Helpers for the `xlen` function +local xlen_type = { + ["nil"] = function ( ) return 3 end; + number = function (x) return #tostring(x) end; + boolean = function (x) return x and 4 or 5 end; + string = function (x) return #string.format("%q",x) end; +} + +function xlen_type.table (adt, cfg, nested) + local custom_string = __tostring(adt) + if custom_string then return #custom_string end + + -- Circular referenced objects are printed with the plain + -- `tostring` function in nested positions. + if nested [adt] then return #tostring(adt) end + nested [adt] = true + + local has_tag = cfg.metalua_tag and valid_id(cfg, adt.tag) + local alen = #adt + local has_arr = alen>0 + local has_hash = false + local x = 0 + + if not cfg.hide_hash then + -- first pass: count hash-part + for k, v in pairs(adt) do + if k=="tag" and has_tag then + -- this is the tag -> do nothing! + elseif type(k)=="number" and k<=alen and math.fmod(k,1)==0 and k>0 then + -- array-part pair -> do nothing! + else + has_hash = true + if valid_id(cfg, k) then x=x+#k + else x = x + xlen (k, cfg, nested) + 2 end -- count surrounding brackets + x = x + xlen (v, cfg, nested) + 5 -- count " = " and ", " + end + end + end + + for i = 1, alen do x = x + xlen (adt[i], nested) + 2 end -- count ", " + + nested[adt] = false -- No more nested calls + + if not (has_tag or has_arr or has_hash) then return 3 end + if has_tag then x=x+#adt.tag+1 end + if not (has_arr or has_hash) then return x end + if not has_hash and alen==1 and type(adt[1])~="table" then + return x-2 -- substract extraneous ", " + end + return x+2 -- count "{ " and " }", substract extraneous ", " +end + + +-- Compute the number of chars it would require to display the table +-- on a single line. Helps to decide whether some carriage returns are +-- required. Since the size of each sub-table is required many times, +-- it's cached in [xlen_cache]. +xlen = function (x, cfg, nested) + -- no need to compute length for 1-line prints + if not cfg.line_max then return 0 end + nested = nested or { } + if x==nil then return #"nil" end + local len = xlen_cache[x] + if len then return len end + local f = xlen_type[type(x)] + if not f then return #tostring(x) end + len = f (x, cfg, nested) + xlen_cache[x] = len + return len +end + +local function consider_newline(p, len) + if not p.cfg.line_max then return end + if p.current_offset + len <= p.cfg.line_max then return end + if p.indent < p.current_offset then + p:acc "\n"; p:acc ((" "):rep(p.indent)) + p.current_offset = p.indent + end +end + +local acc_value + +local acc_type = { + ["nil"] = function(p) p:acc("nil") end; + number = function(p, adt) p:acc (tostring (adt)) end; + string = function(p, adt) p:acc ((string.format ("%q", adt):gsub("\\\n", "\\n"))) end; + boolean = function(p, adt) p:acc (adt and "true" or "false") end } + +-- Indentation: +-- * if `cfg.fix_indent` is set to a number: +-- * add this number of space for each level of depth +-- * return to the line as soon as it flushes things further left +-- * if not, tabulate to one space after the opening brace. +-- * as a result, it never saves right-space to return before first element + +function acc_type.table(p, adt) + if p.nested[adt] then p:acc(tostring(adt)); return end + p.nested[adt] = true + + local has_tag = p.cfg.metalua_tag and valid_id(p.cfg, adt.tag) + local alen = #adt + local has_arr = alen>0 + local has_hash = false + + local previous_indent = p.indent + + if has_tag then p:acc("`"); p:acc(adt.tag) end + + local function indent(p) + if not p.cfg.fix_indent then p.indent = p.current_offset + else p.indent = p.indent + p.cfg.fix_indent end + end + + -- First pass: handle hash-part + if not p.cfg.hide_hash then + for k, v in pairs(adt) do + + if has_tag and k=='tag' then -- pass the 'tag' field + elseif type(k)=="number" and k<=alen and k>0 and math.fmod(k,1)==0 then + -- pass array-part keys (consecutive ints less than `#adt`) + else -- hash-part keys + if has_hash then p:acc ", " else -- 1st hash-part pair ever found + p:acc "{ "; indent(p) + end + + -- Determine whether a newline is required + local is_id, expected_len=valid_id(p.cfg, k) + if is_id then expected_len=#k+xlen(v, p.cfg, p.nested)+#" = , " + else expected_len = xlen(k, p.cfg, p.nested)+xlen(v, p.cfg, p.nested)+#"[] = , " end + consider_newline(p, expected_len) + + -- Print the key + if is_id then p:acc(k); p:acc " = " else + p:acc "["; acc_value (p, k); p:acc "] = " + end + + acc_value (p, v) -- Print the value + has_hash = true + end + end + end + + -- Now we know whether there's a hash-part, an array-part, and a tag. + -- Tag and hash-part are already printed if they're present. + if not has_tag and not has_hash and not has_arr then p:acc "{ }"; + elseif has_tag and not has_hash and not has_arr then -- nothing, tag already in acc + else + assert (has_hash or has_arr) -- special case { } already handled + local no_brace = false + if has_hash and has_arr then p:acc ", " + elseif has_tag and not has_hash and alen==1 and type(adt[1])~="table" then + -- No brace required; don't print "{", remember not to print "}" + p:acc (" "); acc_value (p, adt[1]) -- indent= indent+(cfg.fix_indent or 0)) + no_brace = true + elseif not has_hash then + -- Braces required, but not opened by hash-part handler yet + p:acc "{ "; indent(p) + end + + -- 2nd pass: array-part + if not no_brace and has_arr then + local expected_len = xlen(adt[1], p.cfg, p.nested) + consider_newline(p, expected_len) + acc_value(p, adt[1]) -- indent+(cfg.fix_indent or 0) + for i=2, alen do + p:acc ", "; + consider_newline(p, xlen(adt[i], p.cfg, p.nested)) + acc_value (p, adt[i]) --indent+(cfg.fix_indent or 0) + end + end + if not no_brace then p:acc " }" end + end + p.nested[adt] = false -- No more nested calls + p.indent = previous_indent +end + + +function acc_value(p, v) + local custom_string = __tostring(v) + if custom_string then p:acc(custom_string) else + local f = acc_type[type(v)] + if f then f(p, v) else p:acc(tostring(v)) end + end +end + + +-- FIXME: new_indent seems to be always nil?!s detection +-- FIXME: accumulator function should be configurable, +-- so that print() doesn't need to bufferize the whole string +-- before starting to print. +function M.tostring(t, cfg) + + cfg = cfg or M.DEFAULT_CFG or { } + + local p = { + cfg = cfg; + indent = 0; + current_offset = cfg.initial_indent or 0; + buffer = { }; + nested = { }; + acc = function(self, str) + table.insert(self.buffer, str) + self.current_offset = self.current_offset + #str + end; + } + acc_value(p, t) + return table.concat(p.buffer) +end + +function M.print(...) return print(M.tostring(...)) end +function M.sprintf(fmt, ...) + local args={...} + for i, v in pairs(args) do + local t=type(v) + if t=='table' then args[i]=M.tostring(v) + elseif t=='nil' then args[i]='nil' end + end + return string.format(fmt, unpack(args)) +end + +function M.printf(...) print(M.sprintf(...)) end + +return M \ No newline at end of file diff --git a/src/util/string b/src/util/string deleted file mode 160000 index ff9be4b9e..000000000 --- a/src/util/string +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ff9be4b9e4cd3c20e9714f2e513b69d96bb03b90 diff --git a/src/util/string/README.md b/src/util/string/README.md new file mode 100644 index 000000000..b4b6c3783 --- /dev/null +++ b/src/util/string/README.md @@ -0,0 +1,8 @@ +# stringutils + +Small collection of utf-8 string utils. +To use without LOVE2D, install `luautf8`: + +```sh +luarocks --local --lua-version 5.1 install luautf8 +``` diff --git a/src/util/string/string.lua b/src/util/string/string.lua new file mode 100644 index 000000000..dbd4f19ba --- /dev/null +++ b/src/util/string/string.lua @@ -0,0 +1,411 @@ +--- @diagnostic disable: duplicate-set-field +utf8 = utf8 or require((...):match("(.+)%.[^%.]+$") .. '.utf') + +--- @alias str string|string[] + +--- @param s string +--- @param p string +--- @param regex boolean? +--- @return boolean +string.matches = function(s, p, regex) + local r = not (regex or false) + local f = string.find(s, p, nil, r) + if f then return true end + return false +end + +--- @param s string +--- @param p string +--- @return boolean +string.matches_r = function(s, p) + return string.matches(s, p, true) +end + +--- @param s string +--- @param sub string +--- @return boolean +string.starts_with = function(s, sub) + local p = '^' .. sub + return string.matches_r(s, p) +end + +--- @param t string? +--- @return string? +string.debug_text = function(t) + if not t or type(t) ~= 'string' then return end + return string.format("'%s'", t) +end + +--- @param s string +--- @return string +string.normalize = function(s) + local r, _ = string.gsub(s, "%s+", "") + return r +end + +--- @param s string +--- @return string +string.trim = function(s) + if not s then return '' end + local pre = string.gsub(s, "^%s+", "") + local post = string.gsub(pre, "%s+$", "") + return post +end + +--- @param s string? +--- @param no_trim boolean? +--- @return boolean +string.is_non_empty_string = function(s, no_trim) + if type(s) == 'string' and s ~= '' then + local str = (function() + if no_trim then + return s + else + return string.normalize(s) + end + end)() + if str ~= '' then + return true + end + end + return false +end + +--- @param sa string[]? +--- @return boolean +string.is_non_empty_string_array = function(sa) + if type(sa) ~= 'table' then + return false + else + for _, s in ipairs(sa) do + if string.is_non_empty_string(s) then + return true + end + end + return false + end +end + +--- @param s str? +--- @param no_trim boolean? +string.is_non_empty = function(s, no_trim) + if type(s) == 'table' then + return string.is_non_empty_string_array(s) + elseif type(s) == 'string' then + return string.is_non_empty_string(s, no_trim) + end + return false +end + +--- @param s string +--- @return integer +string.ulen = function(s) + if s then + return utf8.len(s) + else + return 0 + end +end + +-- original from http://lua-users.org/lists/lua-l/2014-04/msg00590.html +--- @param s string +--- @param i integer +--- @param j integer? +--- @return string +string.usub = function(s, i, j) + i = i or 1 + j = j or -1 + if i < 1 or j < 1 then + local n = string.ulen(s) + if not n then return '' end + if i > n then return '' end + if i < 0 then i = n + 1 + i end + if j < 0 then + j = n + 1 + j + end + if i < 0 then i = 1 elseif i > n then i = n end + if j < 0 then + j = 1 + elseif j > n then + j = n + end + end + if j < i then return "" end + i = utf8.offset(s, i) + j = utf8.offset(s, j + 1) + if i and j then + return s:sub(i, j - 1) + elseif i then + return s:sub(i) + else + return "" + end +end + +--- @param s string +--- @param i integer +--- @return string +string.char_at = function(s, i) + return string.usub(s, i, i) +end + +--- @param s string +--- @param i integer +--- @return string +--- @return string +string.split_at = function(s, i) + local str = s or '' + local pre, post = '', '' + local ulen = string.ulen(str) + if ulen ~= #str then -- branch off for UTF-8 + pre = string.usub(str, 1, i - 1) + post = string.usub(str, i) + else + pre = string.sub(str, 1, i - 1) + post = string.sub(str, i, #str) + end + return pre, post +end + +--- @param s string +--- @param i integer +--- @return string[] +string.wrap_at = function(s, i) + if + not s or type(s) ~= 'string' or s == '' or + not i or type(i) ~= 'number' or i < 1 then + return { '' } + end + local len = string.ulen(s) or 0 + local mod = math.floor(i) + local n = math.floor(len / mod) + local res = {} + local chunk = '' + local rem = s + for _ = 1, n do + chunk, rem = string.split_at(rem, mod + 1) + table.insert(res, chunk) + end + if string.is_non_empty_string(rem, true) then + table.insert(res, rem) + end + + return res +end + +--- @param t string[] +--- @param i integer +--- @return string[] +string.wrap_array = function(t, i) + local res = {} + for _, s in ipairs(t) do + local ws = string.wrap_at(s, i) + for _, l in ipairs(ws) do + table.insert(res, l) + end + end + + return res +end + +-- https://stackoverflow.com/a/51893646 +--- @param str string +--- @param delimiter string +--- @return string[] +string.split = function(str, delimiter) + local del = delimiter or ' ' + if str and type(str) == 'string' then + if string.is_non_empty_string(str, true) then + local result = {} + local from = 1 + local delim_from, delim_to = string.find(str, del, from) + while delim_from do + table.insert(result, string.sub(str, from, delim_from - 1)) + from = delim_to + 1 + delim_from, delim_to = string.find(str, del, from) + end + table.insert(result, string.sub(str, from)) + return result + else + return { '' } + end + else + return {} + end +end + +--- @param str_arr string[] +--- @param char string +--- @return string[] +string.split_array = function(str_arr, char) + if not type(str_arr) == 'table' then return {} end + local words = {} + for _, line in ipairs(str_arr) do + if line == '' then + table.insert(words, line) + else + local ws = string.split(line, char) + for _, word in ipairs(ws) do + table.insert(words, word) + end + end + end + return words +end + +--- @param s str +--- @return string[] +string.lines = function(s) + if type(s) == 'string' then + return string.split(s, '\n') + end + if type(s) == 'table' then + return string.split_array(s, '\n') + end + return {} +end + +--- @param strs str +--- @param char string? +--- @return string +string.join = function(strs, char) + local res = '' + if type(strs) == 'table' then + local j = char or ' ' + for i, word in ipairs(strs) do + -- TODO recursive join + if type(word) == 'string' then + res = res .. word + end + if i ~= #strs then + res = res .. j + end + end + end + if type(strs) == 'string' then + res = strs + end + return res +end + +--- @param strs str +--- @return string +string.unlines = function(strs) + return string.join(strs, '\n') +end + +--- @param prefix string +--- @param text string +--- @param postfix string +--- @return string +string.interleave = function(prefix, text, postfix) + return string.join({ prefix, postfix }, text) +end + +--- @param t string +--- @return string? +string.quote = function(t) + if not t or type(t) ~= 'string' then return end + return string.format("'%s'", t) +end + +--- Split a string into three around specified indices +--- @param str string +--- @param si number +--- @param ei number +--- @return string +--- @return string +--- @return string +string.splice = function(str, si, ei) + if not str or type(str) ~= 'string' + or not string.is_non_empty_string(str, true) or si > ei then + return '', '', '' + end + local l = string.ulen(str) + local start = si or 1 + local fin = ei or l + local split1 = start + 1 + local split2 = fin - start + 1 + local pre, rem = string.split_at(str, split1) + local mid, post = string.split_at(rem, split2) + return pre, mid, post +end + +--- @param s string +--- @param n number +--- @return string? +string.times = function(s, n) + local till = n or 1 + if type(till) ~= 'number' then return end + local str = s or '' + local res = '' + for _ = 1, till do + res = res .. str + end + return res +end + +---------------------------- +--- validation utilities --- +---------------------------- + +Char = { + --- 'c' is assumed to be a single character/grapheme, these + --- functions won't be checking for it. + + --- @param c string + --- @return boolean + is_alpha = function(c) + return string.match(c, "%a") ~= nil + end, + --- @param c string + --- @return boolean + is_alnum = function(c) + return string.match(c, "%w") ~= nil + end, + --- @param c string + --- @return boolean + is_upper = function(c) + return string.match(c, "%u") ~= nil + end, + --- @param c string + --- @return boolean + is_lower = function(c) + return string.match(c, "%l") ~= nil + end, + --- @param c string + --- @return boolean + is_digit = function(c) + return string.match(c, "%d") ~= nil + end, + --- @param c string + --- @return boolean + is_space = function(c) + return string.match(c, "%s") ~= nil + end, + --- @param c string + --- @return boolean + is_punct = function(c) + return string.match(c, "%p") ~= nil + end, + --- @param c string + --- @return boolean + is_ascii = function(c) + local byte = string.byte(c, 1) + return byte < 128 + end +} + +--- @param s string +--- @param f fun(string): boolean +--- @return boolean +--- @return integer? +string.forall = function(s, f) + for i = 1, string.ulen(s) do + local v = string.usub(s, i, i) + if not f(v) then + return false, i + end + end + return true +end diff --git a/src/util/string/utf.lua b/src/util/string/utf.lua new file mode 100644 index 000000000..3258df855 --- /dev/null +++ b/src/util/string/utf.lua @@ -0,0 +1,7 @@ +return (function() + if _VERSION == 'Lua 5.1' and not love then + return require("lua-utf8") + else + return require("utf8") + end +end)() diff --git a/tests/interpreter/ast_inputs.lua b/tests/interpreter/ast_inputs.lua new file mode 100644 index 000000000..cf79b4ab8 --- /dev/null +++ b/tests/interpreter/ast_inputs.lua @@ -0,0 +1,1388 @@ +--- @param s string|string[] +--- @param canonized string|string[]? +--- @return table {string[], string[]} +local prep = function(s, canonized) + local orig = (function() + if type(s) == 'string' then + return string.lines(s) + elseif type(s) == 'table' then + local ret = {} + for i, v in ipairs(s) do + ret[i] = v + end + return ret + end + end)() + local canon = canonized and string.lines(canonized) or orig + + return { orig, canon } +end + +local sierpinski = [[function sierpinski(depth) + lines = { '*' } + for i = 2, depth + 1 do + sp = string.rep(' ', 2 ^ (i - 2)) + tmp = {} -- comment + for idx, line in ipairs(lines) do + tmp[idx] = sp .. line .. sp + tmp[idx + #lines] = line .. ' ' .. line + end + lines = tmp + end + return table.concat(lines, '\n') +end + +print(sierpinski(4))]] + +local sierpinski_res = { + 'function sierpinski(depth)', + -- --- [[ ]] version + -- ' lines = { [[*]] }', + ' lines = { "*" }', + + ' for i = 2, depth + 1 do', + -- --- [[ ]] version + -- ' sp = string.rep([[ ]], 2 ^ (j - 2))', + ' sp = string.rep(" ", 2 ^ (i - 2))', + + ' tmp = { }', + ' -- comment', + ' for idx, line in ipairs(lines) do', + -- ' tmp[idx] = sp .. (line .. sp)', + ' tmp[idx] = sp .. line .. sp', + -- --- [[ ]] version + -- ' tmp.add = line .. ([[ ]] .. line)', + --- TODO what's up with the paren + -- ' tmp[idx + #lines] = line .. (" " .. line)', + ' tmp[idx + #lines] = line .. " " .. line', + + ' end', + ' lines = tmp', + ' end', + -- --- [[ ]] version + -- ' return table.concat(lines, [[', + -- ']])', + -- --- '' version + -- " return table.concat(lines, '\\n')", + --- "" version + ' return table.concat(lines, "\\n")', + 'end', + -- '', + 'print(sierpinski(4))', +} + +local meta = +[[ +--- @param node token +--- @return table +function M:extract_comments(node) + local lfi = node.lineinfo.first + local lla = node.lineinfo.last + local comments = {} + + --- @param c table + --- @param pos 'first'|'last' + local function add_comment(c, pos) + local idf = c.lineinfo.first.id + local idl = c.lineinfo.last.id + local present = self.comment_ids[idf] or self.comment_ids[idl] + if not present then + local comment_text = c[1] + local len = string.len(comment_text) + local n_l = #(string.lines(comment_text)) + local cfi = c.lineinfo.first + local cla = c.lineinfo.last + local cfirst = { l = cfi.line, c = cfi.column } + local clast = { l = cla.line, c = cla.column } + local off = cla.offset - cfi.offset + local d = off - len + local l_d = cla.line - cfi.line + local newline = (n_l ~= 0 and n_l == l_d) + local li = { + idf = idf, + idl = idl, + first = cfirst, + last = clast, + text = comment_text, + multiline = (d > 4), + position = pos, + prepend_newline = newline + } + self.comment_ids[idf] = true + self.comment_ids[idl] = true + table.insert(comments, li) + end + end + if lfi.comments then + for _, c in ipairs(lfi.comments) do + add_comment(c, 'first') + end + end + if lla.comments then + for _, c in ipairs(lla.comments) do + add_comment(c, 'last') + end + end + + return comments +end +]] +local meta_res = { + '--- @param node token', + '--- @return table', + --- functions are values + --- 'M.extract_comments = function(self, node)', + --- self syntax sugar + 'function M:extract_comments(node)', + -- 'function M.extract_comments(self, node)', + ' local lfi = node.lineinfo.first', + ' local lla = node.lineinfo.last', + ' local comments = { }', + -- '', + ' --- @param c table', + " --- @param pos 'first'|'last'", + ' local function add_comment(c, pos)', + ' local idf = c.lineinfo.first.id', + ' local idl = c.lineinfo.last.id', + -- ' local present = self.comment_ids[idf] or self.comment_ids[idl]', + ' local present = self.comment_ids[idf]', + ' or self.comment_ids[idl]', + ' if not present then', + ' local comment_text = c[1]', + ' local len = string.len(comment_text)', + ' local n_l = #(string.lines(comment_text))', + ' local cfi = c.lineinfo.first', + ' local cla = c.lineinfo.last', + --- splits tables + ' local cfirst = {', + ' l = cfi.line,', + ' c = cfi.column', + ' }', + ' local clast = {', + ' l = cla.line,', + ' c = cla.column', + ' }', + ' local off = cla.offset - cfi.offset', + ' local d = off - len', + ' local l_d = cla.line - cfi.line', + --- normalizes logic conditions + ' local newline = (n_l ~= 0 and n_l == l_d)', + ' local li = {', + ' idf = idf,', + ' idl = idl,', + ' first = cfirst,', + ' last = clast,', + ' text = comment_text,', + ' multiline = (4 < d),', + ' position = pos,', + ' prepend_newline = newline', + ' }', + ' self.comment_ids[idf] = true', + ' self.comment_ids[idl] = true', + ' table.insert(comments, li)', + ' end', + ' end', + ' if lfi.comments then', + ' for _, c in ipairs(lfi.comments) do', + ' add_comment(c, "first")', + ' end', + ' end', + ' if lla.comments then', + ' for _, c in ipairs(lla.comments) do', + ' add_comment(c, "last")', + ' end', + ' end', + -- '', + ' return comments', + 'end', +} + +local clock = { + 'love.draw = function()', + ' draw()', + 'end', + '', + 'function love.update(dt)', + ' t = t + dt', + ' s = math.floor(t)', + ' if s > midnight then s = 0 end', + 'end', + '', + 'function cycle(c)', + ' if c > 7 then return 1 end', + ' return c + 1', + 'end', + '', + 'love.keyreleased = function (k)', + ' if k == \'space\' then', + ' if love.keyboard.isDown("lshift", "rshift") then', + ' bg_color = cycle(bg_color)', + ' else', + ' color = cycle(color)', + ' end', + ' end', + ' if k == \'s\' then', + ' stop(\'STOP THE CLOCKS!\')', + ' end', + 'end' } +local clock_res = { + --- functions are values + -- 'love.draw = function()', + --- syntax sugar + 'function love.draw()', + ' draw()', + 'end', + -- '', + --- functions are values + -- 'love.update = function(dt)', + --- syntax sugar + 'function love.update(dt)', + ' t = t + dt', + ' s = math.floor(t)', + -- canonized compare order + ' if midnight < s then', + ' s = 0', + ' end', + 'end', + -- '', + --- functions are values + -- 'cycle = function(c)', + --- syntax sugar + 'function cycle(c)', + -- canonized compare order + ' if 7 < c then', + ' return 1', + ' end', + ' return c + 1', + 'end', + -- '', + --- functions are values + -- 'love.keyreleased = function(k)', + --- syntax sugar + 'function love.keyreleased(k)', + ' if k == "space" then', + ' if love.keyboard.isDown("lshift", "rshift") then', + ' bg_color = cycle(bg_color)', + ' else', + ' color = cycle(color)', + ' end', + ' end', + ' if k == "s" then', + ' stop("STOP THE CLOCKS!")', + ' end', + 'end', +} + +local basics = { + prep('local str = "asd"'), + prep("local str = 'asd'", { 'local str = "asd"' }), + prep('local n = 3.0e2', 'local n = 300'), +} + +local operators = { + prep('local x = 2 + 3'), + prep('local x = (2 + 3) * 5'), + prep('local x = 2 + 3 * 5'), + prep('x = 3 * 5 + 6 * 9'), + prep('x = 3 + 5 * 6 + 9'), + prep('x = 3 * 5 * 6 * 9'), + prep('p = 4 ^ 2 ^ 3', 'p = 4 ^ (2 ^ 3)'), + prep('x = 2 + - 1', 'x = 2 + -1'), + prep('unm = -10'), + prep('y = 15 / 5 * 3', 'y = (15 / 5) * 3'), + prep('x = 3 - 2 - 1', 'x = (3 - 2) - 1'), + prep('G.setColor(Color[color + Color.bright])'), + prep('G.setColor(Color[(color + Color.bright)])'), + prep('v = t[i + 3]'), + prep('v = t[(i + 3)]'), + prep('local v = #t'), + prep('v = #t'), + prep('tmp[idx + #lines] = line .. (" " .. line)'), + prep('con_line = line .. " " .. line'), + prep('bool = true and false or true'), + prep('bool = not false or false'), + prep('bool = x ~= y'), + prep('bool = not (x == y)', 'bool = x ~= y'), + prep('len = #t1'), + prep('len = #t1.t2', 'len = #(t1.t2)'), + prep('len = #(t1.t2)'), + prep('len = #V:get_text()', 'len = #(V:get_text())'), + prep('len = #V:get_input():get_text()', + 'len = #(V:get_input():get_text())'), + prep('len = # "asd" .. "vfds"', 'len = #"asd" .. "vfds"'), + prep('len = # ("asd" .. "vfds")', 'len = #("asd" .. "vfds")'), + prep('bool = 2 >= 3 ~= 4 < 5', 'bool = (3 <= 2) ~= 4 < 5'), +} + +local comments = { + prep({ + '--- comment', + }), + prep({ + 'y = 10', + '--- comment', + 'z = 99' + }), + + prep({ + 'y = 10', + '--- comment1', + '--- comment2', + '--- comment3', + 'z = 99' + }), + + prep({ + 'x = 0', + '-- comment1', + '-- comment2', + '-- comment3', + 'a = 1' + }), + + prep({ + 'x = 1', + '--[[ comment1', + ' comment2', + ' comment3]]', + 'a = 3' + }), + + prep({ + 'x = 0', + '--[[ comment1', + ' comment2]]--', + 'a = 2', + }, { + 'x = 0', + '--[[ comment1', + ' comment2]]', + '--', --- this is canonical now, no following `--` after `]]` + 'a = 2', + }), + + prep({ + 'x = 0', + '--[[ comment1', + ' comment2]]', + 'a = 2', + }, { + 'x = 0', + '--[[ comment1', + ' comment2]]', + 'a = 2', + }), + + prep({ + 'x = 0', + '--[[line1', + 'line2--]]', + 'a = 2', + } + -- , { + -- 'x = 0', + -- '--[[ comment1', + -- ' comment2]]', + -- 'a = 2', + -- } + ), + + prep({ + 'x = 0', + '-- comment1', + '-- comment2', + 'a = 2', + }), + + prep([[-- comment]]), + prep({ '-- interesting comment', 'a = 1' }), + prep({ + '-- comment1', + '-- comment2' + }), + + prep({ + '--[[ comment1', + ' comment2--]]', + 'a = 2', + '-- asd', -- canonical space after comment marker + '--[[]]', + }), + prep({ '--[[]]', 'a = 1' }), + prep({ 'a = 1', '--[[]]' }), + prep({ + '-- comment1', + '-- comment2', + 'a = 1' + }), + prep({ + 'local head_r = 8', + 'local leg_r = 5', + '-- move offsets', + 'local x_r = 15', + 'local y_r = 20', + }), + prep({ + 'y = 3', + '--[[', + 'multi-', + 'line', + 'comment]]', + }), + prep({ '--[[', + 'multi-', + 'line', + 'comment]]', + }), + prep({ '--[[multi-', + 'line', + 'comment', + ']]', + }), + prep({ '--[[multi-', + 'line', + 'comment]]', + }), + prep({ + '--[[ comment1', + ' comment2]]--', + 'a = 2', + '--asd', + '--[[]]', + }, { + '--[[ comment1', + ' comment2]]', + '--', --- this is canonical now, no following `--` after `]]` + 'a = 2', + '-- asd', -- canonical space after comment marker + '--[[]]', + }), + prep({ + 'local x = 2', + '--[[', + 'multi-', + 'line', + 'comment]]--', + }, + { + 'local x = 2', + '--[[', + 'multi-', + 'line', + 'comment]]', + '--', --- the proper mlc closing is `--]]` + } + ), + prep({ 'a = 3 --cmt', + }, { + 'a = 3', + '-- cmt', + }), + + prep({ + 'function fun()', + ' -- inline', + ' ', + 'end', + }), + prep({ + 'love.draw = function()', + ' -- f', + 'end', + }, { + 'function love.draw()', + ' -- f', + ' ', + 'end', + }), + + prep({ + '', + '-- emptyline before comment must be preserved', + 'function foo(bar)', + ' return ("no comment duplication should happen")', + 'end' + }), + + prep({ + '', + '', + '-- multiple emptylines before comment must be squashed', + }, { + '', + '-- multiple emptylines before comment must be squashed', + }) +} + +local emptylines = { + + -- standalone comment + + prep({'--- standalone comment does not enforce emptylines'}), + prep({'', + '--- standalone comment preserves emptyline before it' + }), + prep({ + '', + '', + '', + '--- standalone comment keeps just one emptyline before it' + },{ + '', + '--- standalone comment keeps just one emptyline before it' + }), + + -- standalone expression + + prep({ + 'print("standalone expression does not enforce emptylines")' + }), + prep({ + '', + '', + '', + 'print("standalone expression eliminates emptylines before it")' + }, { + 'print("standalone expression eliminates emptylines before it")' + }), + + --- expression+comment + + prep({ + '-- comment before expression does not enforce emptylines', + 'print("expectation: no emptylines are injected")', + }), + prep({ + '', + '', + '', + '-- comment before expression allows one emptyline before it', + 'print("expectation: exactly one emptyline is preserved")', + }, { + '', + '-- comment before expression allows one emptyline before it', + 'print("expectation: exactly one emptyline is preserved")', + }), + + -- expressions (multiple) + + prep({ + 'print("rule: no gap between adjacent statemets is enforced")', + 'print("expecting: no emptyline should be injected")' + }), + prep({ + 'print("rule: gap between adjacent statements is eliminated")', + '', + '', + '', + 'print("expecting: no emptylines preserved")' + }, { + 'print("rule: gap between adjacent statements is eliminated")', + 'print("expecting: no emptylines preserved")' + }), + + -- expressions (multiple) + comment + + prep({ + 'print("rule: comment between statements does not preserve gap")', + '', + '-- comment between statements does not preserve gap', + 'print("expecting: emptyline is not preserved")' + }, { + 'print("rule: comment between statements does not preserve gap")', + '-- comment between statements does not preserve gap', + 'print("expecting: emptyline is not preserved")' + }), + + -- standalone block + + prep({ + 'function rule()', + ' return ("standalone block does not emit emptylines")', + 'end' + }), + prep({ + '', + '', + '', + 'function rule()', + ' return ("standalone block erases emptylines before it")', + 'end' + }, { + 'function rule()', + ' return ("standalone block erases emptylines before it")', + 'end' + }), + + -- standalone block + comment + + prep({ + '-- comment before block', + 'function rule()', + ' return ("comment before block does not emit emptylines")', + 'end' + }), + prep({ + '', + '', + '', + '-- comment before block', + 'function rule()', + ' return ("comment before block keeps 1 emptyline before it")', + 'end' + }, { + '', + '-- comment before block', + 'function rule()', + ' return ("comment before block keeps 1 emptyline before it")', + 'end' + }), + + -- multiple blocks + comment in between + + prep({ + 'function rule()', + ' return ("no gaps between blocks are enforced")', + 'end', + '-- comment in between', + 'function expectation()', + ' return ("no emptylines are injected")', + 'end', + }), + prep({ + 'function rule()', + ' return ("no gaps between blocks are preserved")', + 'end', + '', + '-- comment in between', + '', + 'function expectation()', + ' return ("no emptylines are kept")', + 'end', + }, { + 'function rule()', + ' return ("no gaps between blocks are preserved")', + 'end', + '-- comment in between', + 'function expectation()', + ' return ("no emptylines are kept")', + 'end', + }) +} + +local wrapping = { + prep('local t = { b = 2, 3, 4 }', { + 'local t = {', + ' b = 2,', + ' 3,', + ' 4', + '}' }), + prep('a = 1 ; b = 2', { 'a = 1', 'b = 2' }), + --- string literals and comments + prep( + 'local long_string = "яяяяяяяяяяяяяяяяяяя22222222222222222eeeeeeeeeeeeeeeeeee6666666666666666666666666sssssssssss"', + { + 'local long_string = ', + ' "яяяяяяяяяяяяяяяяяяя22222222222222222eeeeeeeeeeeeeeeeeee66" ..', + ' "66666666666666666666666sssssssssss"', + }), + prep( + '-- яяяяяяяяяяяяяяяяяяя22222222222222222eeeeeeeeeeeeeeeeeee6666666666666666666666666sssssssssss', + { + '-- яяяяяяяяяяяяяяяяяяя22222222222222222eeeeeeeeeeeeeeeeeee66666', + '-- 66666666666666666666sssssssssss', + }), + prep( + { + '--[[Bacon ipsum dolor amet sint meatball pork loin, shankle kiel', + 'basa nulla mollit quis elit dolore tenderloin swine.', + 'Elit beef pancetta, lorem sirloin spare ribs tenderloin exercitation laborum tongue eiusmod dolor fatback.', + 'In ut dolore corned beef flank eiusmod, burgdoggen capicola ham enim culpa hamburger chuck. Beef burgdoggen qui meatloaf cupidatat sunt. Lorem spare ribs dolor mollit porchetta. Nostrud pig shoulder beef veniam shank pork loin landjaeger chuck ball tip.', + 'Tri-tip elit culpa deserunt.]]' }, + { + '--[[Bacon ipsum dolor amet sint meatball pork loin, shankle kiel', + 'basa nulla mollit quis elit dolore tenderloin swine.', + 'Elit beef pancetta, lorem sirloin spare ribs tenderloin exercita', + 'tion laborum tongue eiusmod dolor fatback.', + 'In ut dolore corned beef flank eiusmod, burgdoggen capicola ham ', + 'enim culpa hamburger chuck. Beef burgdoggen qui meatloaf cupidat', + 'at sunt. Lorem spare ribs dolor mollit porchetta. Nostrud pig sh', + 'oulder beef veniam shank pork loin landjaeger chuck ball tip.', + 'Tri-tip elit culpa deserunt.]]', + }), + prep({ + '-- яяяяяяяяяяяяяяяяяяя22222222222222222eeeeeeeeeeeeeeeeeee6666666666666666666666666sssssssssss', + '-- цэфлаэфцжфдэжафдукзщфкхз2щ3х54з2ьахажщд2хфладжьяхадыхжахдхыжхахдыалджлождлод' }, + { + '-- яяяяяяяяяяяяяяяяяяя22222222222222222eeeeeeeeeeeeeeeeeee66666', + '-- 66666666666666666666sssssssssss', + '-- цэфлаэфцжфдэжафдукзщфкхз2щ3х54з2ьахажщд2хфладжьяхадыхжахдхыж', + '-- хахдыалджлождлод', + }), + prep({ + 'function fun()', + ' doSomething() -- very long comment that will go over the line length', + 'end', + }, { + 'function fun()', + ' doSomething()', + ' -- very long comment that will go over the line length', + 'end' } + ), + prep({ + 'local s = [[asd', + 'string', + ']]', + }, { [[local s = "asd\nstring\n"]] }), + prep({ + 'local s = [[asd', + 'string', + '', + '', + ']]', + }, { [[local s = "asd\nstring\n\n\n"]] }), + + prep({ + 'local ms= [[█Bacon ipsum dolor amet ribeye hamburger', + 'c█hislic pork short ribs', + 'po█rchetta. Pork loin meatball ball tip', + 'por█k chop pork capicola fatback andouille beef sausage short', + 'loin█ bresaola venison.\\t]]', + }, + -- --- [[ ]] version + -- { + -- 'local ms = [[█Bacon ipsum dolor amet ribeye hamburger', + -- 'c█hislic pork short ribs', + -- 'po█rchetta. Pork loin meatball ball tip',Debug.text(comment_text) + -- } + --- " " version + { + 'local ms = ', + [[ "█Bacon ipsum dolor amet ribeye hamburger\n" ..]], + [[ "c█hislic pork short ribs\n" ..]], + [[ "po█rchetta. Pork loin meatball ball tip\n" ..]], + [[ "por█k chop pork capicola fatback andouille beef sausage s" ..]], + [[ "hort\n" ..]], + [[ "loin█ bresaola venison.\t"]], + } + ), + prep({ + 'local str = "asd\\nbgf"', + 'local mstr = [[rty', + 'qwe]]', + 'local ms = [[ms]]', + 'local m_s = [[m\ns]]', + }, + -- --- [[ ]] version + -- { + -- 'local str = [[asd', + -- 'bgf]]', + -- 'local mstr = [[rty', + -- 'qwe]]', + -- 'local ms = "ms"', + -- } + --- " " version + { + [[local str = "asd\nbgf"]], + [[local mstr = "rty\nqwe"]], + 'local ms = "ms"', + [[local m_s = "m\ns"]], + } + ), + --- compound conditions + prep({ + 'if type(w) ~= "number" or w < 1 then', + ' fun()', + 'end', + }), + prep({ + 'if type(w) ~= "number"', + ' or w < 1 then', + ' fun()', + 'end', + }, { + 'if type(w) ~= "number"', + ' or w < 1', + 'then', + ' fun()', + 'end', + }), + prep({ + 'local f = function()', + ' for k, v in pairs(x) do', + ' if love.keyboard.isDown("lshift", "rshift") and not love.keyboard.isDown("lalt", "ralt") then', + ' bg_color = cycle(bg_color)', + ' else', + ' bg_color = Color.blue', + ' end', + ' end', + 'end', + }, { + 'local f = function()', + ' for k, v in pairs(x) do', + --- wrong: + -- ' if love.keyboard.isDown("lshift", "rshift") and not love.' + -- ' keyboard.isDown("lalt", "ralt") then' + ' if love.keyboard.isDown("lshift", "rshift")', + ' and not love.keyboard.isDown("lalt", "ralt")', + ' then', + ' bg_color = cycle(bg_color)', + ' else', + ' bg_color = Color.blue', + ' end', + ' end', + 'end', + } + ), + prep({ + 'local f = function()', + ' for k, v in pairs(x) do', + ' if love.keyboard.isDown("lshift", "rshift") and not love.keyboard.isDown("lalt", "ralt") and not love.keyboard.isDown("lctrl", "rctrl") then', + ' bg_color = cycle(bg_color)', + ' else', + ' bg_color = Color.blue', + ' end', + ' end', + 'end', + }, { + 'local f = function()', + ' for k, v in pairs(x) do', + ' if love.keyboard.isDown("lshift", "rshift")', + --- wrong: + -- ' if love.keyboard.isDown("lshift", "rshift") and not love.' + -- ' keyboard.isDown("lalt", "ralt") and not love.keyboard.isDown(' + -- ' "lctrl", "rctrl") then' + ' and not love.keyboard.isDown("lalt", "ralt")', + ' and not love.keyboard.isDown("lctrl", "rctrl")', + ' then', + ' bg_color = cycle(bg_color)', + ' else', + ' bg_color = Color.blue', + ' end', + ' end', + 'end', + } + ), + prep({ + 'function M:extract_comments(node)', + ' local function add_comment(c, pos)', + ' local idf = c.lineinfo.first.id', + ' local idl = c.lineinfo.last.id', + ' local present = self.comment_ids[idf] or self.comment_ids[idl]', + ' end', + 'end', + }, + { + 'function M:extract_comments(node)', + ' local function add_comment(c, pos)', + ' local idf = c.lineinfo.first.id', + ' local idl = c.lineinfo.last.id', + -- ' local present = self.comment_ids[idf] or self.comment_ids[' + -- 'idl]', + ' local present = self.comment_ids[idf]', + ' or self.comment_ids[idl]', + ' end', + 'end', + }), + prep({ + 'function M:extract_comments(node)', + ' local function add_comment(c, pos)', + ' local idf = c.lineinfo.first.id', + ' local idl = c.lineinfo.last.id', + ' local present = self.comment_ids[idf]', + ' or self.comment_ids[idl]', + ' end', + 'end', + }), + + --- complicated calculations that should probably be broken up + prep('local longcomp = (3749182734 + 1928340918 - 239420985) * (274927 + 820479 - 2973842)', { + 'local longcomp = (3749182734 + 1928340918 - 239420985) * ', + ' (274927 + 820479 - 2973842)', + }), + prep('local longcomp2 = 1001 + 1002 + 1003 + 1004 + 1005 + 1006 + 1007 + 1008 + 1009', { + 'local longcomp2 = 1001 + 1002 + 1003 + 1004 + 1005 + 1006 + 1007', + ' + 1008 + 1009' + }), + + --- multi-assignments that again, probably should be broken up + prep('local declaring, a, lot, of, variables, in_, one, go = 1, 2, 3, 4, 5, 6, 7, 8', { + 'local declaring, a, lot, of, variables, in_, one, go = 1, 2, 3, ', + ' 4, 5, 6, 7, 8' }), + prep({ + 'local declaring, a, lot, of, ', + ' variables, in_, one, go = 1, 2, 3,', + ' 4, 5, 6, 7, 8', + } + , + { + 'local declaring, a, lot, of, variables, in_, one, go = 1, 2, 3, ', + ' 4, 5, 6, 7, 8' } + ), + + prep( + 'local assigning, an, amount, of, variables, that, cannot, possibly, fit, on, one, line = 101, 102, 103, 4, 5, 6, 7, 8, 9, 10, 11, 12' + , { + 'local assigning, an, amount, of, variables, that, cannot, ', + ' possibly, fit, on, one, line = 101, 102, 103, 4, 5, 6, 7, 8, 9, 10, ', + ' 11, 12', } + ), + prep({ + 'Globally, declaring, a, lot, of, ', + ' variables, in_, one, go = 1, 2, 3,', + ' 4, 5, 6, 7, 8', + } + , + { + 'Globally, declaring, a, lot, of, variables, in_, one, go = 1, 2', + ' , 3, 4, 5, 6, 7, 8' } + ), + + --- returning too many values + prep({ + 'function ret_gaming()', + ' return a, ridiculous, amouns, of, named, values, cannot, possibly, fit', + 'end' }, { + 'function ret_gaming()', + ' return a, ridiculous, amouns, of, named, values, cannot, ', + ' possibly, fit', + 'end' } + ), + + --- Very Large Numbers + prep( + 'N = 398492087598247598237529834759827345928375928734958729387459283787459837452987345982375', + 'N = 3.9849208759825e+86' + ), + + + --- functions with way too many parameters + prep({ + 'function fun(copious, amounts, of, parameters, which, cannot, possibly, fit)', + ' doSomething()', + 'end', + } + , + --- naiive solution + -- { + -- 'function fun(copious, amounts, of, parameters, which, cannot, ', + -- ' possibly, fit)', + -- ' doSomething()', + -- 'end', + -- }, + { + 'function fun(', + ' copious,', + ' amounts,', + ' of,', + ' parameters,', + ' which,', + ' cannot,', + ' possibly,', + ' fit', + ')', + ' doSomething()', + 'end', + } + ), + prep({ + 'function O:method(copious, amounts, of, parameters, which, cannot, possibly, fit)', + ' methodBody()', + 'end', + }, + { + 'function O:method(', + ' copious,', + ' amounts,', + ' of,', + ' parameters,', + ' which,', + ' cannot,', + ' possibly,', + ' fit', + ')', + ' methodBody()', + 'end', + } + ), + prep({ + 'local function localfun(inordinate, amounts, of, parameters, which, cannot, possibly, fit)', + ' localFunBody()', + 'end', + }, + { + 'local function localfun(', + ' inordinate,', + ' amounts,', + ' of,', + ' parameters,', + ' which,', + ' cannot,', + ' possibly,', + ' fit', + ')', + ' localFunBody()', + 'end', + } + ), + + prep({ + 'local ft = {', + ' zx = 3,', + ' tablefun = function(inordinate, amounts, of, parameters, which, cannot, and_, will, not_, fit)', + ' ', + ' end', + '}', + }, + { + 'local ft = {', + ' zx = 3,', + ' tablefun = function(', + ' inordinate,', + ' amounts,', + ' of,', + ' parameters,', + ' which,', + ' cannot,', + ' and_,', + ' will,', + ' not_,', + ' fit', + ' )', + ' ', + ' end', + -- ' ', + '}', + } + ), + + prep({ + 'local t1 = {', + ' a1 = 1,', + ' t2 = {', + ' a2 = 2,', + ' t3 = {', + ' a3 = 3,', + ' t4 = {', + ' tablefun = function(deeply, nested, tables, lots, of, params)', + ' ', + ' end', + ' }', + ' }', + ' }', + '}', + }, { + 'local t1 = {', + ' a1 = 1,', + ' t2 = {', + ' a2 = 2,', + ' t3 = {', + ' a3 = 3,', + ' t4 = {', + ' tablefun = function(', + ' deeply,', + ' nested,', + ' tables,', + ' lots,', + ' of,', + ' params', + ' )', + ' ', + ' end', + -- ' ', + ' }', + ' }', + ' }', + '}', + }), + + prep({ + 'local call = localfun(inordinate, amounts, of, parameters, which, cannot, possibly, fit)', + }, { + 'local call = localfun(', + ' inordinate,', + ' amounts,', + ' of,', + ' parameters,', + ' which,', + ' cannot,', + ' possibly,', + ' fit', + ')', + } + ), + prep({ + 'local invoke = O:method(inordinate, amounts, of, parameters, which, cannot, possibly, fit)', + }, { + 'local invoke = O:method(', + ' inordinate,', + ' amounts,', + ' of,', + ' parameters,', + ' which,', + ' cannot,', + ' possibly,', + ' fit', + ')', + } + ), + + --- nested calls + prep('local computedValue = function1(function2(function3(function4(function5()))))', { + 'local computedValue = function1(', + ' function2(function3(function4(function5())))', + ')', + }), + prep('local computedValue = there(really(should(be(a(pipe(operator()))))))', { + 'local computedValue = there(', + ' really(should(be(a(pipe(operator())))))', + ')', + }), + prep( + 'local computedValue = there(really(should(be(a(pipe(operator(so, this, could, go, more, smoothly, and_, in_, a, readable, fashion)))))))', + { + 'local computedValue = there(really(should(be(a(pipe(operator(', + ' so,', + ' this,', + ' could,', + ' go,', + ' more,', + ' smoothly,', + ' and_,', + ' in_,', + ' a,', + ' readable,', + ' fashion', + ')))))))', + }), +} + +local functions = { + prep({ 'fun(1)' }), + prep({ 'fun(1)', 'fun(3)' }), + prep({ + 'love.draw = function()', + ' draw()', + 'end', + }, { + 'function love.draw()', + ' draw()', + 'end', + }), + prep({ + 'function love.draw()', + ' draw()', + 'end', + }), + prep('local function x() end', { + 'local function x()', + ' ', + 'end', + }), + prep('local x = function() end', { + 'local x = function()', + ' ', + 'end', + --- TODO #46 consistent function sugar + -- 'local function x()', + -- ' ', + -- 'end', + } + ), + prep('x["y"] = function(a) end', { + 'function x.y(a)', + ' ', + 'end', + }), + prep('x[1] = function() end', { + 'x[1] = function()', + ' ', + 'end', + }), + prep('x[y][z] = a'), +} +local self = { + prep({ + '--- @param t string|string[]', + 'function InputController:set_text(t)', + ' self.model:set_text(t)', + 'end', + }), + prep({ + '--- @protected', + '--- @param w integer', + '--- @param text string[]?', + 'function WrappedText:_init(w, text)', + ' if type(w) ~= "number" or w < 1 then', + " error('invalid wrap length')", + ' end', + ' self.text = {}', + ' self.wrap_w = w', + ' self.wrap_forward = {}', + ' self.wrap_reverse = {}', + ' self.n_breaks = 0', + ' if text then', + ' self:wrap(text)', + ' end', + 'end', + }, { + '--- @protected', + '--- @param w integer', + '--- @param text string[]?', + 'function WrappedText:_init(w, text)', + ' if type(w) ~= "number" or w < 1 then', + ' error("invalid wrap length")', + ' end', + ' self.text = { }', + ' self.wrap_w = w', + ' self.wrap_forward = { }', + ' self.wrap_reverse = { }', + ' self.n_breaks = 0', + ' if text then', + ' self:wrap(text)', + ' end', + 'end', + } + ), + prep({ + '--- @return Range', + 'function VisibleContent:get_range()', + ' return self.range', + 'end', + }), + + prep({ + 'local t = {', + ' draw = function(value)', + ' V:dibujar()', + ' end', + -- ' ', + '}' + }), + prep({ + 'local t = {', + ' draw = function(value)', + ' V:dibujar()', + ' end', + -- ' ', + '}' + }), + prep({ + 'if a == 2 then', + ' function love.draw(sugar)', + ' V:dibujar()', + ' end', + -- ' ', + 'end', + }), + prep({ + 'if a == 2 then', + ' love.draw = function(value)', + ' V:dibujar()', + ' end', + -- ' ', + 'end', + }, { + 'if a == 2 then', + ' function love.draw(value)', + -- ' love.draw = function(value)', + ' V:dibujar()', + ' end', + -- ' ', + 'end', + }), + + prep({ + 'function love.draw(sugar)', + ' V:dibujar()', + 'end', + }), + prep({ + 'love.draw = function(value)', + ' V:dibujar()', + 'end', + }, { + 'function love.draw(value)', + -- ' love.draw = function(value)', + ' V:dibujar()', + 'end', + }), +} + +local canon = { + prep('if a > b then return a else return b end', { + 'if b < a then', + ' return a', + 'else', + ' return b', + 'end' } + ), + + prep({ + 'local draw = function() x:draw() end', + }, { + 'local draw = function()', + ' x:draw()', + 'end', + }), + --- operators + prep('if not (a == b) then end', + { + 'if a ~= b then', + ' ', + 'end' + }), + + prep(' x = 2', 'x = 2'), + prep('x["y"] = 2', 'x.y = 2'), + prep({ + [[function inPaletteRange(x, y) + return + (height - pal_h <= y and width - pal_w <= x and x <= width) + end]] + }, { + 'function inPaletteRange(x, y)', + ' return ', + ' (height - pal_h <= y and width - pal_w <= x and x <= width)', + 'end', + }), + + --- invoking on string literals needs it to be enclosed + prep('"string literal":gsub()', '("string literal"):gsub()'), + prep('("string literal"):gsub()'), +} + +local full = { + prep(sierpinski, sierpinski_res), + prep(clock, clock_res), + prep(meta, meta_res), +} + +local todo = { + prep({ + 'direction = {', + ' up = function(n)', + ' head.y = head.y - n', + ' end', + '}' + }), + prep('f()'), + prep('o:asd()'), + prep('t = { a = 1 }'), + prep('t = { c = 2 }'), + prep('a = 1'), +} + +return { + { 'basics', basics }, + { 'operators', operators }, + { 'functions', functions }, + { 'self', self }, + { 'comments', comments }, + { 'emptylines', emptylines }, + { 'wrap', wrapping }, + { 'canon', canon }, + { 'full', full }, + + { 'todo', todo }, +} diff --git a/tests/interpreter/ast_spec.lua b/tests/interpreter/ast_spec.lua index 38ed750b9..bcee18e20 100644 --- a/tests/interpreter/ast_spec.lua +++ b/tests/interpreter/ast_spec.lua @@ -4,15 +4,7 @@ local FS = require("util.filesystem") require("util.color") require("util.debug") -local inputs = (function() - local ok, i = pcall(require, "lib.metalua.spec.ast_inputs") - if ok then - return i - else - Log.warn('AST inputs missing, are submodules checked out?') - return {} - end -end)() +local inputs = require("tests.interpreter.ast_inputs") if not orig_print then _G.orig_print = print