diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f2595a..028dfa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,3 +92,102 @@ jobs: LD_LIBRARY_PATH="$PWD/target/release" \ busted --lua=$(which luajit) tests/lua \ --lpath='./lua/?.lua' + + - name: Validate LuaRocks package + run: | + rm -rf /tmp/lua-qjson-rock + ROCKSPEC="$(python3 - <<'PY' + import re + from pathlib import Path + + pattern = re.compile(r"^lua-qjson-(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?-(\d+)\.rockspec$") + def prerelease_key(value): + if value is None: + return () + key = [] + for part in value.split("."): + if part.isdigit(): + key.append((0, int(part))) + else: + key.append((1, part)) + return tuple(key) + + matches = [] + for path in Path("rockspec").glob("lua-qjson-*.rockspec"): + match = pattern.match(path.name) + if match: + major, minor, patch, prerelease, revision = match.groups() + matches.append((int(major), int(minor), int(patch), prerelease is None, prerelease_key(prerelease), int(revision), str(path))) + if not matches: + raise SystemExit("no lua-qjson rockspec found") + print(max(matches)[-1]) + PY + )" + luarocks make "$ROCKSPEC" --tree /tmp/lua-qjson-rock + eval "$(luarocks path --tree /tmp/lua-qjson-rock)" + unset LD_LIBRARY_PATH + luajit -e 'local qjson = require("qjson"); local doc = qjson.parse("{\"a\":42}"); assert(doc:get_i64("a") == 42)' + busted --lua=$(which luajit) tests/lua + + package: + name: LuaRocks package (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: rust + strategy: + matrix: + os: [ubuntu-latest, macos-14] + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Rust (stable) + run: | + rustup toolchain install stable --profile minimal --no-self-update + rustup default stable + + - name: Install LuaJIT + uses: leafo/gh-actions-lua@v13 + with: + luaVersion: "luajit-2.1.0-beta3" + + - name: Install LuaRocks + uses: leafo/gh-actions-luarocks@v4 + with: + luarocksVersion: "3.11.1" + + - name: Validate LuaRocks package + run: | + rm -rf /tmp/lua-qjson-rock + ROCKSPEC="$(python3 - <<'PY' + import re + from pathlib import Path + + pattern = re.compile(r"^lua-qjson-(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?-(\d+)\.rockspec$") + def prerelease_key(value): + if value is None: + return () + key = [] + for part in value.split("."): + if part.isdigit(): + key.append((0, int(part))) + else: + key.append((1, part)) + return tuple(key) + + matches = [] + for path in Path("rockspec").glob("lua-qjson-*.rockspec"): + match = pattern.match(path.name) + if match: + major, minor, patch, prerelease, revision = match.groups() + matches.append((int(major), int(minor), int(patch), prerelease is None, prerelease_key(prerelease), int(revision), str(path))) + if not matches: + raise SystemExit("no lua-qjson rockspec found") + print(max(matches)[-1]) + PY + )" + luarocks make "$ROCKSPEC" --tree /tmp/lua-qjson-rock + eval "$(luarocks path --tree /tmp/lua-qjson-rock)" + unset LD_LIBRARY_PATH + unset DYLD_LIBRARY_PATH + lua -e 'assert(jit, "LuaJIT required"); local qjson = require("qjson"); local doc = qjson.parse("{\"a\":42}"); assert(doc:get_i64("a") == 42)' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..006489c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,111 @@ +name: Release + +on: + push: + branches: + - "main" + paths: + - "rockspec/**" + +permissions: + contents: write + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install Lua + uses: leafo/gh-actions-lua@v10 + with: + luaVersion: "5.1.5" + + - name: Install LuaRocks + uses: leafo/gh-actions-luarocks@v4 + with: + luarocksVersion: "3.11.1" + + - name: Install upload dependency + run: luarocks install dkjson + + - name: Extract release name + id: release_env + shell: bash + env: + HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + title="${HEAD_COMMIT_MESSAGE%%$'\n'*}" + re="^feat: release v?([0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?)$" + if [[ $title =~ $re ]]; then + v=v${BASH_REMATCH[1]} + echo "version=${v}" >> $GITHUB_OUTPUT + echo "version_without_v=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT + if [[ ${BASH_REMATCH[1]} == *-* ]]; then + echo "is_prerelease=true" >> $GITHUB_OUTPUT + else + echo "is_prerelease=false" >> $GITHUB_OUTPUT + fi + else + echo "commit format is not correct (expected: feat: release vX.Y.Z[-prerelease])" + exit 1 + fi + + - name: Verify rockspec exists + id: verify_rockspec + shell: bash + env: + VERSION_WITHOUT_V: ${{ steps.release_env.outputs.version_without_v }} + run: | + rockspec_file="$(python3 - <<'PY' + import os + import re + from pathlib import Path + + version = os.environ["VERSION_WITHOUT_V"] + pattern = re.compile(r"^lua-qjson-" + re.escape(version) + r"-(\d+)\.rockspec$") + matches = [] + for path in Path("rockspec").glob("lua-qjson-" + version + "-*.rockspec"): + match = pattern.match(path.name) + if match: + matches.append((int(match.group(1)), str(path))) + if not matches: + raise SystemExit("rockspec file not found for version " + version) + print(max(matches)[1]) + PY + )" + echo "rockspec_file=${rockspec_file}" >> $GITHUB_OUTPUT + + - name: Validate LuaRocks token + shell: bash + env: + LUAROCKS_TOKEN: ${{ secrets.LUAROCKS_TOKEN }} + run: | + if [ -z "$LUAROCKS_TOKEN" ]; then + echo "LUAROCKS_TOKEN secret is not configured" + exit 1 + fi + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.release_env.outputs.version }} + name: ${{ steps.release_env.outputs.version }} + draft: false + prerelease: ${{ steps.release_env.outputs.is_prerelease }} + + - name: Upload to LuaRocks + env: + LUAROCKS_TOKEN: ${{ secrets.LUAROCKS_TOKEN }} + ROCKSPEC_FILE: ${{ steps.verify_rockspec.outputs.rockspec_file }} + shell: bash + run: | + mkdir -p "$HOME/.luarocks" + chmod 700 "$HOME/.luarocks" + lua -e 'local token = os.getenv("LUAROCKS_TOKEN"); assert(token and token ~= "", "missing LUAROCKS_TOKEN"); local f = assert(io.open(os.getenv("HOME") .. "/.luarocks/upload_config.lua", "w")); f:write(("key = %q\n"):format(token)); f:write("server = \"https://luarocks.org\"\nversion = \"1.0\"\n"); f:close()' + chmod 600 "$HOME/.luarocks/upload_config.lua" + luarocks upload "$ROCKSPEC_FILE" diff --git a/README.md b/README.md index 64f34d7..c8a1462 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,20 @@ cargo build --release A `Makefile` wraps the common workflows; run `make help` to see `build`, `test`, `lint`, `bench`, and `clean` targets. Override `LUAJIT` / `LUA_CPATH` per invocation if your environment differs from the defaults. +## Installing + +```sh +luarocks install lua-qjson +``` + +The rock builds the Rust native library during installation, so Rust/Cargo +and LuaJIT must be available on the target system. The Lua module name remains +`qjson`: + +```lua +local qjson = require("qjson") +``` + ## Testing ```sh diff --git a/lua/qjson.lua b/lua/qjson.lua index 3e57324..d30b72b 100644 --- a/lua/qjson.lua +++ b/lua/qjson.lua @@ -1,49 +1,6 @@ local ffi = require("ffi") -ffi.cdef[[ -typedef struct qjson_doc qjson_doc; -typedef struct { - const qjson_doc* doc; - uint32_t idx_start, idx_end, _reserved0, _reserved1; -} qjson_cursor; - -typedef struct { - uint32_t mode; - uint32_t max_depth; -} qjson_options; - -const char* qjson_strerror(int code); -qjson_doc* qjson_parse (const uint8_t* buf, size_t len, int* err_out); -qjson_doc* qjson_parse_ex(const uint8_t* buf, size_t len, - const qjson_options* opts, int* err_out); -void qjson_free (qjson_doc* doc); - -int qjson_get_str (qjson_doc*, const char* path, size_t path_len, const uint8_t** p, size_t* n); -int qjson_get_i64 (qjson_doc*, const char* path, size_t path_len, int64_t* out); -int qjson_get_f64 (qjson_doc*, const char* path, size_t path_len, double* out); -int qjson_get_bool(qjson_doc*, const char* path, size_t path_len, int* out); -int qjson_is_null (qjson_doc*, const char* path, size_t path_len, int* out); -int qjson_typeof (qjson_doc*, const char* path, size_t path_len, int* out); -int qjson_len (qjson_doc*, const char* path, size_t path_len, size_t* out); - -int qjson_open (qjson_doc*, const char* path, size_t path_len, qjson_cursor* out); -int qjson_cursor_open (const qjson_cursor*, const char* path, size_t path_len, qjson_cursor* out); -int qjson_cursor_field(const qjson_cursor*, const char* key, size_t key_len, qjson_cursor* out); -int qjson_cursor_index(const qjson_cursor*, size_t i, qjson_cursor* out); - -int qjson_cursor_get_str (const qjson_cursor*, const char*, size_t, const uint8_t**, size_t*); -int qjson_cursor_get_i64 (const qjson_cursor*, const char*, size_t, int64_t*); -int qjson_cursor_get_f64 (const qjson_cursor*, const char*, size_t, double*); -int qjson_cursor_get_bool(const qjson_cursor*, const char*, size_t, int*); -int qjson_cursor_typeof (const qjson_cursor*, const char*, size_t, int*); -int qjson_cursor_len (const qjson_cursor*, const char*, size_t, size_t*); -int qjson_cursor_bytes(const qjson_cursor*, size_t* byte_start, size_t* byte_end); -int qjson_cursor_object_entry_at(const qjson_cursor*, size_t i, - const uint8_t** key_ptr, size_t* key_len, - qjson_cursor* value_out); -]] - -local C = ffi.load("qjson") +local C = require("qjson.lib") local err_box = ffi.new("int[1]") local i64_box = ffi.new("int64_t[1]") diff --git a/lua/qjson/lib.lua b/lua/qjson/lib.lua new file mode 100644 index 0000000..3e6c686 --- /dev/null +++ b/lua/qjson/lib.lua @@ -0,0 +1,121 @@ +local ffi = require("ffi") + +ffi.cdef[[ +typedef struct qjson_doc qjson_doc; +typedef struct { + const qjson_doc* doc; + uint32_t idx_start, idx_end, _reserved0, _reserved1; +} qjson_cursor; + +typedef struct { + uint32_t mode; + uint32_t max_depth; +} qjson_options; + +const char* qjson_strerror(int code); +qjson_doc* qjson_parse (const uint8_t* buf, size_t len, int* err_out); +qjson_doc* qjson_parse_ex(const uint8_t* buf, size_t len, + const qjson_options* opts, int* err_out); +void qjson_free (qjson_doc* doc); + +int qjson_get_str (qjson_doc*, const char* path, size_t path_len, const uint8_t** p, size_t* n); +int qjson_get_i64 (qjson_doc*, const char* path, size_t path_len, int64_t* out); +int qjson_get_f64 (qjson_doc*, const char* path, size_t path_len, double* out); +int qjson_get_bool(qjson_doc*, const char* path, size_t path_len, int* out); +int qjson_is_null (qjson_doc*, const char* path, size_t path_len, int* out); +int qjson_typeof (qjson_doc*, const char* path, size_t path_len, int* out); +int qjson_len (qjson_doc*, const char* path, size_t path_len, size_t* out); + +int qjson_open (qjson_doc*, const char* path, size_t path_len, qjson_cursor* out); +int qjson_cursor_open (const qjson_cursor*, const char* path, size_t path_len, qjson_cursor* out); +int qjson_cursor_field(const qjson_cursor*, const char* key, size_t key_len, qjson_cursor* out); +int qjson_cursor_index(const qjson_cursor*, size_t i, qjson_cursor* out); + +int qjson_cursor_get_str (const qjson_cursor*, const char*, size_t, const uint8_t**, size_t*); +int qjson_cursor_get_i64 (const qjson_cursor*, const char*, size_t, int64_t*); +int qjson_cursor_get_f64 (const qjson_cursor*, const char*, size_t, double*); +int qjson_cursor_get_bool(const qjson_cursor*, const char*, size_t, int*); +int qjson_cursor_typeof (const qjson_cursor*, const char*, size_t, int*); +int qjson_cursor_len (const qjson_cursor*, const char*, size_t, size_t*); +int qjson_cursor_bytes(const qjson_cursor*, size_t* byte_start, size_t* byte_end); +int qjson_cursor_object_entry_at(const qjson_cursor*, size_t i, + const uint8_t** key_ptr, size_t* key_len, + qjson_cursor* value_out); +]] + +local tried = {} +local attempts = {} +local last_error +local required_symbols = { + "qjson_strerror", + "qjson_parse", + "qjson_parse_ex", + "qjson_free", + "qjson_get_str", + "qjson_get_i64", + "qjson_get_f64", + "qjson_get_bool", + "qjson_is_null", + "qjson_typeof", + "qjson_len", + "qjson_open", + "qjson_cursor_open", + "qjson_cursor_field", + "qjson_cursor_index", + "qjson_cursor_get_str", + "qjson_cursor_get_i64", + "qjson_cursor_get_f64", + "qjson_cursor_get_bool", + "qjson_cursor_typeof", + "qjson_cursor_len", + "qjson_cursor_bytes", + "qjson_cursor_object_entry_at", +} + +local function try_load(name) + if tried[name] then + return nil + end + tried[name] = true + attempts[#attempts + 1] = name + local ok, lib = pcall(ffi.load, name) + if ok then + for _, required_symbol in ipairs(required_symbols) do + local has_symbol, symbol = pcall(function() + return lib[required_symbol] + end) + if not has_symbol or symbol == nil then + last_error = "loaded " .. name .. " but missing required symbol " .. required_symbol + return nil + end + end + return lib + end + last_error = lib + return nil +end + +local function load_from_cpath() + local names = { "qjson", "libqjson" } + for template in string.gmatch(package.cpath, "[^;]+") do + if string.find(template, "?", 1, true) then + for _, name in ipairs(names) do + local path = string.gsub(template, "%?", name) + local lib = try_load(path) + if lib then + return lib + end + end + end + end + return nil +end + +local lib = try_load("qjson") or try_load("libqjson") or load_from_cpath() +if not lib then + error("qjson: failed to load native library qjson; tried " + .. table.concat(attempts, ", ") + .. "; last error: " .. tostring(last_error)) +end + +return lib diff --git a/lua/qjson/table.lua b/lua/qjson/table.lua index f23c076..86f50d0 100644 --- a/lua/qjson/table.lua +++ b/lua/qjson/table.lua @@ -1,11 +1,7 @@ -- Lazy table view + cjson-compatible encoder for qjson. --- --- This module relies on the FFI cdef set up by `lua/qjson.lua`, so --- callers must `require("qjson")` (transitively or directly) before --- they require this module. local ffi = require("ffi") -local C = ffi.load("qjson") +local C = require("qjson.lib") -- Defer the require to avoid a circular dependency when qjson.lua -- re-exports this module. By the time _M.decode is called, qjson -- is already registered in package.loaded. diff --git a/rockspec/lua-qjson-0.1.0-1.rockspec b/rockspec/lua-qjson-0.1.0-1.rockspec new file mode 100644 index 0000000..acc0010 --- /dev/null +++ b/rockspec/lua-qjson-0.1.0-1.rockspec @@ -0,0 +1,39 @@ +package = "lua-qjson" +version = "0.1.0-1" +supported_platforms = { "linux", "macosx" } + +source = { + url = "git+https://github.com/api7/lua-qjson.git", + tag = "v0.1.0", +} + +description = { + summary = "Fast JSON decoder for LuaJIT FFI consumers", + detailed = [[ + qjson is a Rust-implemented JSON decoder exposed to LuaJIT through FFI. + It is optimized for parsing large JSON payloads while reading only the + fields needed by Lua callers. + ]], + homepage = "https://github.com/api7/lua-qjson", + license = "Apache-2.0", +} + +dependencies = { + "lua == 5.1", +} + +build = { + type = "command", + build_command = "CARGO_TARGET_DIR=target cargo build --release", + install_command = [[ + set -e + mkdir -p "$(LUADIR)/qjson" "$(LIBDIR)" + cp lua/qjson.lua "$(LUADIR)/qjson.lua" + cp lua/qjson/lib.lua "$(LUADIR)/qjson/lib.lua" + cp lua/qjson/table.lua "$(LUADIR)/qjson/table.lua" + lib=target/release/libqjson.so + if [ ! -f "$lib" ]; then lib=target/release/libqjson.dylib; fi + if [ ! -f "$lib" ]; then echo "qjson native library not found"; exit 1; fi + cp "$lib" "$(LIBDIR)/qjson.so" + ]], +} diff --git a/tests/lua/loader_spec.lua b/tests/lua/loader_spec.lua new file mode 100644 index 0000000..c78b16d --- /dev/null +++ b/tests/lua/loader_spec.lua @@ -0,0 +1,65 @@ +describe("qjson native loader", function() + it("reports attempted libraries and the last loader error", function() + local old_lib = package.loaded["qjson.lib"] + local old_ffi = package.loaded.ffi + local old_cpath = package.cpath + + package.loaded["qjson.lib"] = nil + package.loaded.ffi = { + cdef = function() end, + load = function(name) + error("mock load failure for " .. name) + end, + } + package.cpath = "/tmp/qjson-test/?.so;/tmp/qjson-test/loadall.so" + + local ok, err = pcall(require, "qjson.lib") + + package.loaded["qjson.lib"] = old_lib + package.loaded.ffi = old_ffi + package.cpath = old_cpath + + assert.is_false(ok) + assert.matches("qjson: failed to load native library qjson", err, 1, true) + assert.matches("qjson, libqjson, /tmp/qjson-test/qjson.so, /tmp/qjson-test/libqjson.so", err, 1, true) + assert.matches("mock load failure for /tmp/qjson-test/libqjson.so", err, 1, true) + assert.is_nil(string.find(err, "loadall.so", 1, true)) + end) + + it("skips loadable libraries without qjson symbols", function() + local old_lib = package.loaded["qjson.lib"] + local old_ffi = package.loaded.ffi + local old_cpath = package.cpath + local valid_lib = setmetatable({}, { + __index = function(_, key) + if string.sub(key, 1, 6) == "qjson_" then + return true + end + end, + }) + + package.loaded["qjson.lib"] = nil + package.loaded.ffi = { + cdef = function() end, + load = function(name) + if name == "qjson" then + return {} + end + if name == "libqjson" then + return valid_lib + end + error("unexpected load: " .. name) + end, + } + package.cpath = "" + + local ok, lib = pcall(require, "qjson.lib") + + package.loaded["qjson.lib"] = old_lib + package.loaded.ffi = old_ffi + package.cpath = old_cpath + + assert.is_true(ok) + assert.are.equal(valid_lib, lib) + end) +end)