diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index b7af262b..d3bf08d2 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -35,47 +35,6 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.22 - - name: Ready msys2 - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: git mingw-w64-x86_64-toolchain - path-type: inherit - if: matrix.os == 'windows-latest' + go-version: 1.23 - name: Go code test run: go test ./... - - cross-compile-test: - runs-on: macos-12 - defaults: - run: - working-directory: "go" - steps: - - name: Git checkout - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.22 - - uses: korandoru/setup-zig@v1 - with: - zig-version: master - - name: Set output - id: macos_sdk - run: echo "path=$(xcrun --show-sdk-path)" >> $GITHUB_OUTPUT - - name: Go cross compile test on Windows - run: | - CGO_ENABLED=1 GOOS=windows GOARCH=amd64 CC='zig cc -target x86_64-windows-gnu' go build ./... - - name: Go cross compile test on Linux - run: | - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC='zig cc -target x86_64-linux-musl' go build ./... - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC='zig cc -target aarch64-linux-musl' go build ./... - - name: Go cross compile test on Macos - run: | - export SDK_PATH=$(xcrun --show-sdk-path) - CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 CC='zig cc -target x86_64-macos-none -F'"${SDK_PATH}"'/System/Library/Frameworks' go build ./... - CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 CC='zig cc -target aarch64-macos-none -F'"${SDK_PATH}"'/System/Library/Frameworks' go build ./... - env: - SDK_PATH: ${{ steps.macos_sdk.outputs.path }} diff --git a/.github/workflows/nodejs-test.yaml b/.github/workflows/nodejs-test.yaml index 945d57b7..f0e57154 100644 --- a/.github/workflows/nodejs-test.yaml +++ b/.github/workflows/nodejs-test.yaml @@ -163,7 +163,7 @@ jobs: path: nodejs/*.node macos: - runs-on: macos-latest + runs-on: macos-12 strategy: matrix: settings: diff --git a/.github/workflows/zig-test.yaml b/.github/workflows/zig-test.yaml index 19344ace..f7fb88bc 100644 --- a/.github/workflows/zig-test.yaml +++ b/.github/workflows/zig-test.yaml @@ -29,7 +29,7 @@ jobs: working-directory: "zig" strategy: matrix: - os: [macos-12, macos-latest, ubuntu-latest, windows-latest] + os: [macos-12, macos-latest, ubuntu-20.04, ubuntu-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/go.mod b/go.mod index 3324a521..98ee9135 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,9 @@ module kcl-lang.io/lib go 1.19 -require google.golang.org/protobuf v1.34.2 +require ( + github.com/ebitengine/purego v0.7.1 + google.golang.org/protobuf v1.34.2 +) + +require golang.org/x/sys v0.25.0 // indirect diff --git a/go.sum b/go.sum index 73d32b16..748d1834 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,8 @@ +github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA= +github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/go/README.md b/go/README.md new file mode 100644 index 00000000..2f49e864 --- /dev/null +++ b/go/README.md @@ -0,0 +1,13 @@ +# KCL Artifact Library for Go + +## Developing + +### Prerequisites + ++ Go 1.23+ + +### Build and Test + +```shell +go test ./... +``` diff --git a/go/install/install.go b/go/install/install.go index 1bfd82e7..4555a566 100644 --- a/go/install/install.go +++ b/go/install/install.go @@ -53,34 +53,32 @@ func InstallKclvm(installRoot string) error { if err != nil { return err } - binPath := filepath.Join(installRoot, "bin") - - versionMatched, err := checkVersion(binPath) + versionMatched, err := checkVersion(installRoot) if err != nil { return err } // Install kclvm binary. - err = installBin(binPath, "kclvm_cli", lib.CliBin, versionMatched) + err = installBin(installRoot, "kclvm_cli", lib.CliBin, versionMatched) if err != nil { return err } // Install kclvm libs. - err = installLib(binPath, "kclvm_cli_cdylib", versionMatched) + err = installLib(installRoot, "kclvm_cli_cdylib", versionMatched) if err != nil { return err } if !versionMatched { - kclvmVersionPath := filepath.Join(binPath, "kclvm.version") + kclvmVersionPath := filepath.Join(installRoot, "kclvm.version") err = os.WriteFile(kclvmVersionPath, []byte(getVersion()), os.FileMode(os.O_WRONLY|os.O_TRUNC)) if err != nil { return err } } - os.Setenv("PATH", os.Getenv("PATH")+string(os.PathListSeparator)+binPath) + os.Setenv("PATH", os.Getenv("PATH")+string(os.PathListSeparator)+installRoot) return nil } diff --git a/go/lib/windows-arm64/static/libkclvm_cli_cdylib.a b/go/lib/windows-arm64/static/libkclvm_cli_cdylib.a deleted file mode 100644 index 05cf5cb0..00000000 Binary files a/go/lib/windows-arm64/static/libkclvm_cli_cdylib.a and /dev/null differ diff --git a/go/native/cgo.go b/go/native/cgo.go deleted file mode 100644 index 3d1dcce4..00000000 --- a/go/native/cgo.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build cgo -// +build cgo - -package native - -// #cgo CFLAGS:-I${SRCDIR}/../include -// #cgo !windows LDFLAGS:-lkclvm_cli_cdylib -lm -ldl -pthread -// #cgo windows LDFLAGS:-lkclvm_cli_cdylib -lmsvcrt -luserenv -lole32 -lntdll -lws2_32 -lkernel32 -lbcrypt -// #cgo linux,amd64 LDFLAGS:-L${SRCDIR}/../lib/linux-amd64 -Wl,-rpath,${SRCDIR}/../lib/linux-amd64 -// #cgo linux,arm64 LDFLAGS:-L${SRCDIR}/../lib/linux-arm64 -Wl,-rpath,${SRCDIR}/../lib/linux-arm64 -// #cgo darwin,amd64 LDFLAGS:-L${SRCDIR}/../lib/darwin-amd64 -Wl,-rpath,${SRCDIR}/../lib/darwin-amd64 -// #cgo darwin,arm64 LDFLAGS:-L${SRCDIR}/../lib/darwin-arm64 -Wl,-rpath,${SRCDIR}/../lib/darwin-arm64 -// #cgo windows,amd64 LDFLAGS:-L${SRCDIR}/../lib/windows-amd64/static -Wl,-rpath,${SRCDIR}/../lib/windows-amd64 -// #include -import "C" - -// NewKclvmService takes a pluginAgent and returns a pointer of capi kclvm_service. -// pluginAgent is the address of C function pointer with type :const char * (*)(const char *method,const char *args,const char *kwargs), -// in which "method" is the fully qualified name of plugin method, -// and "args" ,"kwargs" and return value of pluginAgent are JSON string -func NewKclvmService(pluginAgent C.uint64_t) *C.kclvm_service { - return C.kclvm_service_new(pluginAgent) -} - -// NewKclvmService releases the memory of kclvm_service -func DeleteKclvmService(serv *C.kclvm_service) { - C.kclvm_service_delete(serv) -} - -// KclvmServiceFreeString releases the memory of the return value of KclvmServiceCall -func KclvmServiceFreeString(str *C.char) { - C.kclvm_service_free_string(str) -} - -// KclvmServiceCall calls kclvm service by c api -// args should be serialized as protobuf byte stream -func KclvmServiceCall(serv *C.kclvm_service, method *C.char, args *C.char, args_len C.size_t) (*C.char, C.size_t) { - var size C.size_t = 0 - buf := C.kclvm_service_call_with_length(serv, method, args, args_len, &size) - return buf, size -} diff --git a/go/native/client.go b/go/native/client.go index 82bf0ec4..17742cb4 100644 --- a/go/native/client.go +++ b/go/native/client.go @@ -1,33 +1,46 @@ -//go:build cgo -// +build cgo - package native -/* -#include -#include -typedef struct kclvm_service kclvm_service; -*/ -import "C" import ( "bytes" "errors" "runtime" "strings" + "sync" "unsafe" + "github.com/ebitengine/purego" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "kcl-lang.io/lib/go/api" "kcl-lang.io/lib/go/plugin" ) +var libInit sync.Once + +var ( + lib uintptr + serviceNew func(uint64) uintptr + serviceDelete func(uintptr) + serviceCall func(uintptr, string, string, uint, *uint) uintptr + freeString func(uintptr) +) + type validator interface { Validate() error } type NativeServiceClient struct { - client *C.kclvm_service + svc uintptr +} + +func initLib() { + libInit.Do(func() { + lib, _ = loadServiceNativeLib() + purego.RegisterLibFunc(&serviceNew, lib, "kclvm_service_new") + purego.RegisterLibFunc(&serviceDelete, lib, "kclvm_service_delete") + purego.RegisterLibFunc(&serviceCall, lib, "kclvm_service_call_with_length") + purego.RegisterLibFunc(&freeString, lib, "kclvm_service_free_string") + }) } func NewNativeServiceClient() api.ServiceClient { @@ -35,11 +48,12 @@ func NewNativeServiceClient() api.ServiceClient { } func NewNativeServiceClientWithPluginAgent(pluginAgent uint64) *NativeServiceClient { + initLib() c := new(NativeServiceClient) - c.client = NewKclvmService(C.uint64_t(pluginAgent)) + c.svc = serviceNew(pluginAgent) runtime.SetFinalizer(c, func(x *NativeServiceClient) { - DeleteKclvmService(x.client) - x.client = nil + serviceDelete(x.svc) + closeLibrary(lib) }) return c } @@ -68,20 +82,10 @@ func cApiCall[I interface { if err != nil { return nil, err } + var cOutSize uint + cOut := serviceCall(c.svc, callName, string(inBytes), uint(len(inBytes)), &cOutSize) - cCallName := C.CString(callName) - - defer C.free(unsafe.Pointer(cCallName)) - - cIn := C.CString(string(inBytes)) - - defer C.free(unsafe.Pointer(cIn)) - - cOut, cOutSize := KclvmServiceCall(c.client, cCallName, cIn, C.size_t(len(inBytes))) - - defer KclvmServiceFreeString(cOut) - - msg := C.GoBytes(unsafe.Pointer(cOut), C.int(cOutSize)) + msg := GoByte(cOut, cOutSize) if bytes.HasPrefix(msg, []byte("ERROR:")) { return nil, errors.New(strings.TrimPrefix(string(msg), "ERROR:")) @@ -96,6 +100,16 @@ func cApiCall[I interface { return out, nil } +// GoByte copies a null-terminated char* to a Go string. +func GoByte(c uintptr, length uint) []byte { + // We take the address and then dereference it to trick go vet from creating a possible misuse of unsafe.Pointer + ptr := *(*unsafe.Pointer)(unsafe.Pointer(&c)) + if ptr == nil { + return []byte{} + } + return unsafe.Slice((*byte)(ptr), length) +} + func (c *NativeServiceClient) Ping(in *api.Ping_Args) (*api.Ping_Result, error) { return cApiCall[*api.Ping_Args, *api.Ping_Result](c, "KclvmService.Ping", in) } diff --git a/go/native/client_plugin_test.go b/go/native/client_plugin_test.go new file mode 100644 index 00000000..c0730ce3 --- /dev/null +++ b/go/native/client_plugin_test.go @@ -0,0 +1,65 @@ +//go:build cgo +// +build cgo + +package native + +import ( + "strings" + "testing" + + "kcl-lang.io/lib/go/api" + "kcl-lang.io/lib/go/plugin" +) + +func init() { + // Add a plugin named hello + plugin.RegisterPlugin(plugin.Plugin{ + Name: "hello", + MethodMap: map[string]plugin.MethodSpec{ + "add": { + Body: func(args *plugin.MethodArgs) (*plugin.MethodResult, error) { + v := args.IntArg(0) + args.IntArg(1) + return &plugin.MethodResult{V: v}, nil + }, + }, + }, + }) +} + +func TestExecProgramWithPlugin(t *testing.T) { + client := NewNativeServiceClient() + result, err := client.ExecProgram(&api.ExecProgram_Args{ + KFilenameList: []string{"main.k"}, + KCodeList: []string{code}, + Args: []*api.Argument{ + { + Name: "a", + Value: "1", + }, + { + Name: "b", + Value: "2", + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if result.ErrMessage != "" { + t.Fatal("error message must be empty") + } +} + +func TestExecProgramWithPluginError(t *testing.T) { + client := NewNativeServiceClient() + result, err := client.ExecProgram(&api.ExecProgram_Args{ + KFilenameList: []string{"main.k"}, + KCodeList: []string{code}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(result.ErrMessage, "strconv.ParseInt: parsing \"\": invalid syntax") { + t.Fatal(result.ErrMessage) + } +} diff --git a/go/native/client_test.go b/go/native/client_test.go index 00a0c35b..af1989b5 100644 --- a/go/native/client_test.go +++ b/go/native/client_test.go @@ -1,6 +1,3 @@ -//go:build cgo -// +build cgo - package native import ( @@ -11,7 +8,6 @@ import ( "time" "kcl-lang.io/lib/go/api" - "kcl-lang.io/lib/go/plugin" ) const code = ` @@ -21,59 +17,6 @@ name = "kcl" sum = hello.add(option("a"), option("b")) ` -func init() { - // Add a plugin named hello - plugin.RegisterPlugin(plugin.Plugin{ - Name: "hello", - MethodMap: map[string]plugin.MethodSpec{ - "add": { - Body: func(args *plugin.MethodArgs) (*plugin.MethodResult, error) { - v := args.IntArg(0) + args.IntArg(1) - return &plugin.MethodResult{V: v}, nil - }, - }, - }, - }) -} - -func TestExecProgramWithPlugin(t *testing.T) { - client := NewNativeServiceClient() - result, err := client.ExecProgram(&api.ExecProgram_Args{ - KFilenameList: []string{"main.k"}, - KCodeList: []string{code}, - Args: []*api.Argument{ - { - Name: "a", - Value: "1", - }, - { - Name: "b", - Value: "2", - }, - }, - }) - if err != nil { - t.Fatal(err) - } - if result.ErrMessage != "" { - t.Fatal("error message must be empty") - } -} - -func TestExecProgramWithPluginError(t *testing.T) { - client := NewNativeServiceClient() - result, err := client.ExecProgram(&api.ExecProgram_Args{ - KFilenameList: []string{"main.k"}, - KCodeList: []string{code}, - }) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(result.ErrMessage, "strconv.ParseInt: parsing \"\": invalid syntax") { - t.Fatal(result.ErrMessage) - } -} - func TestParseFile(t *testing.T) { // Example: Test with string source src := `schema Name: diff --git a/go/native/loader.go b/go/native/loader.go new file mode 100644 index 00000000..997f9eb0 --- /dev/null +++ b/go/native/loader.go @@ -0,0 +1,47 @@ +package native + +import ( + "os" + "path/filepath" + "runtime" + + "kcl-lang.io/lib/go/install" +) + +const libName = "kclvm_cli_cdylib" + +func libPath() (path string, err error) { + libHome := os.Getenv("KCL_LIB_HOME") + if libHome == "" { + return os.MkdirTemp("", "kcl_lib_home") + } + return libHome, nil +} + +func fullLibName() string { + fullLibName := "lib" + libName + ".so" + if runtime.GOOS == "darwin" { + fullLibName = "lib" + libName + ".dylib" + } else if runtime.GOOS == "windows" { + fullLibName = libName + ".dll" + } + return fullLibName +} + +func loadServiceNativeLib() (uintptr, error) { + libPath, err := libPath() + if err != nil { + return 0, err + } + err = install.InstallKclvm(libPath) + if err != nil { + return 0, err + } + libFullPath := filepath.Join(libPath, fullLibName()) + // open the C library + libm, err := openLibrary(libFullPath) + if err != nil { + return 0, err + } + return libm, nil +} diff --git a/go/native/open_lib_unix.go b/go/native/open_lib_unix.go new file mode 100644 index 00000000..1f1c0615 --- /dev/null +++ b/go/native/open_lib_unix.go @@ -0,0 +1,14 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + +package native + +import "github.com/ebitengine/purego" + +func openLibrary(name string) (uintptr, error) { + return purego.Dlopen(name, purego.RTLD_NOW|purego.RTLD_GLOBAL) +} + +func closeLibrary(handle uintptr) error { + return purego.Dlclose(handle) +} diff --git a/go/native/open_lib_windows.go b/go/native/open_lib_windows.go new file mode 100644 index 00000000..44fd56c0 --- /dev/null +++ b/go/native/open_lib_windows.go @@ -0,0 +1,17 @@ +//go:build windows +// +build windows + +package native + +import "syscall" + +func openLibrary(name string) (uintptr, error) { + // Use [syscall.LoadLibrary] here to avoid external dependencies (#270). + // For actual use cases, [golang.org/x/sys/windows.NewLazySystemDLL] is recommended. + handle, err := syscall.LoadLibrary(name) + return uintptr(handle), err +} + +func closeLibrary(handle uintptr) error { + return syscall.Close(syscall.Handle(handle)) +} diff --git a/go/plugin/api.go b/go/plugin/api.go index 0abe221d..c797ce1e 100644 --- a/go/plugin/api.go +++ b/go/plugin/api.go @@ -6,20 +6,10 @@ package plugin import ( - "os" "strings" "sync" ) -// Debug flag for kcl-plugin -var DebugMode = false - -func init() { - if s := strings.ToLower(os.Getenv("KCLVM_PLUGIN_DEBUG")); s != "" && s != "false" && s != "0" { - DebugMode = true - } -} - var pluginManager struct { allPlugin map[string]Plugin allMethodSpec map[string]MethodSpec diff --git a/go/plugin/api_test.go b/go/plugin/api_test.go index 7047dced..3d15c81f 100644 --- a/go/plugin/api_test.go +++ b/go/plugin/api_test.go @@ -36,9 +36,6 @@ func init() { } func TestPlugin_strings_join(t *testing.T) { - if !CgoEnabled { - t.Skip("cgo disabled") - } result_json := Invoke("kcl_plugin.strings.join", []interface{}{"KCL", "KCL", 123}, nil) if result_json != `"KCL.KCL.123"` { t.Fatal(result_json) @@ -46,9 +43,6 @@ func TestPlugin_strings_join(t *testing.T) { } func TestPlugin_strings_panic(t *testing.T) { - if !CgoEnabled { - t.Skip("cgo disabled") - } result_json := Invoke("kcl_plugin.strings.panic", []interface{}{"KCL", "KCL", 123}, nil) if result_json != `{"__kcl_PanicInfo__":"[KCL KCL 123]"}` { t.Fatal(result_json) diff --git a/go/plugin/plugin.go b/go/plugin/plugin.go index d63c9d68..95f846e2 100644 --- a/go/plugin/plugin.go +++ b/go/plugin/plugin.go @@ -18,8 +18,6 @@ import ( "fmt" ) -const CgoEnabled = true - //export kcl_go_plugin_proxy_func func kcl_go_plugin_proxy_func(_method, _args_json, _kwargs_json *C.char) (result_json *C.char) { var method, args_json, kwargs_json string diff --git a/go/plugin/plugin_non_cgo.go b/go/plugin/plugin_non_cgo.go new file mode 100644 index 00000000..68ecefb8 --- /dev/null +++ b/go/plugin/plugin_non_cgo.go @@ -0,0 +1,10 @@ +//go:build !cgo +// +build !cgo + +// Copyright The KCL Authors. All rights reserved. + +package plugin + +func GetInvokeJsonProxyPtr() uint64 { + return 0 +} diff --git a/zig/Makefile b/zig/Makefile new file mode 100644 index 00000000..af3b389a --- /dev/null +++ b/zig/Makefile @@ -0,0 +1,2 @@ +test: + zig build test diff --git a/zig/README.md b/zig/README.md index acc81cc1..8e202969 100644 --- a/zig/README.md +++ b/zig/README.md @@ -4,11 +4,11 @@ This repo is under development, PRs welcome! ## Developing -## Prerequisites +### Prerequisites + Zig 0.13.0+ -## Build and Test +### Build and Test ```shell zig build test diff --git a/zig/build.zig b/zig/build.zig index b1d3a90c..252c74d6 100644 --- a/zig/build.zig +++ b/zig/build.zig @@ -32,6 +32,8 @@ pub fn build(b: *std.Build) void { lib.linkSystemLibrary(kclLibName()); if (os == .windows) { linkWindowsLibraries(lib); + } else if (os == .macos) { + linkMacOSLibraries(lib); } // This declares intent for the library to be installed into the standard @@ -53,6 +55,8 @@ pub fn build(b: *std.Build) void { lib_unit_tests.linkSystemLibrary(kclLibName()); if (os == .windows) { linkWindowsLibraries(lib_unit_tests); + } else if (os == .macos) { + linkMacOSLibraries(lib_unit_tests); } const run_lib_unit_tests = b.addRunArtifact(lib_unit_tests); @@ -69,6 +73,11 @@ fn linkWindowsLibraries(lib: *std.Build.Step.Compile) void { lib.linkSystemLibrary("ws2_32"); } +fn linkMacOSLibraries(lib: *std.Build.Step.Compile) void { + lib.linkFramework("CoreFoundation"); + lib.linkFramework("Security"); +} + fn kclLibName() []const u8 { return "kclvm_cli_cdylib"; } @@ -80,13 +89,13 @@ fn kclLibPath(b: *std.Build, target: *const std.Build.ResolvedTarget) std.Build. .windows => { switch (arch) { .x86_64 => { - return b.path("../go/lib/windows-amd64/static"); + return b.path("../go/lib/windows-amd64/"); }, .x86 => { - return b.path("../go/lib/windows-amd64/static"); + return b.path("../go/lib/windows-amd64/"); }, .aarch64 => { - return b.path("../go/lib/windows-arm64/static"); + return b.path("../go/lib/windows-arm64/"); }, else => @panic("Unsupported Windows architecture"), }