diff --git a/go.mod b/go.mod index 0ad40a1f..b5585508 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,8 @@ require ( golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/sync v0.7.0 golang.org/x/term v0.19.0 + gopkg.in/yaml.v3 v3.0.1 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -80,7 +82,6 @@ require ( golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.20.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect mvdan.cc/gofumpt v0.6.0 // indirect ) diff --git a/go.sum b/go.sum index 89161f17..a4cbe1b6 100644 --- a/go.sum +++ b/go.sum @@ -488,3 +488,5 @@ mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index 4c138260..8fe3a96f 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -12,10 +12,13 @@ import ( "path" "path/filepath" "slices" + "strconv" "strings" "time" "unicode/utf8" + "github.com/getkin/kin-openapi/openapi2" + "github.com/getkin/kin-openapi/openapi2conv" "github.com/getkin/kin-openapi/openapi3" "github.com/gptscript-ai/gptscript/pkg/assemble" "github.com/gptscript-ai/gptscript/pkg/builtin" @@ -24,6 +27,8 @@ import ( "github.com/gptscript-ai/gptscript/pkg/parser" "github.com/gptscript-ai/gptscript/pkg/system" "github.com/gptscript-ai/gptscript/pkg/types" + "gopkg.in/yaml.v3" + kyaml "sigs.k8s.io/yaml" ) const CacheTimeout = time.Hour @@ -142,9 +147,34 @@ func loadOpenAPI(prg *types.Program, data []byte) *openapi3.T { prg.OpenAPICache = map[string]any{} } - openAPIDocument, err = openapi3.NewLoader().LoadFromData(data) - if err != nil || openAPIDocument.Paths.Len() == 0 { - openAPIDocument = nil + switch isOpenAPI(data) { + case 2: + // Convert OpenAPI v2 to v3 + jsondata := data + if !json.Valid(data) { + jsondata, err = kyaml.YAMLToJSON(data) + if err != nil { + return nil + } + } + + doc := &openapi2.T{} + if err := doc.UnmarshalJSON(jsondata); err != nil { + return nil + } + + openAPIDocument, err = openapi2conv.ToV3(doc) + if err != nil { + return nil + } + case 3: + // Use OpenAPI v3 as is + openAPIDocument, err = openapi3.NewLoader().LoadFromData(data) + if err != nil { + return nil + } + default: + return nil } prg.OpenAPICache[openAPICacheKey] = openAPIDocument @@ -399,3 +429,42 @@ func input(ctx context.Context, cache *cache.Client, base *source, name string) return nil, fmt.Errorf("can not load tools path=%s name=%s", base.Path, name) } + +// isOpenAPI checks if the data is an OpenAPI definition and returns the version if it is. +func isOpenAPI(data []byte) int { + var fragment struct { + Paths map[string]any `json:"paths,omitempty"` + Swagger string `json:"swagger,omitempty"` + OpenAPI string `json:"openapi,omitempty"` + } + + if err := json.Unmarshal(data, &fragment); err != nil { + if err := yaml.Unmarshal(data, &fragment); err != nil { + return 0 + } + } + if len(fragment.Paths) == 0 { + return 0 + } + + if v, _, _ := strings.Cut(fragment.OpenAPI, "."); v != "" { + ver, err := strconv.Atoi(v) + if err != nil { + log.Debugf("invalid OpenAPI version: openapi=%q", fragment.OpenAPI) + return 0 + } + return ver + } + + if v, _, _ := strings.Cut(fragment.Swagger, "."); v != "" { + ver, err := strconv.Atoi(v) + if err != nil { + log.Debugf("invalid Swagger version: swagger=%q", fragment.Swagger) + return 0 + } + return ver + } + + log.Debugf("no OpenAPI version found in input data: openapi=%q, swagger=%q", fragment.OpenAPI, fragment.Swagger) + return 0 +} diff --git a/pkg/loader/loader_test.go b/pkg/loader/loader_test.go index b7ad4828..b5ae6843 100644 --- a/pkg/loader/loader_test.go +++ b/pkg/loader/loader_test.go @@ -3,8 +3,10 @@ package loader import ( "context" "encoding/json" + "os" "testing" + "github.com/gptscript-ai/gptscript/pkg/types" "github.com/hexops/autogold/v2" "github.com/stretchr/testify/require" ) @@ -17,6 +19,64 @@ func toString(obj any) string { return string(s) } +func TestIsOpenAPI(t *testing.T) { + datav2, err := os.ReadFile("testdata/openapi_v2.yaml") + require.NoError(t, err) + v := isOpenAPI(datav2) + require.Equal(t, 2, v, "(yaml) expected openapi v2") + + datav2, err = os.ReadFile("testdata/openapi_v2.json") + require.NoError(t, err) + v = isOpenAPI(datav2) + require.Equal(t, 2, v, "(json) expected openapi v2") + + datav3, err := os.ReadFile("testdata/openapi_v3.yaml") + require.NoError(t, err) + v = isOpenAPI(datav3) + require.Equal(t, 3, v, "(json) expected openapi v3") +} + +func TestLoadOpenAPI(t *testing.T) { + numOpenAPITools := func(set types.ToolSet) int { + num := 0 + for _, v := range set { + if v.IsOpenAPI() { + num++ + } + } + return num + } + + prgv3 := types.Program{ + ToolSet: types.ToolSet{}, + } + datav3, err := os.ReadFile("testdata/openapi_v3.yaml") + require.NoError(t, err) + _, err = readTool(context.Background(), nil, &prgv3, &source{Content: datav3}, "") + require.NoError(t, err, "failed to read openapi v3") + require.Equal(t, 3, numOpenAPITools(prgv3.ToolSet), "expected 3 openapi tools") + + prgv2json := types.Program{ + ToolSet: types.ToolSet{}, + } + datav2, err := os.ReadFile("testdata/openapi_v2.json") + require.NoError(t, err) + _, err = readTool(context.Background(), nil, &prgv2json, &source{Content: datav2}, "") + require.NoError(t, err, "failed to read openapi v2") + require.Equal(t, 3, numOpenAPITools(prgv2json.ToolSet), "expected 3 openapi tools") + + prgv2yaml := types.Program{ + ToolSet: types.ToolSet{}, + } + datav2, err = os.ReadFile("testdata/openapi_v2.yaml") + require.NoError(t, err) + _, err = readTool(context.Background(), nil, &prgv2yaml, &source{Content: datav2}, "") + require.NoError(t, err, "failed to read openapi v2 (yaml)") + require.Equal(t, 3, numOpenAPITools(prgv2yaml.ToolSet), "expected 3 openapi tools") + + require.EqualValuesf(t, prgv2json.ToolSet, prgv2yaml.ToolSet, "expected same toolset for openapi v2 json and yaml") +} + func TestHelloWorld(t *testing.T) { prg, err := Program(context.Background(), "https://raw.githubusercontent.com/ibuildthecloud/test/bafe5a62174e8a0ea162277dcfe3a2ddb7eea928/example/sub/tool.gpt", diff --git a/pkg/loader/testdata/openapi_v2.json b/pkg/loader/testdata/openapi_v2.json new file mode 100644 index 00000000..415eb3f9 --- /dev/null +++ b/pkg/loader/testdata/openapi_v2.json @@ -0,0 +1,153 @@ +{ + "swagger": "2.0", + "info": { + "version": "1.0.0", + "title": "Swagger Petstore", + "license": { + "name": "MIT" + } + }, + "host": "petstore.swagger.io", + "basePath": "/v1", + "schemes": [ + "http" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": { + "/pets": { + "get": { + "summary": "List all pets", + "operationId": "listPets", + "tags": [ + "pets" + ], + "parameters": [ + { + "name": "limit", + "in": "query", + "description": "How many items to return at one time (max 100)", + "required": false, + "type": "integer", + "format": "int32" + } + ], + "responses": { + "200": { + "description": "An paged array of pets", + "headers": { + "x-next": { + "type": "string", + "description": "A link to the next page of responses" + } + }, + "schema": { + "$ref": "#/definitions/Pets" + } + }, + "default": { + "description": "unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + }, + "post": { + "summary": "Create a pet", + "operationId": "createPets", + "tags": [ + "pets" + ], + "responses": { + "201": { + "description": "Null response" + }, + "default": { + "description": "unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/pets/{petId}": { + "get": { + "summary": "Info for a specific pet", + "operationId": "showPetById", + "tags": [ + "pets" + ], + "parameters": [ + { + "name": "petId", + "in": "path", + "required": true, + "description": "The id of the pet to retrieve", + "type": "string" + } + ], + "responses": { + "200": { + "description": "Expected response to a valid request", + "schema": { + "$ref": "#/definitions/Pets" + } + }, + "default": { + "description": "unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + } + }, + "definitions": { + "Pet": { + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "Pets": { + "type": "array", + "items": { + "$ref": "#/definitions/Pet" + } + }, + "Error": { + "required": [ + "code", + "message" + ], + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/pkg/loader/testdata/openapi_v2.yaml b/pkg/loader/testdata/openapi_v2.yaml new file mode 100644 index 00000000..afd10adf --- /dev/null +++ b/pkg/loader/testdata/openapi_v2.yaml @@ -0,0 +1,104 @@ +# yaml comment for testing +swagger: "2.0" +info: + version: 1.0.0 + title: Swagger Petstore + license: + name: MIT +host: petstore.swagger.io +basePath: /v1 +schemes: + - http +consumes: + - application/json +produces: + - application/json +paths: + /pets: + get: + summary: List all pets + operationId: listPets + tags: + - pets + parameters: + - name: limit + in: query + description: How many items to return at one time (max 100) + required: false + type: integer + format: int32 + responses: + "200": + description: A paged array of pets + headers: + x-next: + type: string + description: A link to the next page of responses + schema: + $ref: '#/definitions/Pets' + default: + description: unexpected error + schema: + $ref: '#/definitions/Error' + post: + summary: Create a pet + operationId: createPets + tags: + - pets + responses: + "201": + description: Null response + default: + description: unexpected error + schema: + $ref: '#/definitions/Error' + /pets/{petId}: + get: + summary: Info for a specific pet + operationId: showPetById + tags: + - pets + parameters: + - name: petId + in: path + required: true + description: The id of the pet to retrieve + type: string + responses: + "200": + description: Expected response to a valid request + schema: + $ref: '#/definitions/Pets' + default: + description: unexpected error + schema: + $ref: '#/definitions/Error' +definitions: + Pet: + type: "object" + required: + - id + - name + properties: + id: + type: integer + format: int64 + name: + type: string + tag: + type: string + Pets: + type: array + items: + $ref: '#/definitions/Pet' + Error: + type: "object" + required: + - code + - message + properties: + code: + type: integer + format: int32 + message: + type: string \ No newline at end of file diff --git a/pkg/loader/testdata/openapi_v3.yaml b/pkg/loader/testdata/openapi_v3.yaml new file mode 100644 index 00000000..4712f908 --- /dev/null +++ b/pkg/loader/testdata/openapi_v3.yaml @@ -0,0 +1,119 @@ +openapi: "3.0.0" +info: + version: 1.0.0 + title: Swagger Petstore + license: + name: MIT +servers: + - url: http://petstore.swagger.io/v1 +paths: + /pets: + get: + summary: List all pets + operationId: listPets + tags: + - pets + parameters: + - name: limit + in: query + description: How many items to return at one time (max 100) + required: false + schema: + type: integer + maximum: 100 + format: int32 + responses: + '200': + description: A paged array of pets + headers: + x-next: + description: A link to the next page of responses + schema: + type: string + content: + application/json: + schema: + $ref: "#/components/schemas/Pets" + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + post: + summary: Create a pet + operationId: createPets + tags: + - pets + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Pet' + required: true + responses: + '201': + description: Null response + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /pets/{petId}: + get: + summary: Info for a specific pet + operationId: showPetById + tags: + - pets + parameters: + - name: petId + in: path + required: true + description: The id of the pet to retrieve + schema: + type: string + responses: + '200': + description: Expected response to a valid request + content: + application/json: + schema: + $ref: "#/components/schemas/Pet" + default: + description: unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" +components: + schemas: + Pet: + type: object + required: + - id + - name + properties: + id: + type: integer + format: int64 + name: + type: string + tag: + type: string + Pets: + type: array + maxItems: 100 + items: + $ref: "#/components/schemas/Pet" + Error: + type: object + required: + - code + - message + properties: + code: + type: integer + format: int32 + message: + type: string \ No newline at end of file