javascript bindings + docs generation enhancements ( generate typescr…

…ipt defination .d.ts files) (#4487)

* introduce typescript files generation using ast + tmpl

* feat valid ts with scraping

* feat remove old logic + tsdocs for all modules

* fix ikev and related bugs

* typescript docs for js modules

* lint,build + ldap realm fix

* go mod tidy

* fix named imports ast parsing

* fix ast code generation errors

* complete support for ts files generation

* support go global/const in ts docs

* updated template

* feat: typescript using go code generation

* nuke jsdoc generator

* update generated ts dir structure

* fix multifile ts gen issue

* fix panic in ts code gen

* fix test

* update docs of js libs

* feat: add doc+example for every js class,function,method

* fix missing quotes in ikev example

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>

.gitignore

@@ -20,11 +20,14 @@ pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
 .gitignore
 pkg/js/devtools/bindgen/cmd/bindgen/bindgen
 pkg/js/devtools/jsdocgen/jsdocgen
+pkg/js/devtools/tsgen/tsgen
+pkg/js/devtools/tsgen/cmd/tsgen/tsgen
 *.DS_Store
 pkg/protocols/headless/engine/.cache
 /nuclei
 /bindgen
 /jsdocgen
+/tsgen
 /scrapefuncs
 /integration_tests/.cache/
 /integration_tests/.nuclei-config/

Makefile

@@ -32,6 +32,14 @@ tidy:
 	$(GOMOD) tidy
 devtools:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go
-	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "jsdocgen" pkg/js/devtools/jsdocgen/main.go
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go
+jsupdate:
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
+	./bindgen -dir pkg/js/libs -out pkg/js/generated
+	./tsgen -dir pkg/js/libs -out pkg/js/generated/ts
+ts:
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
+	./tsgen -dir pkg/js/libs -out pkg/js/generated/ts
 

go.mod

@@ -288,7 +288,7 @@ require (
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/tools v0.17.0
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect

pkg/js/devtools/README.md

@@ -6,9 +6,6 @@ devtools contains tools and scripts to automate booring tasks related to javascr
 
 [bindgen](./bindgen/README.md) is a tool that automatically generated bindings for native go packages with 'goja'
 
-### jsdocgen
-
-[jsdocgen](./jsdocgen/README.md) is LLM (OpenAI) based dev tool it takes generated javascript files and annotes them with jsdoc comments using predefined prompt
 
 ### scrapefuncs
 

pkg/js/devtools/bindgen/cmd/bindgen/main.go

@@ -16,14 +16,12 @@ var (
 	dir           string
 	generatedDir  string
 	targetModules string
-	goOnly        bool
 )
 
 func main() {
 	flag.StringVar(&dir, "dir", "libs", "directory to process")
 	flag.StringVar(&generatedDir, "out", "generated", "directory to output generated files")
 	flag.StringVar(&targetModules, "target", "", "target modules to generate")
-	flag.BoolVar(&goOnly, "go", false, "generate only go files")
 	flag.Parse()
 	log.SetFlags(0)
 	if !fileutil.FolderExists(dir) {
@@ -54,12 +52,12 @@ func process() error {
 		}
 
 		prefixed := "lib" + module
-		if !goOnly {
-			err = data.WriteJSTemplate(filepath.Join(generatedDir, "js/"+prefixed), module)
-			if err != nil {
-				return fmt.Errorf("could not write js template: %v", err)
-			}
-		}
+		// if !goOnly {
+		// 	err = data.WriteJSTemplate(filepath.Join(generatedDir, "js/"+prefixed), module)
+		// 	if err != nil {
+		// 		return fmt.Errorf("could not write js template: %v", err)
+		// 	}
+		// }
 		err = data.WriteGoTemplate(path.Join(generatedDir, "go/"+prefixed), module)
 		if err != nil {
 			return fmt.Errorf("could not write go template: %v", err)

pkg/js/devtools/bindgen/generator.go

@@ -28,16 +28,18 @@ var (
 
 // TemplateData contains the parameters for the JS code generator
 type TemplateData struct {
-	PackageName             string
-	PackagePath             string
-	PackageFuncs            map[string]string
-	PackageInterfaces       map[string]string
-	PackageFuncsExtraNoType map[string]PackageFunctionExtra
-	PackageFuncsExtra       map[string]PackageFuncExtra
-	PackageVars             map[string]string
-	PackageVarsValues       map[string]string
-	PackageTypes            map[string]string
-	PackageTypesExtra       map[string]PackageTypeExtra
+	PackageName               string
+	PackagePath               string
+	HasObjects                bool
+	PackageFuncs              map[string]string
+	PackageInterfaces         map[string]string
+	PackageFuncsExtraNoType   map[string]PackageFunctionExtra
+	PackageFuncsExtra         map[string]PackageFuncExtra
+	PackageVars               map[string]string
+	PackageVarsValues         map[string]string
+	PackageTypes              map[string]string
+	PackageTypesExtra         map[string]PackageTypeExtra
+	PackageDefinedConstructor map[string]struct{}
 
 	typesPackage *types.Package
 
@@ -68,16 +70,17 @@ type PackageFunctionExtra struct {
 // newTemplateData creates a new template data structure
 func newTemplateData(packagePrefix, pkgName string) *TemplateData {
 	return &TemplateData{
-		PackageName:             pkgName,
-		PackagePath:             packagePrefix + pkgName,
-		PackageFuncs:            make(map[string]string),
-		PackageFuncsExtraNoType: make(map[string]PackageFunctionExtra),
-		PackageFuncsExtra:       make(map[string]PackageFuncExtra),
-		PackageVars:             make(map[string]string),
-		PackageVarsValues:       make(map[string]string),
-		PackageTypes:            make(map[string]string),
-		PackageInterfaces:       make(map[string]string),
-		PackageTypesExtra:       make(map[string]PackageTypeExtra),
+		PackageName:               pkgName,
+		PackagePath:               packagePrefix + pkgName,
+		PackageFuncs:              make(map[string]string),
+		PackageFuncsExtraNoType:   make(map[string]PackageFunctionExtra),
+		PackageFuncsExtra:         make(map[string]PackageFuncExtra),
+		PackageVars:               make(map[string]string),
+		PackageVarsValues:         make(map[string]string),
+		PackageTypes:              make(map[string]string),
+		PackageInterfaces:         make(map[string]string),
+		PackageTypesExtra:         make(map[string]PackageTypeExtra),
+		PackageDefinedConstructor: make(map[string]struct{}),
 	}
 }
 
@@ -144,6 +147,24 @@ func CreateTemplateData(directory string, packagePrefix string) (*TemplateData,
 			delete(data.PackageFuncsExtra, item)
 		}
 	}
+
+	// map types with corresponding constructors
+	for constructor := range data.PackageDefinedConstructor {
+	object:
+		for k := range data.PackageTypes {
+			if strings.Contains(constructor, k) {
+				data.PackageTypes[k] = constructor
+				break object
+			}
+		}
+	}
+	for k, v := range data.PackageTypes {
+		if k == v || v == "" {
+			data.HasObjects = true
+			data.PackageTypes[k] = ""
+		}
+	}
+
 	return data, nil
 }
 
@@ -399,11 +420,21 @@ func (d *TemplateData) collectFuncDecl(decl *ast.FuncDecl) (extra PackageFunctio
 	extra.Name = decl.Name.Name
 	extra.Doc = convertCommentsToJavascript(decl.Doc.Text())
 
+	isConstructor := false
+
 	for _, arg := range decl.Type.Params.List {
+		p := exprToString(arg.Type)
+		if strings.Contains(p, "goja.ConstructorCall") {
+			isConstructor = true
+		}
 		for _, name := range arg.Names {
 			extra.Args = append(extra.Args, name.Name)
 		}
 	}
+	if isConstructor {
+		d.PackageDefinedConstructor[decl.Name.Name] = struct{}{}
+	}
+
 	extra.Returns = d.extractReturns(decl)
 	return extra
 }
@@ -413,3 +444,22 @@ func convertCommentsToJavascript(comments string) string {
 	suffix := strings.Trim(strings.TrimSuffix(strings.ReplaceAll(comments, "\n", "\n// "), "// "), "\n")
 	return fmt.Sprintf("// %s", suffix)
 }
+
+// exprToString converts an expression to a string
+func exprToString(expr ast.Expr) string {
+	switch t := expr.(type) {
+	case *ast.Ident:
+		return t.Name
+	case *ast.SelectorExpr:
+		return exprToString(t.X) + "." + t.Sel.Name
+	case *ast.StarExpr:
+		return exprToString(t.X)
+	case *ast.ArrayType:
+		return "[]" + exprToString(t.Elt)
+	case *ast.InterfaceType:
+		return "interface{}"
+	// Add more cases to handle other types
+	default:
+		return fmt.Sprintf("%T", expr)
+	}
+}

pkg/js/devtools/bindgen/templates/go_class.tmpl

@@ -27,18 +27,15 @@ func init() {
 			"{{$objName}}": {{$pkgName}}.{{$objDefine}},
 			{{- end}}
 
-			// Types (value type)
+			// Objects / Classes
 			{{- range $objName, $objDefine := .PackageTypes}}
-			"{{$objName}}": {{printf "func() %s.%s { return %s.%s{} }" $pkgName $objDefine $pkgName $objDefine}},
+			{{- if $objDefine}}
+			"{{$objName}}": {{$pkgName}}.{{$objDefine}},
+			{{- else}}
+			"{{$objName}}": gojs.GetClassConstructor[{{$pkgName}}.{{$objName}}](&{{$pkgName}}.{{$objName}}{}),
+			{{- end}}
 			{{- end}}
 
-			// Types (pointer type)
-			{{range $objName, $objDefine := .PackageTypes}}
-			{{- $newObjName := printf "%s%s" "New" $objName -}}
-			{{- if not (exist $pkgFuncs $newObjName) -}}
-			"{{$newObjName}}": {{printf "func() *%s.%s { return &%s.%s{} }" $pkgName $objDefine $pkgName $objDefine}},
-			{{end -}}
-			{{- end -}}
 			},
 		).Register()
 }