Skip to content

Commit

Permalink
Merge pull request #330 from projectdiscovery/feat-memoizer
Browse files Browse the repository at this point in the history
AST + runtime memoizer
  • Loading branch information
tarunKoyalwar committed Feb 26, 2024
2 parents 9b85085 + 7bbb4b8 commit ef2665f
Show file tree
Hide file tree
Showing 10 changed files with 750 additions and 2 deletions.
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ go 1.21

require (
github.com/Masterminds/semver/v3 v3.2.1
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
github.com/cespare/xxhash v1.1.0
github.com/charmbracelet/glamour v0.6.0
github.com/google/go-github/v30 v30.1.0
github.com/hdm/jarm-go v0.0.7
Expand Down Expand Up @@ -33,7 +35,6 @@ require (
)

require (
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
Expand Down Expand Up @@ -114,7 +115,7 @@ require (
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/term v0.16.0
golang.org/x/tools v0.13.0 // indirect
golang.org/x/tools v0.13.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
Expand All @@ -28,6 +30,8 @@ github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5M
github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo=
Expand Down Expand Up @@ -217,6 +221,8 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
Expand Down
15 changes: 15 additions & 0 deletions memoize/cmd/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package main

import (
"log"

"github.com/projectdiscovery/utils/memoize"
)

func main() {
out, err := memoize.File(memoize.PackageTemplate, "../tests/test.go", "test")
if err != nil {
panic(err)
}
log.Println(string(out))
}
68 changes: 68 additions & 0 deletions memoize/gen/generic/memoize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// this small cli tool is specific for those functions with arbitrary parameters and with result-error tuple as return values
// func(x,y) => result, error
// it works by creating a new memoized version of the functions in the same path as memo.original.file.go
// some parts are specific for nuclei and hardcoded within the template
package main

import (
"flag"
"io/fs"
"log"
"os"
"path/filepath"

"github.com/projectdiscovery/utils/memoize"
stringsutil "github.com/projectdiscovery/utils/strings"
)

var (
src = flag.String("src", "", "go sources")
)

func main() {
flag.Parse()

err := filepath.Walk(*src, walk)
if err != nil {
log.Fatal(err)
}
}

func walk(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
return nil
}

if err != nil {
return err
}

ext := filepath.Ext(path)
base := filepath.Base(path)

if !stringsutil.EqualFoldAny(ext, ".go") {
return nil
}

basePath := filepath.Dir(path)
outPath := filepath.Join(basePath, "memo."+base)

// filename := filepath.Base(path)
data, err := os.ReadFile(path)
if err != nil {
return err
}
if !stringsutil.ContainsAnyI(string(data), "@memo") {
return nil
}
out, err := memoize.Src(memoize.PackageTemplate, path, data, "test")
if err != nil {
return err
}

if err := os.WriteFile(outPath, out, os.ModePerm); err != nil {
return err
}

return nil
}
Loading

0 comments on commit ef2665f

Please sign in to comment.