Skip to content

Commit

Permalink
Avoid over usage of memory: apply resolving on
Browse files Browse the repository at this point in the history
snapshot instead of graph to avoid numerous in loop
call to triplestore.Source.Snapshot() (call that
pre allocates memory)

Tackle part of issue #116

Overall rough improvements:
- CPU: 50% better on fetching mainly Access resources
- Mem alloc_object: from ~70% to ~40% of cumulated alloc_objects
      for Snapshot() method calls
  • Loading branch information
simcap committed Jul 28, 2017
1 parent 4979754 commit 417a0a9
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 18 deletions.
6 changes: 4 additions & 2 deletions aws/services/relations.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,8 +333,9 @@ func addManagedPoliciesRelations(g *graph.Graph, i interface{}) error {
return fmt.Errorf("add parent to %s: not a valid attached policy list: %T", res.Id(), structField.Interface())
}

immutableRDFGraph := g.AsRDFGraphSnaphot()
for _, policy := range policies {
policies, err := g.ResolveResources(&graph.And{Resolvers: []graph.Resolver{&graph.ByProperty{Key: "Name", Value: awssdk.StringValue(policy.PolicyName)}, &graph.ByType{Typ: cloud.Policy}}})
policies, err := graph.ResolveResourcesOnSnapShot(immutableRDFGraph, &graph.And{Resolvers: []graph.Resolver{&graph.ByProperty{Key: "Name", Value: awssdk.StringValue(policy.PolicyName)}, &graph.ByType{Typ: cloud.Policy}}})
if err != nil {
return err
}
Expand All @@ -357,9 +358,10 @@ func userAddGroupsRelations(g *graph.Graph, i interface{}) error {
return err
}

immutableRDFGraph := g.AsRDFGraphSnaphot()
for _, group := range user.GroupList {
groupName := awssdk.StringValue(group)
resources, err := g.ResolveResources(&graph.And{Resolvers: []graph.Resolver{
resources, err := graph.ResolveResourcesOnSnapShot(immutableRDFGraph, &graph.And{Resolvers: []graph.Resolver{
&graph.ByProperty{Key: "Name", Value: groupName},
&graph.ByType{Typ: cloud.Group},
}})
Expand Down
26 changes: 22 additions & 4 deletions graph/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ func NewGraphFromFile(filepath string) (*Graph, error) {
return g, nil
}

func (g *Graph) AsRDFGraphSnaphot() tstore.RDFGraph {
return g.store.Snapshot()
}

func (g *Graph) AddResource(resources ...*Resource) error {
for _, res := range resources {
triples, err := res.marshalFullRDF()
Expand Down Expand Up @@ -105,7 +109,7 @@ func (g *Graph) GetResource(t string, id string) (*Resource, error) {

func (g *Graph) FindResource(id string) (*Resource, error) {
byId := &ById{id}
resources, err := byId.Resolve(g)
resources, err := byId.Resolve(g.store.Snapshot())
if err != nil {
return nil, err
}
Expand All @@ -120,18 +124,32 @@ func (g *Graph) FindResource(id string) (*Resource, error) {

func (g *Graph) FindResourcesByProperty(key string, value interface{}) ([]*Resource, error) {
byProperty := ByProperty{key, value}
return byProperty.Resolve(g)
return byProperty.Resolve(g.store.Snapshot())
}

func (g *Graph) GetAllResources(typs ...string) ([]*Resource, error) {
byTypes := &ByTypes{typs}
return byTypes.Resolve(g)
return byTypes.Resolve(g.store.Snapshot())
}

func (g *Graph) ResolveResources(resolvers ...Resolver) ([]*Resource, error) {
var resources []*Resource
snap := g.store.Snapshot()
for _, resolv := range resolvers {
rs, err := resolv.Resolve(snap)
if err != nil {
return resources, err
}
resources = append(resources, rs...)
}

return resources, nil
}

func ResolveResourcesOnSnapShot(snap tstore.RDFGraph, resolvers ...Resolver) ([]*Resource, error) {
var resources []*Resource
for _, resolv := range resolvers {
rs, err := resolv.Resolve(g)
rs, err := resolv.Resolve(snap)
if err != nil {
return resources, err
}
Expand Down
22 changes: 10 additions & 12 deletions graph/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,24 @@ import (
)

type Resolver interface {
Resolve(g *Graph) ([]*Resource, error)
Resolve(snap tstore.RDFGraph) ([]*Resource, error)
}

type ById struct {
Id string
}

func (r *ById) Resolve(g *Graph) ([]*Resource, error) {
func (r *ById) Resolve(snap tstore.RDFGraph) ([]*Resource, error) {
resolver := &ByProperty{Key: properties.ID, Value: r.Id}
return resolver.Resolve(g)
return resolver.Resolve(snap)
}

type ByProperty struct {
Key string
Value interface{}
}

func (r *ByProperty) Resolve(g *Graph) ([]*Resource, error) {
func (r *ByProperty) Resolve(snap tstore.RDFGraph) ([]*Resource, error) {
var resources []*Resource
if r.Value == nil {
return resources, nil
Expand All @@ -43,7 +43,6 @@ func (r *ByProperty) Resolve(g *Graph) ([]*Resource, error) {
if err != nil {
return resources, fmt.Errorf("resolve by property: unmarshaling property '%s': %s", r.Key, err)
}
snap := g.store.Snapshot()
for _, t := range snap.WithPredObj(rdfpropLabel, obj) {
rt, err := resolveResourceType(snap, t.Subject())
if err != nil {
Expand All @@ -63,11 +62,11 @@ type And struct {
Resolvers []Resolver
}

func (r *And) Resolve(g *Graph) (result []*Resource, err error) {
func (r *And) Resolve(snap tstore.RDFGraph) (result []*Resource, err error) {
if len(r.Resolvers) == 0 {
return
}
result, err = r.Resolvers[0].Resolve(g)
result, err = r.Resolvers[0].Resolve(snap)
if err != nil {
return
}
Expand All @@ -77,7 +76,7 @@ func (r *And) Resolve(g *Graph) (result []*Resource, err error) {
return
}
for _, resolv := range r.Resolvers {
result, err = resolv.Resolve(gg)
result, err = resolv.Resolve(gg.store.Snapshot())
if err != nil {
return
}
Expand All @@ -94,9 +93,8 @@ type ByType struct {
Typ string
}

func (r *ByType) Resolve(g *Graph) ([]*Resource, error) {
func (r *ByType) Resolve(snap tstore.RDFGraph) ([]*Resource, error) {
var resources []*Resource
snap := g.store.Snapshot()
typ := namespacedResourceType(r.Typ)
for _, t := range snap.WithPredObj(rdf.RdfType, tstore.Resource(typ)) {
r := InitResource(r.Typ, t.Subject())
Expand All @@ -113,11 +111,11 @@ type ByTypes struct {
Typs []string
}

func (r *ByTypes) Resolve(g *Graph) ([]*Resource, error) {
func (r *ByTypes) Resolve(snap tstore.RDFGraph) ([]*Resource, error) {
var res []*Resource
for _, t := range r.Typs {
bt := &ByType{t}
all, err := bt.Resolve(g)
all, err := bt.Resolve(snap)
if err != nil {
return res, err
}
Expand Down

0 comments on commit 417a0a9

Please sign in to comment.