Adding Simulation API

README.md

@@ -26,6 +26,7 @@ UI to view the detailed approval status of any pull request.
   - [Approval Policies](#approval-policies)
   - [Disapproval Policy](#disapproval-policy)
   - [Testing and Debugging Policies](#testing-and-debugging-policies)
+    - [Simulation API](#simulation-api)
   - [Caveats and Notes](#caveats-and-notes)
     - [Disapproval is Disabled by Default](#disapproval-is-disabled-by-default)
     - [Interactions with GitHub Reviews](#interactions-with-github-reviews)
@@ -549,6 +550,84 @@ $ rcode=$(curl https://policybot.domain/api/validate -XPUT -T path/to/policy.yml
 $ if [[ "${rcode}" -gt 299 ]]; then cat /tmp/response && exit 1; fi
 ```
 
+#### Simulation API
+
+It can be useful to simulate how Policy Bot would evaluate a pull request if certain conditions were changed. For example: adding a review from a specific user or group, or adjusting the base branch.
+
+An API endpoint exists at `api/simulate/:org/:repo/:prNumber` to simiulate the result of a pull request. Simulations using this endpoint will NOT write the result back to the pull request status check and will instead return the result.
+
+This API requires a GitHub token be passed as a bearer token. The token must have the ability to read the pull request the simulation is being run against.
+
+The API can be used as such:
+
+```sh
+$ curl https://policybot.domain/api/simulate/:org/:repo/:number -H 'authorization: Bearer <token>' -H 'content-type: application/json' -X POST -d '<data>'
+```
+
+Currently the data payload can be configured with a few options:
+
+Ignore any comments from specific users, team members, org members or with specific permissions
+```json
+{
+  "ignore_comments":{
+    "users":["ignored-user"],
+    "teams":["ignored-team"],
+    "organizations":["ignored-org"],
+    "permissions":["admin"]
+  }
+}
+```
+
+Ignore any reviews from specific users, team members, org members or with specific permissions
+```json
+{
+  "ignore_reviews":{
+    "users":["ignored-user"],
+    "teams":["ignored-team"],
+    "organizations":["ignored-org"],
+    "permissions":["admin"]
+  }
+}
+```
+
+Simulate the pull request as if the following comments from the following users had also been added
+```json
+{
+  "add_comments":[
+    {
+      "author":"not-ignored-user",
+      "body":":+1:",
+      "created_at": "2020-11-30T14:20:28.000+07:00",
+      "last_edited_at": "2020-11-30T14:20:28.000+07:00"
+    }
+  ]
+}
+```
+
+Simulate the pull request as if the following reviews from the following users had also been added
+```json
+{
+  "add_reviews":[
+    {
+      "author":"not-ignored-user",
+      "state": "approved",
+      "body": "test approved review",
+      "created_at": "2020-11-30T14:20:28.000+07:00",
+      "last_edited_at": "2020-11-30T14:20:28.000+07:00"
+    }
+  ]
+}
+```
+
+Choose a different base branch when simulating the pull request evaluation
+```json
+{
+  "base_branch": "test-branch"
+}
+```
+
+The above can be combined to form more complex simulations. If a Simulation is run without any data being passed, the pull request is evaluated as is.
+
 ### Caveats and Notes
 
 There are several additional behaviors that follow from the rules above that

policy/common/actor.go

@@ -31,12 +31,12 @@ type Actors struct {
 	Organizations []string `yaml:"organizations" json:"organizations"`
 
 	// Deprecated: use Permissions with "admin" or "write"
-	Admins             bool `yaml:"admins"`
-	WriteCollaborators bool `yaml:"write_collaborators"`
+	Admins             bool `yaml:"admins" json:"-"`
+	WriteCollaborators bool `yaml:"write_collaborators" json:"-"`
 
 	// A list of GitHub collaborator permissions that are allowed. Values may
 	// be any of "admin", "maintain", "write", "triage", and "read".
-	Permissions []pull.Permission `json:"permissions"`
+	Permissions []pull.Permission `yaml:"permissions" json:"permissions"`
 }
 
 // IsEmpty returns true if no conditions for actors are defined.

policy/simulated/context.go

@@ -16,7 +16,6 @@ package simulated
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/palantir/policy-bot/pull"
 )
@@ -71,7 +70,7 @@ func (c *Context) filterIgnoredComments(prCtx pull.Context, comments []*pull.Com
 func (c *Context) addApprovalComment(comments []*pull.Comment) []*pull.Comment {
 	var commentsToAdd []*pull.Comment
 	for _, comment := range c.options.AddComments {
-		commentsToAdd = append(commentsToAdd, &comment)
+		commentsToAdd = append(commentsToAdd, comment.toPullComment())
 	}
 
 	return append(comments, commentsToAdd...)
@@ -116,10 +115,8 @@ func (c *Context) filterIgnoredReviews(prCtx pull.Context, reviews []*pull.Revie
 
 func (c *Context) addApprovalReview(reviews []*pull.Review) []*pull.Review {
 	var reviewsToAdd []*pull.Review
-	for i, review := range c.options.AddReviews {
-		review.ID = fmt.Sprintf("simulated-reviewID-%d", i)
-		review.SHA = fmt.Sprintf("simulated-reviewSHA-%d", i)
-		reviewsToAdd = append(reviewsToAdd, &review)
+	for _, review := range c.options.AddReviews {
+		reviewsToAdd = append(reviewsToAdd, review.toPullReview())
 	}
 
 	return append(reviews, reviewsToAdd...)

policy/simulated/context_test.go

@@ -107,7 +107,7 @@ func TestComments(t *testing.T) {
 				{Author: "iignore"},
 			},
 			Options: Options{
-				AddComments: []pull.Comment{
+				AddComments: []Comment{
 					{Author: "sperson", Body: ":+1:"},
 				},
 			},
@@ -119,7 +119,7 @@ func TestComments(t *testing.T) {
 				{Author: "iignore"},
 			},
 			Options: Options{
-				AddComments: []pull.Comment{
+				AddComments: []Comment{
 					{Author: "sperson", Body: ":+1:"},
 				},
 				IgnoreComments: &common.Actors{
@@ -246,7 +246,7 @@ func TestReviews(t *testing.T) {
 				{Author: "iignore"},
 			},
 			Options: Options{
-				AddReviews: []pull.Review{
+				AddReviews: []Review{
 					{Author: "sperson", State: "approved"},
 				},
 			},
@@ -258,7 +258,7 @@ func TestReviews(t *testing.T) {
 				{Author: "iignore"},
 			},
 			Options: Options{
-				AddReviews: []pull.Review{
+				AddReviews: []Review{
 					{Author: "sperson", State: "approved"},
 				},
 				IgnoreReviews: &common.Actors{

policy/simulated/options.go

@@ -29,8 +29,8 @@ import (
 type Options struct {
 	IgnoreComments *common.Actors `json:"ignore_comments"`
 	IgnoreReviews  *common.Actors `json:"ignore_reviews"`
-	AddComments    []pull.Comment `json:"add_comments"`
-	AddReviews     []pull.Review  `json:"add_reviews"`
+	AddComments    []Comment      `json:"add_comments"`
+	AddReviews     []Review       `json:"add_reviews"`
 	BaseBranch     string         `json:"base_branch"`
 }
 
@@ -51,30 +51,83 @@ func NewOptionsFromRequest(r *http.Request) (Options, error) {
 // setDefaults sets any values for the options that were not intentionally set in the request body but which should have
 // consistent values for the length of the simulation, such as the created time for a comment or review.
 func (o *Options) setDefaults() {
-	now := time.Now()
 	for i, review := range o.AddReviews {
-		if review.CreatedAt.IsZero() {
-			review.CreatedAt = now
-		}
-
-		if review.LastEditedAt.IsZero() {
-			review.LastEditedAt = now
-		}
+		id := fmt.Sprintf("simulated-reviewID-%d", i)
+		sha := fmt.Sprintf("simulated-reviewSHA-%d", i)
 
-		review.ID = fmt.Sprintf("simulated-reviewID-%d", i)
-		review.SHA = fmt.Sprintf("simulated-reviewSHA-%d", i)
+		review.setDefaults(id, sha)
 		o.AddReviews[i] = review
 	}
 
 	for i, comment := range o.AddComments {
-		if comment.CreatedAt.IsZero() {
-			comment.CreatedAt = now
-		}
+		comment.setDefaults()
+		o.AddComments[i] = comment
+	}
+}
 
-		if comment.LastEditedAt.IsZero() {
-			comment.LastEditedAt = now
-		}
+type Comment struct {
+	CreatedAt    *time.Time `json:"created_at"`
+	LastEditedAt *time.Time `json:"last_edited_at"`
+	Author       string     `json:"author"`
+	Body         string     `json:"body"`
+}
 
-		o.AddComments[i] = comment
+// setDefaults sets the createdAt and lastEdtedAt values to time.Now() if they are otherwise unset
+func (c *Comment) setDefaults() {
+	now := time.Now()
+	if c.CreatedAt == nil {
+		c.CreatedAt = &now
+	}
+
+	if c.LastEditedAt == nil {
+		c.LastEditedAt = &now
+	}
+}
+
+func (c *Comment) toPullComment() *pull.Comment {
+	return &pull.Comment{
+		CreatedAt:    *c.CreatedAt,
+		LastEditedAt: *c.LastEditedAt,
+		Author:       c.Author,
+		Body:         c.Body,
+	}
+}
+
+type Review struct {
+	ID           string
+	SHA          string
+	CreatedAt    *time.Time `json:"created_at"`
+	LastEditedAt *time.Time `json:"last_edited_at"`
+	Author       string     `json:"author"`
+	Body         string     `json:"body"`
+	State        string     `json:"state"`
+	Teams        []string   `json:"teams"`
+}
+
+// setDefaults sets the createdAt and lastEdtedAt values to time.Now() if they are otherwise unset
+func (r *Review) setDefaults(id, sha string) {
+	now := time.Now()
+	if r.CreatedAt == nil {
+		r.CreatedAt = &now
+	}
+
+	if r.LastEditedAt == nil {
+		r.LastEditedAt = &now
+	}
+
+	r.ID = id
+	r.SHA = sha
+}
+
+func (r *Review) toPullReview() *pull.Review {
+	return &pull.Review{
+		ID:           r.ID,
+		SHA:          r.SHA,
+		CreatedAt:    *r.CreatedAt,
+		LastEditedAt: *r.LastEditedAt,
+		Author:       r.Author,
+		State:        pull.ReviewState(r.State),
+		Body:         r.Body,
+		Teams:        r.Teams,
 	}
 }

policy/simulated/options_test.go

@@ -63,17 +63,17 @@ func TestOptionsFromRequest(t *testing.T) {
 	assert.Equal(t, "iignore", opt.AddReviews[0].Author)
 	assert.Equal(t, ":+1:", opt.AddReviews[0].Body)
 
-	assert.Equal(t, pull.ReviewApproved, opt.AddReviews[0].State)
+	assert.Equal(t, "approved", opt.AddReviews[0].State)
 	assert.Equal(t, "test-base", opt.BaseBranch)
 }
 
 func TestOptionDefaults(t *testing.T) {
 	options := Options{
-		AddComments: []pull.Comment{
+		AddComments: []Comment{
 			{Author: "aperson", Body: ":+1:"},
 			{Author: "otherperson", Body: ":+1:"},
 		},
-		AddReviews: []pull.Review{
+		AddReviews: []Review{
 			{Author: "aperson", Body: ":+1:"},
 			{Author: "otherperson", Body: ":+1:"},
 		},

pull/context.go

@@ -191,10 +191,10 @@ type Signature struct {
 }
 
 type Comment struct {
-	CreatedAt    time.Time `json:"created_at"`
-	LastEditedAt time.Time `json:"last_edited_at"`
-	Author       string    `json:"author"`
-	Body         string    `json:"body"`
+	CreatedAt    time.Time
+	LastEditedAt time.Time
+	Author       string
+	Body         string
 }
 
 type ReviewState string
@@ -208,15 +208,15 @@ const (
 )
 
 type Review struct {
-	ID           string      `json:"id"`
-	CreatedAt    time.Time   `json:"created_at"`
-	LastEditedAt time.Time   `json:"last_edited_at"`
-	Author       string      `json:"author"`
-	State        ReviewState `json:"state"`
-	Body         string      `json:"body"`
-	SHA          string      `json:"sha"`
-
-	Teams []string `json:"teams"`
+	ID           string
+	CreatedAt    time.Time
+	LastEditedAt time.Time
+	Author       string
+	State        ReviewState
+	Body         string
+	SHA          string
+
+	Teams []string
 }
 
 type ReviewerType string