Skip to content

Commit

Permalink
api package and reddit package + Binding refac #2
Browse files Browse the repository at this point in the history
- Refactored all the Binding + Paginator + API types and interfaces to the api package. This works similarly to the Binding interface that existed within the monday package except the request produced by Binding.Request is an interface (api.Request), and the client taken by Binding.Execute is also an interface (api.Client) (07/03/2023 - 16:28:13)
- This allows us to create entire schema's of bindings that we can then add to an instance of the API type which acts as a wrapper for a set of Bindings (07/03/2023 - 16:29:14)
- Removed the definitions of types that are now defined in api from the monday package (07/03/2023 - 16:29:44)
- Updated all the bindings in the monday and models packages to use the new function signatures (07/03/2023 - 16:30:07)
- Updated the monday subcommand in the CLI and the Measure phase to use the new paginator type (07/03/2023 - 16:30:38)
- Upgrade gotils to v2.1.2 (08/03/2023 - 12:59:02)
- Refactored all the NewBinding logic into the api package. This means that Bindings for the entire project can be created through the api.NewBinding method. Due to this, I have removed the now unecessary code from monday/bindings.go as well as changed the Bindings in the monday and models packages to use this new API (08/03/2023 - 15:17:54)
- Added the reddit/types.go file to hold all the return and response types for the Reddit API bindings (08/03/2023 - 16:23:19)
- Paginator instance has now been replaced by the Paginator interface and two new Paginator implementations: typedPaginator and paginator. NewTypedPaginator returns a Paginator that is type aware. This can only be used by Bindings that are set to their own global variables. NewPaginator returns a Paginator that is not type aware, instead it returns a Paginator[[]any, []any] (08/03/2023 - 18:27:25)
- Added a refresh to reddit.Client.Run (08/03/2023 - 18:50:40)
  • Loading branch information
andygello555 committed Mar 8, 2023
1 parent 19e9128 commit 95b16a1
Show file tree
Hide file tree
Showing 15 changed files with 1,302 additions and 268 deletions.
530 changes: 530 additions & 0 deletions api/api.go

Large diffs are not rendered by default.

152 changes: 152 additions & 0 deletions api/api_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package api

import (
"context"
"encoding/json"
"fmt"
myErrors "github.com/andygello555/game-scout/errors"
"github.com/pkg/errors"
"io"
"net/http"
"net/url"
"strconv"
)

type httpClient struct {
}

func (h httpClient) Run(ctx context.Context, attrs map[string]any, req Request, res any) (err error) {
request := req.(HTTPRequest).Request

var response *http.Response
if response, err = http.DefaultClient.Do(request); err != nil {
return err
}

if response.Body != nil {
defer func(body io.ReadCloser) {
err = myErrors.MergeErrors(err, errors.Wrapf(body.Close(), "could not close response body to %s", request.URL.String()))
}(response.Body)
}

var body []byte
if body, err = io.ReadAll(response.Body); err != nil {
err = errors.Wrapf(err, "could not read response body to %s", request.URL.String())
return
}

err = json.Unmarshal(body, res)
return
}

func ExampleNewAPI() {
// First we need to define our API's response and return structures.
type Product struct {
ID int `json:"id"`
Title string `json:"title"`
Price float64 `json:"price"`
Category string `json:"category"`
Description string `json:"description"`
Image string `json:"image"`
}

type User struct {
ID int `json:"id"`
Email string `json:"email"`
Username string `json:"username"`
Password string `json:"password"`
Name struct {
Firstname string `json:"firstname"`
Lastname string `json:"lastname"`
} `json:"name"`
Address struct {
City string `json:"city"`
Street string `json:"street"`
Number int `json:"number"`
Zipcode string `json:"zipcode"`
Geolocation struct {
Lat string `json:"lat"`
Long string `json:"long"`
} `json:"geolocation"`
} `json:"address"`
Phone string `json:"phone"`
}

// Then we create a Client instance. Here httpClient is a type that implements the Client interface, where
// Client.Run performs an HTTP request using http.DefaultClient, and then unmarshals the JSON response into the
// response wrapper.
client := httpClient{}

// Finally, we create the API itself by creating and registering all our Bindings within the Schema using the
// NewWrappedBinding method. The "users" and "products" Bindings take only one argument: the limit argument. This
// limits the number of resources returned by the fakestoreapi. This is applied to the Request by setting the query
// params for the http.Request.
api := NewAPI(client, Schema{
// Note: we do not supply a wrap and an unwrap method for the "users" and "products" Bindings because the
// fakestoreapi returns JSON that can be unmarshalled straight into an appropriate instance of type ResT.
// We also don't need to supply a response method because the ResT type is the same as the RetT type.
"users": NewWrappedBinding[[]User, []User]("users",
func(b Binding[[]User, []User], args ...any) (request Request) {
u, _ := url.Parse("https://fakestoreapi.com/users")
if len(args) > 0 {
query := u.Query()
query.Add("limit", strconv.Itoa(args[0].(int)))
u.RawQuery = query.Encode()
}
req, _ := http.NewRequest(http.MethodGet, u.String(), nil)
return HTTPRequest{req}
}, nil, nil, nil, false,
),
"products": NewWrappedBinding[[]Product, []Product]("products",
func(b Binding[[]Product, []Product], args ...any) Request {
u, _ := url.Parse("https://fakestoreapi.com/products")
if len(args) > 0 {
query := u.Query()
query.Add("limit", strconv.Itoa(args[0].(int)))
u.RawQuery = query.Encode()
}
req, _ := http.NewRequest(http.MethodGet, u.String(), nil)
return HTTPRequest{req}
}, nil, nil, nil, false,
),
// The "first_product" Binding showcases how to set the response method. This will execute a similar HTTP request
// to the "products" Binding but Binding.Execute will instead return a single Product instance.
// Note: how the RetT type param is set to Product.
"first_product": NewWrappedBinding[[]Product, Product]("first_product",
func(b Binding[[]Product, Product], args ...any) Request {
req, _ := http.NewRequest(http.MethodGet, "https://fakestoreapi.com/products?limit=1", nil)
return HTTPRequest{req}
}, nil, nil,
func(b Binding[[]Product, Product], response []Product, args ...any) Product {
return response[0]
}, false,
),
})

// Then we can execute our "users" binding with a limit of 3...
var resp any
var err error
if resp, err = api.Execute("users", 3); err != nil {
fmt.Println(err)
return
}
fmt.Println(resp.([]User))

// ...and we can also execute our "products" binding with a limit of 1...
if resp, err = api.Execute("products", 1); err != nil {
fmt.Println(err)
return
}
fmt.Println(resp.([]Product))

// ...and we can also execute our "first_product" binding.
if resp, err = api.Execute("first_product"); err != nil {
fmt.Println(err)
return
}
fmt.Println(resp.(Product))
// Output:
// [{1 john@gmail.com johnd m38rmF$ {john doe} {kilcoole new road 7682 12926-3874 {-37.3159 81.1496}} 1-570-236-7033} {2 morrison@gmail.com mor_2314 83r5^_ {david morrison} {kilcoole Lovers Ln 7267 12926-3874 {-37.3159 81.1496}} 1-570-236-7033} {3 kevin@gmail.com kevinryan kev02937@ {kevin ryan} {Cullman Frances Ct 86 29567-1452 {40.3467 -30.1310}} 1-567-094-1345}]
// [{1 Fjallraven - Foldsack No. 1 Backpack, Fits 15 Laptops 109.95 men's clothing Your perfect pack for everyday use and walks in the forest. Stash your laptop (up to 15 inches) in the padded sleeve, your everyday https://fakestoreapi.com/img/81fPKd-2AYL._AC_SL1500_.jpg}]
// {1 Fjallraven - Foldsack No. 1 Backpack, Fits 15 Laptops 109.95 men's clothing Your perfect pack for everyday use and walks in the forest. Stash your laptop (up to 15 inches) in the padded sleeve, your everyday https://fakestoreapi.com/img/81fPKd-2AYL._AC_SL1500_.jpg}
}
23 changes: 23 additions & 0 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,28 @@ func (c *TwitterConfig) TwitterQuery() string {
return fmt.Sprintf("(%s) %s", query, strings.Join(hashtags, " "))
}

type RedditConfig struct {
// PersonalUseScript is the ID of the personal use script that was set up for game-scout scraping.
PersonalUseScript string `json:"personal_use_script"`
// Secret is the secret that must be sent to the Reddit API access-token endpoint to acquire an OAuth token.
Secret string `json:"secret"`
// UserAgent that is used in requests to the Reddit API to identify game-scout.
UserAgent string `json:"user_agent"`
// Username for the Reddit account related to the personal use script.
Username string `json:"username"`
// Password for the Reddit account related to the personal use script.
Password string `json:"password"`
// Subreddits is the list of subreddits to scrape in the form: "GameDevelopment" (sans "r/" prefix).
Subreddits []string `json:"subreddits"`
}

func (rc *RedditConfig) RedditPersonalUseScript() string { return rc.PersonalUseScript }
func (rc *RedditConfig) RedditSecret() string { return rc.Secret }
func (rc *RedditConfig) RedditUserAgent() string { return rc.UserAgent }
func (rc *RedditConfig) RedditUsername() string { return rc.Username }
func (rc *RedditConfig) RedditPassword() string { return rc.Password }
func (rc *RedditConfig) RedditSubreddits() []string { return rc.Subreddits }

type MondayMappingConfig struct {
// ModelName is the name of the model that this MondayMappingConfig is for. This should either be "models.SteamApp"
// or "models.Game".
Expand Down Expand Up @@ -757,6 +779,7 @@ type Config struct {
Email *EmailConfig `json:"email"`
Tasks *TaskConfig `json:"tasks"`
Twitter *TwitterConfig `json:"twitter"`
Reddit *RedditConfig `json:"reddit"`
Monday *MondayConfig `json:"monday,omitempty"`
Scrape *ScrapeConfig `json:"scrape"`
SteamWebPipes *SteamWebPipesConfig `json:"SteamWebPipes"`
Expand Down
41 changes: 26 additions & 15 deletions db/models/game.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"github.com/RichardKnop/machinery/v1/log"
"github.com/anaskhan96/soup"
"github.com/andygello555/game-scout/api"
"github.com/andygello555/game-scout/browser"
myErrors "github.com/andygello555/game-scout/errors"
"github.com/andygello555/game-scout/monday"
Expand All @@ -14,7 +15,6 @@ import (
mapset "github.com/deckarep/golang-set/v2"
"github.com/google/uuid"
"github.com/lib/pq"
"github.com/machinebox/graphql"
"github.com/pkg/errors"
"github.com/volatiletech/null/v9"
"gorm.io/gorm"
Expand Down Expand Up @@ -362,10 +362,10 @@ func (g *Game) Update(db *gorm.DB, config ScrapeConfig) error {
return db.Omit(g.OnCreateOmit()...).Save(g).Error
}

// GetGamesFromMonday is a monday.Binding to retrieve multiple Game from the mapped board and group. Arguments provided
// GetGamesFromMonday is a api.Binding to retrieve multiple Game from the mapped board and group. Arguments provided
// to Execute:
//
// • page (int): The page of results to retrieve. This means that GetGamesFromMonday can be passed to a monday.Paginator.
// • page (int): The page of results to retrieve. This means that GetGamesFromMonday can be passed to an api.Paginator.
//
// • config (monday.Config): The monday.Config to use to find the monday.MappingConfig for the Game model.
//
Expand All @@ -375,15 +375,17 @@ func (g *Game) Update(db *gorm.DB, config ScrapeConfig) error {
// Execute returns a list of Game instances within their mapped board and group combination for the given page of
// results. It does this by retrieving the Game.ID from the appropriate column from each item and then searching the
// gorm.DB instance which is provided in the 3rd argument.
var GetGamesFromMonday = monday.NewBinding[monday.ItemResponse, []*Game](
func(args ...any) *graphql.Request {
var GetGamesFromMonday = api.NewBinding[monday.ItemResponse, []*Game](
func(b api.Binding[monday.ItemResponse, []*Game], args ...any) api.Request {
page := args[0].(int)
mapping := args[1].(monday.Config).MondayMappingForModel(Game{})
boardIds := mapping.MappingBoardIDs()
groupIds := mapping.MappingGroupIDs()
return monday.GetItems.Request(page, boardIds, groupIds)
},
func(response monday.ItemResponse, args ...any) []*Game {
monday.ResponseWrapper[monday.ItemResponse, []*Game],
monday.ResponseUnwrapped[monday.ItemResponse, []*Game],
func(b api.Binding[monday.ItemResponse, []*Game], response monday.ItemResponse, args ...any) []*Game {
items := monday.GetItems.Response(response)
mapping := args[1].(monday.Config).MondayMappingForModel(Game{})
db := args[2].(*gorm.DB)
Expand Down Expand Up @@ -455,8 +457,9 @@ var GetGamesFromMonday = monday.NewBinding[monday.ItemResponse, []*Game](
game.Votes = int32(voteValues[0] - voteValues[1])
}
return games
},
"boards", true,
}, true,
func(client api.Client) (string, any) { return "jsonResponseKey", "boards" },
func(client api.Client) (string, any) { return "config", client.(*monday.Client).Config },
)

// AddGameToMonday adds a Game to the mapped board and group by constructing column values using the
Expand All @@ -470,8 +473,8 @@ var GetGamesFromMonday = monday.NewBinding[monday.ItemResponse, []*Game](
//
// Execute returns the item ID of the newly created item. This can then be used to set the Game.Watched field
// appropriately if necessary.
var AddGameToMonday = monday.NewBinding[monday.ItemId, string](
func(args ...any) *graphql.Request {
var AddGameToMonday = api.NewBinding[monday.ItemId, string](
func(b api.Binding[monday.ItemId, string], args ...any) api.Request {
game := args[0].(*Game)
itemName := game.Website.String
if game.Name.IsValid() {
Expand All @@ -489,10 +492,14 @@ var AddGameToMonday = monday.NewBinding[monday.ItemId, string](
columnValues,
)
},
monday.AddItem.Response, "create_item", false,
monday.ResponseWrapper[monday.ItemId, string],
monday.ResponseUnwrapped[monday.ItemId, string],
monday.AddItem.GetResponseMethod(), false,
func(client api.Client) (string, any) { return "jsonResponseKey", "create_item" },
func(client api.Client) (string, any) { return "config", client.(*monday.Client).Config },
)

// UpdateGameInMonday is a monday.Binding which updates the monday.Item of the given ID within the monday.Board of the
// UpdateGameInMonday is an api.Binding which updates the monday.Item of the given ID within the monday.Board of the
// given ID for Game using the monday.MappingConfig.ColumnValues method to generate values for all the monday.Column IDs
// provided by the monday.MappingConfig.MappingColumnsToUpdate method. Arguments provided to Execute:
//
Expand All @@ -507,8 +514,8 @@ var AddGameToMonday = monday.NewBinding[monday.ItemId, string](
// values of the monday.Item of the given ID in the mapped monday.Board.
//
// Execute returns the ID of the monday.Item that has been mutated.
var UpdateGameInMonday = monday.NewBinding[monday.ItemId, string](
func(args ...any) *graphql.Request {
var UpdateGameInMonday = api.NewBinding[monday.ItemId, string](
func(b api.Binding[monday.ItemId, string], args ...any) api.Request {
game := args[0].(*Game)
itemId := args[1].(int)
boardId := args[2].(int)
Expand All @@ -523,7 +530,11 @@ var UpdateGameInMonday = monday.NewBinding[monday.ItemId, string](
columnValues,
)
},
monday.ChangeMultipleColumnValues.Response, "change_multiple_column_values", false,
monday.ResponseWrapper[monday.ItemId, string],
monday.ResponseUnwrapped[monday.ItemId, string],
monday.ChangeMultipleColumnValues.GetResponseMethod(), false,
func(client api.Client) (string, any) { return "jsonResponseKey", "boards" },
func(client api.Client) (string, any) { return "config", client.(*monday.Client).Config },
)

func (g *Game) GetVerifiedDeveloperUsernames() []string { return g.VerifiedDeveloperUsernames }
Expand Down
Loading

0 comments on commit 95b16a1

Please sign in to comment.