Skip to content

fetch houses and guildhalls concurrently #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 6, 2022
Merged

fetch houses and guildhalls concurrently #30

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e35810
fetch houses and guildhalls concurrently
phenpessoa Jan 6, 2022
ff4d866
requested changed
phenpessoa Jan 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
253 changes: 128 additions & 125 deletions src/TibiaHousesOverviewV3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,147 +9,77 @@ import (
"github.com/gin-gonic/gin"
)

// Child of House
type Auction struct {
AuctionBid int `json:"current_bid"`
AuctionLeft string `json:"time_left"`
}

// Child of Houses
type House struct {
Name string `json:"name"`
HouseID int `json:"house_id"`
Size int `json:"size"`
Rent int `json:"rent"`
IsRented bool `json:"rented"`
IsAuctioned bool `json:"auctioned"`
Auction Auction `json:"auction"`
}

// Child of JSONData
type Houses struct {
World string `json:"world"`
Town string `json:"town"`
HouseList []House `json:"house_list"`
GuildhallList []House `json:"guildhall_list"`
}

// TibiaHousesOverviewV3 func
func TibiaHousesOverviewV3(c *gin.Context) {

// getting params from URL
world := c.Param("world")
town := c.Param("town")

// Child of House
type Auction struct {
AuctionBid int `json:"current_bid"`
AuctionLeft string `json:"time_left"`
}

// Child of Houses
type House struct {
Name string `json:"name"`
HouseID int `json:"house_id"`
Size int `json:"size"`
Rent int `json:"rent"`
IsRented bool `json:"rented"`
IsAuctioned bool `json:"auctioned"`
Auction Auction `json:"auction"`
}

// Child of JSONData
type Houses struct {
World string `json:"world"`
Town string `json:"town"`
HouseList []House `json:"house_list"`
GuildhallList []House `json:"guildhall_list"`
}
// Adding fix for First letter to be upper and rest lower
world = TibiadataStringWorldFormatToTitleV3(world)
town = TibiadataStringWorldFormatToTitleV3(town)

//
// The base includes two levels: Houses and Information
type JSONData struct {
Houses Houses `json:"houses"`
Information Information `json:"information"`
}

// Creating empty vars
var HouseData, GuildhallData []House
HouseData = []House{}
GuildhallData = []House{}

// Adding fix for First letter to be upper and rest lower
world = TibiadataStringWorldFormatToTitleV3(world)
town = TibiadataStringWorldFormatToTitleV3(town)

// list of different house types
HouseTypes := []string{"houses", "guildhalls"}
// running over the HouseTypes array
for _, HouseType := range HouseTypes {

// Getting data with TibiadataHTMLDataCollectorV3
BoxContentHTML := TibiadataHTMLDataCollectorV3("https://www.tibia.com/community/?subtopic=houses&world=" + TibiadataQueryEscapeStringV3(world) + "&town=" + TibiadataQueryEscapeStringV3(town) + "&type=" + TibiadataQueryEscapeStringV3(HouseType))

// Loading HTML data into ReaderHTML for goquery with NewReader
ReaderHTML, err := goquery.NewDocumentFromReader(strings.NewReader(BoxContentHTML))
if err != nil {
log.Fatal(err)
var (
// Creating empty vars
HouseData, GuildhallData []House

// Channels
done = make(chan struct{})
housesChan = make(chan House)
guildhallsChan = make(chan House)

baseURL = "https://www.tibia.com/community/?subtopic=houses&world=" + TibiadataQueryEscapeStringV3(world) + "&town=" + TibiadataQueryEscapeStringV3(town) + "&type="
)

go houseFetcher(baseURL, "houses", done, housesChan)
go houseFetcher(baseURL, "guildhalls", done, guildhallsChan)

for n := 2; n > 0; {
select {
case h := <-housesChan:
HouseData = append(HouseData, h)
case gh := <-guildhallsChan:
GuildhallData = append(GuildhallData, gh)
case <-done:
n--
}

// Running query over each div
ReaderHTML.Find(".BoxContent table tr").Each(func(index int, s *goquery.Selection) {

// Storing HTML into HousesDivHTML
HousesDivHTML, err := s.Html()
if err != nil {
log.Fatal(err)
}

// Removing linebreaks from HTML
HousesDivHTML = TibiadataHTMLRemoveLinebreaksV3(HousesDivHTML)

// Regex to get data for record values
regex1 := regexp.MustCompile(`<td.*><nobr>(.*)<\/nobr><\/td><td.*><nobr>([0-9]+).sqm<\/nobr><\/td><td.*><nobr>([0-9]+)(k+).gold<\/nobr><\/td><td.*><nobr>(.*)<\/nobr><\/td>.*houseid" value="([0-9]+)"\/><div.*`)
subma1 := regex1.FindAllStringSubmatch(HousesDivHTML, -1)

if len(subma1) > 0 {

// Default of vars
IsRented := false
IsAuctioned := false
AuctionBid := 0
AuctionLeft := ""

switch {
case strings.Contains(subma1[0][5], "rented"):
IsRented = true
case strings.Contains(subma1[0][5], "no bid yet"):
// nothing to set?
case strings.Contains(subma1[0][5], "auctioned"):
IsAuctioned = true
regex1b := regexp.MustCompile(`auctioned..([0-9]+).gold..(.*).`)
subma1b := regex1b.FindAllStringSubmatch(subma1[0][5], -1)
AuctionBid = TibiadataStringToIntegerV3(subma1b[0][1])
AuctionLeft = subma1b[0][2]
}

// Name
Name := TibiaDataSanitizeEscapedString(subma1[0][1])
// HouseID
HouseID := TibiadataStringToIntegerV3(subma1[0][6])
// Size
Size := TibiadataStringToIntegerV3(subma1[0][2])
// Rent
Rent := TibiaDataConvertValuesWithK(subma1[0][3] + subma1[0][4])

switch HouseType {
case "houses":
HouseData = append(HouseData, House{
Name: Name,
HouseID: HouseID,
Size: Size,
Rent: Rent,
IsRented: IsRented,
IsAuctioned: IsAuctioned,
Auction: Auction{
AuctionBid: AuctionBid,
AuctionLeft: AuctionLeft,
},
})
case "guildhalls":
GuildhallData = append(GuildhallData, House{
Name: Name,
HouseID: HouseID,
Size: Size,
Rent: Rent,
IsRented: IsRented,
IsAuctioned: IsAuctioned,
Auction: Auction{
AuctionBid: AuctionBid,
AuctionLeft: AuctionLeft,
},
})
}
}
})

}

close(done)
close(guildhallsChan)
close(housesChan)

//
// Build the data-blob
jsonData := JSONData{
Expand All @@ -168,3 +98,76 @@ func TibiaHousesOverviewV3(c *gin.Context) {
// return jsonData
TibiaDataAPIHandleSuccessResponse(c, "TibiaHousesOverviewV3", jsonData)
}

func houseFetcher(baseURL, houseType string, done chan struct{}, outputChan chan House) {
// Getting data with TibiadataHTMLDataCollectorV3
BoxContentHTML := TibiadataHTMLDataCollectorV3(baseURL + TibiadataQueryEscapeStringV3(houseType))

// Loading HTML data into ReaderHTML for goquery with NewReader
ReaderHTML, err := goquery.NewDocumentFromReader(strings.NewReader(BoxContentHTML))
if err != nil {
log.Fatal(err)
}

ReaderHTML.Find(".BoxContent table tr").Each(func(index int, s *goquery.Selection) {
// Storing HTML into HousesDivHTML
HousesDivHTML, err := s.Html()
if err != nil {
log.Fatal(err)
}

// Removing linebreaks from HTML
HousesDivHTML = TibiadataHTMLRemoveLinebreaksV3(HousesDivHTML)

// Regex to get data for record values
regex1 := regexp.MustCompile(`<td.*><nobr>(.*)<\/nobr><\/td><td.*><nobr>([0-9]+).sqm<\/nobr><\/td><td.*><nobr>([0-9]+)(k+).gold<\/nobr><\/td><td.*><nobr>(.*)<\/nobr><\/td>.*houseid" value="([0-9]+)"\/><div.*`)
subma1 := regex1.FindAllStringSubmatch(HousesDivHTML, -1)

if len(subma1) > 0 {
// Default vars
var (
IsRented, IsAuctioned bool
AuctionBid int
AuctionLeft string
)

s := subma1[0][5]
switch {
case strings.Contains(s, "rented"):
IsRented = true
// case strings.Contains(s, "no bid yet"):
// nothing to set?
case strings.Contains(s, "auctioned"):
IsAuctioned = true
regex1b := regexp.MustCompile(`auctioned..([0-9]+).gold..(.*).`)
subma1b := regex1b.FindAllStringSubmatch(s, -1)
AuctionBid = TibiadataStringToIntegerV3(subma1b[0][1])
AuctionLeft = subma1b[0][2]
}

// Name
Name := TibiaDataSanitizeEscapedString(subma1[0][1])
// HouseID
HouseID := TibiadataStringToIntegerV3(subma1[0][6])
// Size
Size := TibiadataStringToIntegerV3(subma1[0][2])
// Rent
Rent := TibiaDataConvertValuesWithK(subma1[0][3] + subma1[0][4])

outputChan <- House{
Name: Name,
HouseID: HouseID,
Size: Size,
Rent: Rent,
IsRented: IsRented,
IsAuctioned: IsAuctioned,
Auction: Auction{
AuctionBid: AuctionBid,
AuctionLeft: AuctionLeft,
},
}
}
})

done <- struct{}{}
}