-
Notifications
You must be signed in to change notification settings - Fork 0
/
DataPuller.go
66 lines (53 loc) · 1.25 KB
/
DataPuller.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
package utils
import (
"github.com/grokify/html-strip-tags-go"
"github.com/landonp1203/goUtils/aws"
"github.com/landonp1203/goUtils/common"
"github.com/landonp1203/goUtils/loggly"
"github.com/landonp1203/goUtils/networking"
"jobListingsWorker/types"
)
const TableName = "Job-Listings"
func GetJobsJob() {
jobs := getJobPostings()
if jobs != nil {
sendJobsToDB(jobs)
} else {
loggly.Warn("The jobs list was empty.")
}
}
// Retrieves job postings.
func getJobPostings() [] *types.GithubJob {
var jsonData, err = networking.Get("https://jobs.github.com/positions.json?search=java")
if err != nil {
loggly.Error(err)
return nil
} else {
var jobs [] *types.GithubJob
err := common.DecodeJson(jsonData, &jobs)
if err != nil {
loggly.Error(err)
return nil
}
for _, j := range jobs { // strips html tags from `HowToApply` field
stripped := strip.StripTags(j.HowToApply)
j.HowToApply = stripped
}
return jobs
}
}
// Helper to send the jobs array to the database
func sendJobsToDB(jobs [] *types.GithubJob) {
client, err := aws.CreateDynamoClient()
if err != nil {
loggly.Error(err)
return
}
for _, j := range jobs {
err := aws.PutItem(client, TableName, *j)
if err != nil {
loggly.Error(err)
break
}
}
}