Skip to content

Commit

Permalink
Minor cleanup, added filter functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ordspilleren
Ordspilleren committed Jun 7, 2021
1 parent 4ac0d5a commit 12071d5
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 136 deletions.
2 changes: 1 addition & 1 deletion html/html.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ type MonitorListParams struct {
}

type MonitorNewParams struct {
Monitor *monitor.Monitor
Monitor monitor.Monitor
Success bool
}

Expand Down
10 changes: 5 additions & 5 deletions html/monitornew.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,35 +9,35 @@
<div class="field">
<label class="label">Name</label>
<div class="control">
<input name="name" class="input" type="text"{{if .Monitor}} value="{{.Monitor.Name}}"{{end}}>
<input name="name" class="input" type="text"{{with .Monitor.Name}} value="{{.}}"{{end}}>
</div>
</div>

<div class="field">
<label class="label">URL</label>
<div class="control">
<input name="url" class="input" type="text"{{if .Monitor}} value="{{.Monitor.URL}}"{{end}}>
<input name="url" class="input" type="text"{{with .Monitor.URL}} value="{{.}}"{{end}}>
</div>
</div>

<div class="field">
<label class="label">Interval</label>
<div class="control">
<input name="interval" class="input" type="number"{{if .Monitor}} value="{{.Monitor.Interval.Nanoseconds}}"{{end}}>
<input name="interval" class="input" type="number"{{with .Monitor.Interval.Nanoseconds}} value="{{.}}"{{end}}>
</div>
</div>

<div class="field">
<label class="label">CSS Selectors</label>
<div class="control">
<textarea name="cssselectors" class="textarea">{{if .Monitor.Selectors.CSS}}{{StringsJoin .Monitor.Selectors.CSS "\n"}}{{end}}</textarea>
<textarea name="cssselectors" class="textarea">{{with .Monitor.Selectors.CSS}}{{StringsJoin . "\n"}}{{end}}</textarea>
</div>
</div>

<div class="field">
<label class="label">JSON Selectors</label>
<div class="control">
<textarea name="jsonselectors" class="textarea">{{if .Monitor.Selectors.JSON}}{{StringsJoin .Monitor.Selectors.JSON "\n"}}{{end}}</textarea>
<textarea name="jsonselectors" class="textarea">{{with .Monitor.Selectors.JSON}}{{StringsJoin . "\n"}}{{end}}</textarea>
</div>
</div>

Expand Down
94 changes: 4 additions & 90 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,10 @@ import (
"encoding/json"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"sync"

"github.com/Ordspilleren/ChangeMonitor/html"
"github.com/Ordspilleren/ChangeMonitor/monitor"
"github.com/Ordspilleren/ChangeMonitor/notify"
"github.com/Ordspilleren/ChangeMonitor/storage"
Expand Down Expand Up @@ -61,9 +58,9 @@ func init() {
}

notifierService := notify.NewNotifierService(config.Notifiers)
storageManager := storage.InitStorage(StorageDirectory)
storage := storage.InitStorage(StorageDirectory)

monitorService = monitor.NewMonitorService(wg, config.Monitors, storageManager, notifierService)
monitorService = monitor.NewMonitorService(wg, config.Monitors, storage, notifierService)
if ChromeWs != "" {
monitorService.NewMonitorClients(ChromeWs, true)
} else {
Expand All @@ -76,7 +73,8 @@ func main() {
monitorService.StartMonitoring()

if EnableWebUI {
startHTTPServer()
server := server{}
server.start()
} else {
wg.Wait()
}
Expand All @@ -90,87 +88,3 @@ func (t *Config) JSON() ([]byte, error) {
err := encoder.Encode(t)
return buffer.Bytes(), err
}

func startHTTPServer() {
http.Handle("/assets/", http.FileServer(html.GetAssetFS()))
http.HandleFunc("/", monitorList)
http.HandleFunc("/new", monitorNew)
http.ListenAndServe(":8080", nil)
}

func monitorList(w http.ResponseWriter, r *http.Request) {
p := html.MonitorListParams{
MonitorService: monitorService,
}

if r.Method != http.MethodPost {
html.MonitorList(w, p)
return
}

monitorID, err := strconv.ParseInt(r.FormValue("monitorid"), 10, 64)
if err != nil {
log.Print(err)
}
startMonitor := r.FormValue("start")
stopMonitor := r.FormValue("stop")
if startMonitor != "" {
p.MonitorService.Monitors[monitorID].Start(p.MonitorService.WaitGroup)
}
if stopMonitor != "" {
p.MonitorService.Monitors[monitorID].Stop()
}

html.MonitorList(w, p)
}

func monitorNew(w http.ResponseWriter, r *http.Request) {
p := html.MonitorNewParams{}

monitorID := r.URL.Query().Get("id")
if monitorID != "" {
id, _ := strconv.ParseInt(monitorID, 10, 64)
p.Monitor = &monitorService.Monitors[id]
}

if r.Method != http.MethodPost {
html.MonitorNew(w, p)
return
}

name := r.FormValue("name")
url := r.FormValue("url")
interval, err := strconv.ParseInt(r.FormValue("interval"), 10, 64)
if err != nil {
log.Print(err)
}
cssSelectors := r.FormValue("cssselectors")
jsonSelectors := r.FormValue("jsonselectors")

monitor := monitor.NewMonitor(name, url, interval)

if cssSelectors != "" {
cssSelectorSlice := strings.Split(cssSelectors, "\n")
monitor.AddCSSSelectors(cssSelectorSlice...)
}
if jsonSelectors != "" {
jsonSelectorSlice := strings.Split(jsonSelectors, "\n")
monitor.AddCSSSelectors(jsonSelectorSlice...)
}

monitor.Init(*monitorService.NotifierService, *monitorService.Storage, *monitorService.HttpClient, *monitorService.ChromeClient)

p.Success = true

monitorService.AddMonitors(*monitor)

config.Monitors = append(config.Monitors, *monitor)

newConfig, _ := config.JSON()
err = ioutil.WriteFile(ConfigFile, newConfig, 0644)
if err != nil {
log.Print(err)
}

html.MonitorNew(w, p)
}
95 changes: 55 additions & 40 deletions monitor/monitor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
)

type MonitorClient interface {
GetContent(url string, httpHeaders http.Header, selectors Selectors) (string, error)
GetContent(url string, httpHeaders http.Header) (io.ReadCloser, error)
}

type Storage interface {
Expand Down Expand Up @@ -56,8 +56,9 @@ type Monitor struct {
URL string `json:"url"`
HTTPHeaders http.Header `json:"httpHeaders,omitempty"`
UseChrome bool `json:"useChrome"`
Selectors Selectors `json:"selectors,omitempty"`
Interval time.Duration `json:"interval"`
Selectors Selectors `json:"selectors,omitempty"`
Filters Filters `json:"filters,omitempty"`
notifierService NotifierService
started bool
doneChannel chan bool
Expand All @@ -74,6 +75,8 @@ type Selectors struct {
JSON *[]string `json:"json,omitempty"`
}

type Filters []string

func NewMonitorService(wg *sync.WaitGroup, monitors Monitors, storage Storage, notifierService NotifierService) *MonitorService {
monitorService := MonitorService{
WaitGroup: wg,
Expand Down Expand Up @@ -208,55 +211,87 @@ func generateSHA1String(input string) string {
func (m *Monitor) check() {
log.Print(m.URL)

selectorContent, err := m.client.GetContent(m.URL, m.HTTPHeaders, m.Selectors)
content, err := m.client.GetContent(m.URL, m.HTTPHeaders)
if err != nil {
log.Print(err)
return
}
defer content.Close()

processedContent, err := processContent(content, m.Selectors, m.Filters)
if err != nil {
log.Print(err)
return
}

storageContent := m.storage.GetContent(m.id)

if m.compareContent(storageContent, selectorContent) {
m.storage.WriteContent(m.id, selectorContent)
if compareContent(storageContent, processedContent) {
m.storage.WriteContent(m.id, processedContent)
log.Print("Content has changed!")
_ = m.notifierService.Send(
context.Background(),
fmt.Sprintf("<b><u>%s has changed!</u></b>", m.Name),
fmt.Sprintf("<b>New content:</b>\n%.200s\n\n<b>Old content:</b>\n%.200s\n\n<b>URL:</b> %s", selectorContent, storageContent, m.URL),
fmt.Sprintf("<b>New content:</b>\n%.200s\n\n<b>Old content:</b>\n%.200s\n\n<b>URL:</b> %s", processedContent, storageContent, m.URL),
)
} else {
log.Printf("Nothing has changed, waiting %s.", m.Interval*time.Minute)
}
}

func (h HttpClient) GetContent(url string, httpHeaders http.Header, selectors Selectors) (string, error) {
responseBody, err := h.getHTTPBody(url, httpHeaders)
if err != nil {
return "", err
}
defer responseBody.Close()

func processContent(content io.ReadCloser, selectors Selectors, filters Filters) (string, error) {
var selectorContent string
var err error

if selectors.CSS != nil {
selectorContent, err = getCSSSelectorContent(responseBody, *selectors.CSS)
selectorContent, err = getCSSSelectorContent(content, *selectors.CSS)
if err != nil {
return "", err
}
} else if selectors.JSON != nil {
selectorContent, err = getJSONSelectorContent(responseBody, *selectors.JSON)
selectorContent, err = getJSONSelectorContent(content, *selectors.JSON)
if err != nil {
return "", err
}
} else {
selectorContent, err = getHTMLText(responseBody)
selectorContent, err = getHTMLText(content)
if err != nil {
return "", err
}
}

if len(filters) > 0 {
selectorContent = processFilters(filters, selectorContent)
}

return selectorContent, nil
}

func processFilters(filters Filters, content string) string {
var matchedFilters []string

for _, filter := range filters {
if strings.Contains(content, filter) {
matchedFilters = append(matchedFilters, filter)
}
}

if len(matchedFilters) > 0 {
return strings.Join(matchedFilters, " | ")
} else {
return ""
}
}

func (h HttpClient) GetContent(url string, httpHeaders http.Header) (io.ReadCloser, error) {
responseBody, err := h.getHTTPBody(url, httpHeaders)
if err != nil {
return nil, err
}

return responseBody, nil
}

func (h HttpClient) getHTTPBody(url string, headers http.Header) (io.ReadCloser, error) {
request, err := http.NewRequest("GET", url, nil)
if err != nil {
Expand Down Expand Up @@ -318,40 +353,20 @@ func getJSONSelectorContent(body io.ReadCloser, selectors []string) (string, err
return strings.Join(results, "\n"), nil
}

func (m *Monitor) compareContent(storage string, selector string) bool {
func compareContent(storage string, selector string) bool {
log.Printf("Cache content: %s", storage)
log.Printf("New content: %s", selector)

return storage != selector
}

func (h ChromeClient) GetContent(url string, httpHeaders http.Header, selectors Selectors) (string, error) {
func (h ChromeClient) GetContent(url string, httpHeaders http.Header) (io.ReadCloser, error) {
responseBody, err := h.getHTTPBody(url, httpHeaders)
if err != nil {
return "", err
return nil, err
}
defer responseBody.Close()

var selectorContent string

if selectors.CSS != nil {
selectorContent, err = getCSSSelectorContent(responseBody, *selectors.CSS)
if err != nil {
return "", err
}
} else if selectors.JSON != nil {
selectorContent, err = getJSONSelectorContent(responseBody, *selectors.JSON)
if err != nil {
return "", err
}
} else {
selectorContent, err = getHTMLText(responseBody)
if err != nil {
return "", err
}
}

return selectorContent, nil
return responseBody, nil
}

func (h ChromeClient) getHTTPBody(url string, headers http.Header) (io.ReadCloser, error) {
Expand Down
Loading

0 comments on commit 12071d5

Please sign in to comment.