From 66c4b3de6c74e665f517aba0d964dc63a54898de Mon Sep 17 00:00:00 2001 From: SebbeJohansson Date: Fri, 2 Jan 2026 17:37:37 +0100 Subject: [PATCH 1/5] Adds intial support for database --- .gitignore | 6 +++ README.md | 3 ++ database.go | 70 ++++++++++++++++++++++++++ emoProxy.conf => emoProxy.conf.example | 3 +- emoProxy.go | 67 +++++++++++++++++++++--- go.mod | 17 ++++++- go.sum | 23 +++++++++ 7 files changed, 179 insertions(+), 10 deletions(-) create mode 100644 database.go rename emoProxy.conf => emoProxy.conf.example (73%) create mode 100644 go.sum diff --git a/.gitignore b/.gitignore index 438cc37..095d9f5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,9 @@ *.code-workspace **/bin/ **/obj/ + +# App specific ignores +emoProxy.conf +data/ +# tmp is used for air +tmp/ diff --git a/README.md b/README.md index 800f446..06bd8d7 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ This part of the knowledge bank is a Proxy server to analyze the traffic that go ## Start the proxy `go run emoProxy.go` +### Alternatively start with air +`air run emoProxy.go` + ## Docker setup You can find a simplified setup using Docker Compose by SebbeJohansson here: https://github.com/SebbeJohansson/emo-proxy-docker It uses nginx, dnsmasq, and mitmproxy to be able to pass through the api requests to the EMO Proxy. diff --git a/database.go b/database.go new file mode 100644 index 0000000..d74d379 --- /dev/null +++ b/database.go @@ -0,0 +1,70 @@ +package main + +import ( + "database/sql" + "fmt" + + _ "modernc.org/sqlite" +) + +var DB *sql.DB // Capitalized to be visible (though not strictly necessary if in same package) + +func InitDB(path string) error { + _db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + DB = _db // Assign to global DB variable + // Create a simple table for intercepted data + query := ` + CREATE TABLE IF NOT EXISTS requests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + endpoint TEXT, + payload TEXT, + response TEXT + );` + + _, err = DB.Exec(query) + return err +} + +func saveRequest(requestEndPoint string, payload string, response string) { + fmt.Println("Saving request to DB...") + _, err := DB.Exec("INSERT INTO requests (endpoint, payload, response) VALUES (?, ?, ?)", requestEndPoint, payload, response) + if err != nil { + fmt.Println("Failed to save to DB: ", err) + } +} + +func getAllRequests() ([]map[string]interface{}, error) { + rows, err := DB.Query("SELECT id, timestamp, endpoint, payload, response FROM requests") + if err != nil { + return nil, err + } + defer rows.Close() + + var results []map[string]interface{} + for rows.Next() { + var id int + var timestamp string + var endpoint string + var payload string + var response string + + err := rows.Scan(&id, ×tamp, &endpoint, &payload, &response) + if err != nil { + return nil, err + } + + record := map[string]interface{}{ + "id": id, + "timestamp": timestamp, + "endpoint": endpoint, + "payload": payload, + "response": response, + } + results = append(results, record) + } + return results, rows.Err() +} diff --git a/emoProxy.conf b/emoProxy.conf.example similarity index 73% rename from emoProxy.conf rename to emoProxy.conf.example index d882ce9..8ba3dd4 100644 --- a/emoProxy.conf +++ b/emoProxy.conf.example @@ -5,5 +5,6 @@ "livingio_tts_server": "eu-tts.living.ai", "livingio_res_server": "res.living.ai", "postFS": "/tmp/", - "logFileName": "/var/log/emoProxyss.log" + "logFileName": "/var/log/emoProxyss.log", + "sqliteLocation": "/var/data/emo_logs.db" } \ No newline at end of file diff --git a/emoProxy.go b/emoProxy.go index 3c87cb8..c029da5 100644 --- a/emoProxy.go +++ b/emoProxy.go @@ -32,6 +32,7 @@ type Configuration struct { Livingio_RES_Server string `json:"livingio_res_server"` PostFS string `json:"postFS"` LogFileName string `json:"logFileName"` + SqliteLocation string `json:"sqliteLocation"` } var ( @@ -39,6 +40,7 @@ var ( ) func main() { + log.Println("Starting application...") //load config confFile := flag.String("c", "emoProxy.conf", "config file to use") flag.Parse() @@ -47,25 +49,37 @@ func main() { if err != nil { log.Println("can't read conf file", *confFile, "- using default config") } - + log.Println("config loaded") writePid() // disable ssl checks http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + // parse flags + Port := flag.Int("port", 8080, "http port") + flag.Parse() + log.Println("Starting app on port: ", *Port) + // redirect log logFile, err := os.OpenFile(conf.LogFileName, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0644) if err != nil { log.Panic(err) } + defer logFile.Close() log.SetOutput(logFile) log.SetFlags(log.Lshortfile | log.LstdFlags) - // parse flags - Port := flag.Int("port", 8081, "http port") + dbPath := conf.SqliteLocation + flagDbPath := flag.String("db", "", "path to the sqlite database file") + if *flagDbPath != "" { + dbPath = *flagDbPath + } flag.Parse() - log.Println("Port: ", *Port) + dbCreateErr := InitDB(dbPath) + if dbCreateErr != nil { + log.Panic(dbCreateErr) + } // handle time requests http.HandleFunc("/time", func(w http.ResponseWriter, r *http.Request) { @@ -152,7 +166,38 @@ func main() { fmt.Fprint(w, body) }) - log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*Port), nil)) + // proxy-api endpoints + http.HandleFunc("/proxy-api/requests", func(w http.ResponseWriter, r *http.Request) { + logRequest(r) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + requests, err := getAllRequests() + if err != nil { + http.Error(w, fmt.Sprintf(`{"error":"%v"}`, err), http.StatusInternalServerError) + return + } + json.NewEncoder(w).Encode(requests) + }) + + log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*Port), corsMiddleware(http.DefaultServeMux))) +} + +func corsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Replace "*" with "http://localhost:3000" for better security + w.Header().Set("Access-Control-Allow-Origin", "http://localhost:3000") + w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") + w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization") + + // Handle the preflight OPTIONS request + if r.Method == "OPTIONS" { + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) } func loadConfig(filename string) error { @@ -164,6 +209,7 @@ func loadConfig(filename string) error { Livingio_RES_Server: "res.living.ai", PostFS: "/tmp/", LogFileName: "/var/log/emoProxy.log", + SqliteLocation: "/var/data/emo_logs.db", } bytes, err := os.ReadFile(filename) @@ -228,16 +274,17 @@ func logBody(contentType string, body []byte, prefix string) { func makeApiRequest(r *http.Request) string { var request *http.Request + var requestBody []byte switch r.Method { case "GET": request, _ = http.NewRequest("GET", "https://"+conf.Livingio_API_Server+r.URL.RequestURI(), nil) case "POST": - body, _ := io.ReadAll(r.Body) + requestBody, _ := io.ReadAll(r.Body) // write post request body to fs - logBody(r.Header.Get("Content-Type"), body, "apiReq_") + logBody(r.Header.Get("Content-Type"), requestBody, "apiReq_") - request, _ = http.NewRequest("POST", "https://"+conf.Livingio_API_Server+r.URL.RequestURI(), bytes.NewBuffer(body)) + request, _ = http.NewRequest("POST", "https://"+conf.Livingio_API_Server+r.URL.RequestURI(), bytes.NewBuffer(requestBody)) request.Header.Add("Content-Type", r.Header.Get("Content-Type")) request.Header.Add("Content-Length", r.Header.Get("Content-Length")) @@ -269,6 +316,7 @@ func makeApiRequest(r *http.Request) string { log.Println("Server response: ", string(body)) logResponse(response) + saveRequest(r.URL.RequestURI(), string(requestBody), string(body)) return string(body) } @@ -301,6 +349,7 @@ func makeTtsRequest(r *http.Request) string { // write post request body to fs logBody(response.Header.Get("Content-Type"), body, "tts_") logResponse(response) + saveRequest(r.URL.RequestURI(), "", "") return string(body) } @@ -333,6 +382,7 @@ func makeApiTtsRequest(r *http.Request) string { // write post request body to fs logBody(response.Header.Get("Content-Type"), body, "apitts_") logResponse(response) + saveRequest(r.URL.RequestURI(), "", string(body)) return string(body) } @@ -369,5 +419,6 @@ func makeResRequest(r *http.Request, w http.ResponseWriter) string { } logResponse(response) + saveRequest(r.URL.RequestURI(), "", string(body)) return string(body) } diff --git a/go.mod b/go.mod index 807fe65..b4ac792 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,18 @@ module emoProxy -go 1.17 +go 1.24.0 + +require modernc.org/sqlite v1.42.2 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/sys v0.36.0 // indirect + modernc.org/libc v1.66.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3f0d412 --- /dev/null +++ b/go.sum @@ -0,0 +1,23 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74= +modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= From a05c81e6b766c31020b4e40e0459a4d2b7c4554f Mon Sep 17 00:00:00 2001 From: SebbeJohansson Date: Fri, 2 Jan 2026 17:50:17 +0100 Subject: [PATCH 2/5] Adds option to enable database and api. --- database.go | 6 ++-- emoProxy.conf.example | 1 + emoProxy.go | 77 ++++++++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/database.go b/database.go index d74d379..d70d1d6 100644 --- a/database.go +++ b/database.go @@ -2,7 +2,7 @@ package main import ( "database/sql" - "fmt" + "log" _ "modernc.org/sqlite" ) @@ -30,10 +30,10 @@ func InitDB(path string) error { } func saveRequest(requestEndPoint string, payload string, response string) { - fmt.Println("Saving request to DB...") + log.Println("Saving request to DB...") _, err := DB.Exec("INSERT INTO requests (endpoint, payload, response) VALUES (?, ?, ?)", requestEndPoint, payload, response) if err != nil { - fmt.Println("Failed to save to DB: ", err) + log.Println("Failed to save to DB: ", err) } } diff --git a/emoProxy.conf.example b/emoProxy.conf.example index 8ba3dd4..031636d 100644 --- a/emoProxy.conf.example +++ b/emoProxy.conf.example @@ -6,5 +6,6 @@ "livingio_res_server": "res.living.ai", "postFS": "/tmp/", "logFileName": "/var/log/emoProxyss.log", + "enableDatabaseAndAPI": false, # For now, default behavior is still without db and api. "sqliteLocation": "/var/data/emo_logs.db" } \ No newline at end of file diff --git a/emoProxy.go b/emoProxy.go index c029da5..8068de2 100644 --- a/emoProxy.go +++ b/emoProxy.go @@ -32,11 +32,13 @@ type Configuration struct { Livingio_RES_Server string `json:"livingio_res_server"` PostFS string `json:"postFS"` LogFileName string `json:"logFileName"` + EnableDatabaseAndAPI bool `json:"enableDatabaseAndAPI"` SqliteLocation string `json:"sqliteLocation"` } var ( - conf Configuration + conf Configuration + useDatabaseAndAPI bool = false ) func main() { @@ -70,15 +72,23 @@ func main() { log.SetOutput(logFile) log.SetFlags(log.Lshortfile | log.LstdFlags) - dbPath := conf.SqliteLocation - flagDbPath := flag.String("db", "", "path to the sqlite database file") - if *flagDbPath != "" { - dbPath = *flagDbPath - } - flag.Parse() - dbCreateErr := InitDB(dbPath) - if dbCreateErr != nil { - log.Panic(dbCreateErr) + useDatabaseAndAPI = conf.EnableDatabaseAndAPI + + if useDatabaseAndAPI { + log.Println("Database and API enabled") + + dbPath := conf.SqliteLocation + flagDbPath := flag.String("db", "", "path to the sqlite database file") + if *flagDbPath != "" { + dbPath = *flagDbPath + } + flag.Parse() + dbCreateErr := InitDB(dbPath) + if dbCreateErr != nil { + log.Panic(dbCreateErr) + } + } else { + log.Println("Note: Database and API disabled") } // handle time requests @@ -166,19 +176,21 @@ func main() { fmt.Fprint(w, body) }) - // proxy-api endpoints - http.HandleFunc("/proxy-api/requests", func(w http.ResponseWriter, r *http.Request) { - logRequest(r) - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) + if useDatabaseAndAPI { + // proxy-api endpoints + http.HandleFunc("/proxy-api/requests", func(w http.ResponseWriter, r *http.Request) { + logRequest(r) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) - requests, err := getAllRequests() - if err != nil { - http.Error(w, fmt.Sprintf(`{"error":"%v"}`, err), http.StatusInternalServerError) - return - } - json.NewEncoder(w).Encode(requests) - }) + requests, err := getAllRequests() + if err != nil { + http.Error(w, fmt.Sprintf(`{"error":"%v"}`, err), http.StatusInternalServerError) + return + } + json.NewEncoder(w).Encode(requests) + }) + } log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*Port), corsMiddleware(http.DefaultServeMux))) } @@ -209,6 +221,7 @@ func loadConfig(filename string) error { Livingio_RES_Server: "res.living.ai", PostFS: "/tmp/", LogFileName: "/var/log/emoProxy.log", + EnableDatabaseAndAPI: false, SqliteLocation: "/var/data/emo_logs.db", } @@ -316,7 +329,10 @@ func makeApiRequest(r *http.Request) string { log.Println("Server response: ", string(body)) logResponse(response) - saveRequest(r.URL.RequestURI(), string(requestBody), string(body)) + + if useDatabaseAndAPI { + saveRequest(r.URL.RequestURI(), string(requestBody), string(body)) + } return string(body) } @@ -349,7 +365,10 @@ func makeTtsRequest(r *http.Request) string { // write post request body to fs logBody(response.Header.Get("Content-Type"), body, "tts_") logResponse(response) - saveRequest(r.URL.RequestURI(), "", "") + + if useDatabaseAndAPI { + saveRequest(r.URL.RequestURI(), "", "") + } return string(body) } @@ -382,7 +401,10 @@ func makeApiTtsRequest(r *http.Request) string { // write post request body to fs logBody(response.Header.Get("Content-Type"), body, "apitts_") logResponse(response) - saveRequest(r.URL.RequestURI(), "", string(body)) + + if useDatabaseAndAPI { + saveRequest(r.URL.RequestURI(), "", string(body)) + } return string(body) } @@ -419,6 +441,9 @@ func makeResRequest(r *http.Request, w http.ResponseWriter) string { } logResponse(response) - saveRequest(r.URL.RequestURI(), "", string(body)) + + if useDatabaseAndAPI { + saveRequest(r.URL.RequestURI(), "", string(body)) + } return string(body) } From 75c52ca44fa92c307cfe269cffd721ad5433eed9 Mon Sep 17 00:00:00 2001 From: Holger Wolff Date: Sat, 3 Jan 2026 03:24:06 +0100 Subject: [PATCH 3/5] Update Go version in go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b4ac792..eaaca78 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module emoProxy -go 1.24.0 +go 1.24 require modernc.org/sqlite v1.42.2 From 2ea907f24570462b312af64b1771113b04d678b9 Mon Sep 17 00:00:00 2001 From: Holger Wolff Date: Sat, 3 Jan 2026 03:33:19 +0100 Subject: [PATCH 4/5] Revise proxy server start commands in README Updated commands to start the proxy server. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 06bd8d7..5664274 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ This part of the knowledge bank is a Proxy server to analyze the traffic that goes between the EMO robot and the living.ai servers. ## Start the proxy -`go run emoProxy.go` +`go run .` ### Alternatively start with air -`air run emoProxy.go` +`air run .` ## Docker setup You can find a simplified setup using Docker Compose by SebbeJohansson here: https://github.com/SebbeJohansson/emo-proxy-docker From 4d8d502f3197513925e85bfc67bb98938a898a4a Mon Sep 17 00:00:00 2001 From: Holger Wolff Date: Sat, 3 Jan 2026 03:49:02 +0100 Subject: [PATCH 5/5] Refactor port flag parsing in main function Moved port flag parsing to the beginning of the main function. --- emoProxy.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/emoProxy.go b/emoProxy.go index 8068de2..4b65535 100644 --- a/emoProxy.go +++ b/emoProxy.go @@ -45,6 +45,7 @@ func main() { log.Println("Starting application...") //load config confFile := flag.String("c", "emoProxy.conf", "config file to use") + Port := flag.Int("port", 8080, "http port") flag.Parse() err := loadConfig(*confFile) @@ -58,8 +59,6 @@ func main() { http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // parse flags - Port := flag.Int("port", 8080, "http port") - flag.Parse() log.Println("Starting app on port: ", *Port) // redirect log