Skip to content

Commit

Permalink
Merge pull request #345 from kercre123/ui-api-overhaul
Browse files Browse the repository at this point in the history
UI changes
  • Loading branch information
kercre123 committed Jun 23, 2024
2 parents 8e80c70 + 6a74fb9 commit 739ded7
Show file tree
Hide file tree
Showing 28 changed files with 1,639 additions and 1,397 deletions.
5 changes: 1 addition & 4 deletions chipper/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/kercre123/wire-pod/chipper
go 1.18

require (
github.com/Picovoice/leopard/binding/go v1.2.0
github.com/Picovoice/leopard/binding/go/v2 v2.0.2
github.com/asticode/go-asticoqui v0.2.0
github.com/bramvdbogaerde/go-scp v1.2.1
github.com/digital-dream-labs/api v0.0.0-20210824232136-8cc90c1bb12c
Expand Down Expand Up @@ -32,12 +32,10 @@ require (
)

require (
github.com/Picovoice/leopard/binding/go/v2 v2.0.2 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/akavel/rsrc v0.10.2 // indirect
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
github.com/alphacep/vosk-api/go v0.3.50 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/currantlabs/ble v0.0.0-20171229162446-c1d21c164cf8 // indirect
github.com/dchest/jsmin v0.0.0-20220218165748-59f39799265f // indirect
Expand All @@ -50,7 +48,6 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jamesruan/sodium v0.0.0-20181216154042-9620b83ffeae // indirect
github.com/josephspurrier/goversioninfo v1.4.0 // indirect
github.com/kercre123/vosk-api v1.0.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
Expand Down
8 changes: 0 additions & 8 deletions chipper/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tT
github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Picovoice/leopard/binding/go v1.2.0 h1:NbUW+Fni5UydvcFlMx8RZtk2pccFaRJFG14kGaUa4CA=
github.com/Picovoice/leopard/binding/go v1.2.0/go.mod h1:5kaEg9ZcH2dLkrX/H1xMVF6QFM7l3vd9GKxeXSanA8s=
github.com/Picovoice/leopard/binding/go/v2 v2.0.2 h1:Knk/UV51oRuHTHd7MGtlZXwsFF5jxu6AqttB0jGMHxs=
github.com/Picovoice/leopard/binding/go/v2 v2.0.2/go.mod h1:/rYUeRDH4xBgtwBe9D8BwHIauPJ+M7czqLfyeJQJu7c=
github.com/aalpern/go-metrics v0.0.0-20181116155206-644932c99203/go.mod h1:wHUOZ2LlAirciaWYGZM3apvZftM7aRXhLMDsdjqEFB4=
Expand All @@ -55,8 +53,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alphacep/vosk-api/go v0.3.50 h1:2vSN41RCU1WdHEqBrhKtTggfKL6Yu5Dmj+urVszwiuw=
github.com/alphacep/vosk-api/go v0.3.50/go.mod h1:9X8IJsHnFk/b1xyvjlZifo+ZL5VTAx3LW+JQce/eRcA=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
Expand Down Expand Up @@ -308,8 +304,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kercre123/vosk-api v1.0.1 h1:D5CeAMNHPj87M9fKrqP+a2gEQefq7sJCpaiuRscbiJY=
github.com/kercre123/vosk-api v1.0.1/go.mod h1:mJlLhtYS207jVY9QffYGxhX6Up0UfSQ3p0uNbXsf3Zc=
github.com/kercre123/vosk-api/go v1.0.2 h1:NDJUNv2ddw128amiVZ2xE2gKfKHeBRRhboSh+yiH6Wg=
github.com/kercre123/vosk-api/go v1.0.2/go.mod h1:oVZG/VFmg23uNDzjShcw7UhZHWYG2zXgBm5FqioE2Ao=
github.com/kercre123/zeroconf v1.0.1 h1:Mbd8mN6xnNWYIqBN38x3jJjiPP2RmK4orzbGZsa1EOY=
Expand Down Expand Up @@ -430,8 +424,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sashabaranov/go-openai v1.20.4 h1:095xQ/fAtRa0+Rj21sezVJABgKfGPNbyx/sAN/hJUmg=
github.com/sashabaranov/go-openai v1.20.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/sashabaranov/go-openai v1.24.0 h1:4H4Pg8Bl2RH/YSnU8DYumZbuHnnkfioor/dtNlB20D4=
github.com/sashabaranov/go-openai v1.24.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/schollz/progressbar/v3 v3.7.4/go.mod h1:1H8m5kMPW6q5fyjpDqtBHW1JT22mu2NwHQ1ApuCPh/8=
Expand Down
4 changes: 2 additions & 2 deletions chipper/pkg/logger/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func Println(a ...any) {

func LogUI(a ...any) {
LogArray = append(LogArray, time.Now().Format("2006.01.02 15:04:05")+": "+fmt.Sprint(a...)+"\n")
if len(LogArray) >= 30 {
if len(LogArray) >= 50 {
LogArray = LogArray[1:]
}
LogList = ""
Expand All @@ -47,7 +47,7 @@ func LogUI(a ...any) {

func LogTray(a ...any) {
LogTrayArray = append(LogTrayArray, time.Now().Format("2006.01.02 15:04:05")+": "+fmt.Sprint(a...)+"\n")
if len(LogTrayArray) >= 30 {
if len(LogTrayArray) >= 200 {
LogTrayArray = LogTrayArray[1:]
}
LogTrayList = ""
Expand Down
1 change: 1 addition & 0 deletions chipper/pkg/vars/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type apiConfig struct {
OpenAIPrompt string `json:"openai_prompt"`
SaveChat bool `json:"save_chat"`
CommandsEnable bool `json:"commands_enable"`
Endpoint string `json:"endpoint"`
} `json:"knowledge"`
STT struct {
Service string `json:"provider"`
Expand Down
70 changes: 42 additions & 28 deletions chipper/pkg/wirepod/config-ws/webserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,46 +189,57 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
// for Together AI Service
kgModel := r.FormValue("model")

kgEndpoint := r.FormValue("endpoint")

if kgProvider == "" {
vars.APIConfig.Knowledge.Enable = false
vars.APIConfig.Knowledge.OpenAIPrompt = ""
vars.APIConfig.Knowledge.ID = ""
vars.APIConfig.Knowledge.Key = ""
vars.APIConfig.Knowledge.Provider = ""
vars.APIConfig.Knowledge.RobotName = ""
vars.APIConfig.Knowledge.IntentGraph = false
} else {
vars.APIConfig.Knowledge.Enable = true
vars.APIConfig.Knowledge.Provider = kgProvider
vars.APIConfig.Knowledge.Key = strings.TrimSpace(kgAPIKey)
vars.APIConfig.Knowledge.Model = strings.TrimSpace(kgModel)
vars.APIConfig.Knowledge.ID = strings.TrimSpace(kgAPIID)
}
if kgModel == "" && kgProvider == "together" {
logger.Println("Together model wasn't provided, using default meta-llama/Llama-3-70b-chat-hf")
vars.APIConfig.Knowledge.Model = "meta-llama/Llama-3-70b-chat-hf"
}
if kgProvider == "openai" || kgProvider == "together" {
if strings.TrimSpace(r.FormValue("openai_prompt")) != "" {
vars.APIConfig.Knowledge.OpenAIPrompt = r.FormValue("openai_prompt")
} else {
vars.APIConfig.Knowledge.OpenAIPrompt = ""
if kgModel == "" && kgProvider == "together" {
logger.Println("Together model wasn't provided, using default meta-llama/Llama-3-70b-chat-hf")
vars.APIConfig.Knowledge.Model = "meta-llama/Llama-3-70b-chat-hf"
}
if r.FormValue("save_chat") == "true" {
vars.APIConfig.Knowledge.SaveChat = true
} else {
vars.APIConfig.Knowledge.SaveChat = false
if kgProvider == "openai" || kgProvider == "together" || kgProvider == "custom" {
if strings.TrimSpace(r.FormValue("openai_prompt")) != "" {
vars.APIConfig.Knowledge.OpenAIPrompt = r.FormValue("openai_prompt")
} else {
vars.APIConfig.Knowledge.OpenAIPrompt = ""
}
if r.FormValue("save_chat") == "true" {
vars.APIConfig.Knowledge.SaveChat = true
} else {
vars.APIConfig.Knowledge.SaveChat = false
}
if r.FormValue("commands_enable") == "true" {
vars.APIConfig.Knowledge.CommandsEnable = true
} else {
vars.APIConfig.Knowledge.CommandsEnable = false
}
}
if r.FormValue("commands_enable") == "true" {
vars.APIConfig.Knowledge.CommandsEnable = true
} else {
vars.APIConfig.Knowledge.CommandsEnable = false
if (kgProvider == "openai" || kgProvider == "together" || kgProvider == "custom") && kgIntent == "true" {
vars.APIConfig.Knowledge.IntentGraph = true
if r.FormValue("robot_name") == "" {
vars.APIConfig.Knowledge.RobotName = "Vector"
} else {
vars.APIConfig.Knowledge.RobotName = strings.TrimSpace(r.FormValue("robot_name"))
}
} else if (kgProvider == "openai" || kgProvider == "together" || kgProvider == "custom") && kgIntent == "false" {
vars.APIConfig.Knowledge.IntentGraph = false
vars.APIConfig.Knowledge.RobotName = ""
}
}
if (kgProvider == "openai" || kgProvider == "together") && kgIntent == "true" {
vars.APIConfig.Knowledge.IntentGraph = true
if r.FormValue("robot_name") == "" {
vars.APIConfig.Knowledge.RobotName = "Vector"
} else {
vars.APIConfig.Knowledge.RobotName = strings.TrimSpace(r.FormValue("robot_name"))
if kgProvider == "custom" {
vars.APIConfig.Knowledge.Endpoint = kgEndpoint
}
} else if (kgProvider == "openai" || kgProvider == "together") && kgIntent == "false" {
vars.APIConfig.Knowledge.IntentGraph = false
vars.APIConfig.Knowledge.RobotName = ""
}
vars.WriteConfigToDisk()
fmt.Fprintf(w, "Changes successfully applied.")
Expand All @@ -244,6 +255,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
kgOpenAIPrompt := ""
kgSavePrompt := false
kgCommandsEnable := false
kgEndpoint := ""
if vars.APIConfig.Knowledge.Enable {
kgEnabled = true
kgProvider = vars.APIConfig.Knowledge.Provider
Expand All @@ -255,6 +267,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
kgOpenAIPrompt = vars.APIConfig.Knowledge.OpenAIPrompt
kgSavePrompt = vars.APIConfig.Knowledge.SaveChat
kgCommandsEnable = vars.APIConfig.Knowledge.CommandsEnable
kgEndpoint = vars.APIConfig.Knowledge.Endpoint
}
fmt.Fprintf(w, "{ ")
fmt.Fprintf(w, " \"kgEnabled\": %t,", kgEnabled)
Expand All @@ -266,6 +279,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, " \"kgRobotName\": \"%s\",", kgRobotName)
fmt.Fprintf(w, " \"kgOpenAIPrompt\": \"%s\",", kgOpenAIPrompt)
fmt.Fprintf(w, " \"kgSaveChat\": \"%t\",", kgSavePrompt)
fmt.Fprintf(w, " \"kgEndpoint\": \"%s\",", kgEndpoint)
fmt.Fprintf(w, " \"kgCommandsEnable\": \"%t\"", kgCommandsEnable)
fmt.Fprintf(w, "}")
return
Expand Down
2 changes: 1 addition & 1 deletion chipper/pkg/wirepod/preqs/intent.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func (s *Server) ProcessIntent(req *vtt.IntentRequest) (*vtt.IntentResponse, err
return nil, nil
}
if !successMatched {
if vars.APIConfig.Knowledge.IntentGraph {
if vars.APIConfig.Knowledge.IntentGraph && vars.APIConfig.Knowledge.Enable {
logger.Println("Making LLM request for device " + req.Device + "...")
_, err := ttr.StreamingKGSim(req, req.Device, transcribedText)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion chipper/pkg/wirepod/preqs/intent_graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func (s *Server) ProcessIntentGraph(req *vtt.IntentGraphRequest) (*vtt.IntentGra
// return nil, nil
// }
if !successMatched {
if vars.APIConfig.Knowledge.IntentGraph {
if vars.APIConfig.Knowledge.IntentGraph && vars.APIConfig.Knowledge.Enable {
logger.Println("Making LLM request for device " + req.Device + "...")
_, err := ttr.StreamingKGSim(req, req.Device, transcribedText)
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions chipper/pkg/wirepod/preqs/knowledgegraph.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,12 @@ func houndifyKG(req sr.SpeechRequest) string {
}

func togetherRequest(transcribedText string) string {
// will also handle custom
sendString := "You are a helpful robot called Vector. You will be given a question asked by a user and you must provide the best answer you can. It may not be punctuated or spelled correctly. Keep the answer concise yet informative. Here is the question: " + "\\" + "\"" + transcribedText + "\\" + "\"" + " , Answer: "
url := "https://api.together.xyz/inference"
if vars.APIConfig.Knowledge.Provider == "custom" {
url = vars.APIConfig.Knowledge.Endpoint
}
model := vars.APIConfig.Knowledge.Model
formData := `{
"model": "` + model + `",
Expand Down
108 changes: 108 additions & 0 deletions chipper/pkg/wirepod/ttr/convert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package wirepod_ttr

// TODO
// import (
// "bytes"
// "encoding/binary"
// "math"

// "github.com/zaf/resample"
// )

// func bytesToInt16s(data []byte) []int16 {
// int16s := make([]int16, len(data)/2)
// for i := range int16s {
// int16s[i] = int16(binary.LittleEndian.Uint16(data[i*2 : i*2+2]))
// }
// return int16s
// }

// func int16sToBytes(data []int16) []byte {
// bytes := make([]byte, len(data)*2)
// for i, val := range data {
// binary.LittleEndian.PutUint16(bytes[i*2:], uint16(val))
// }
// return bytes
// }

// // entry-point (go, not defaulgt)
// func downsample24kTo16k(input []byte) [][]byte {
// iVolBytes := increaseVolume(input, 6)
// outBytes := downsample24kTo16kLinear(iVolBytes)
// var audioChunks [][]byte
// // the "s" sounds are harsh. put through filter
// //filteredBytes := lowPassFilter(outBytes, 4000, 16000)
// for len(outBytes) >= 1024 {
// audioChunks = append(audioChunks, outBytes[:1024])
// outBytes = outBytes[1024:]
// }

// return audioChunks
// }

// func increaseVolume(data []byte, factor float64) []byte {
// int16s := bytesToInt16s(data)

// for i := range int16s {
// scaled := float64(int16s[i]) * factor
// if scaled > math.MaxInt16 {
// int16s[i] = math.MaxInt16
// } else if scaled < math.MinInt16 {
// int16s[i] = math.MinInt16
// } else {
// int16s[i] = int16(scaled)
// }
// }

// return int16sToBytes(int16s)
// }

// // this is copied
// func lowPassFilter(data []byte, cutoffFreq float64, sampleRate int) []byte {
// int16s := bytesToInt16s(data)
// filtered := make([]int16, len(int16s))
// rc := 1.0 / (2 * 3.1416 * cutoffFreq)
// dt := 1.0 / float64(sampleRate)
// alpha := dt / (rc + dt)
// filtered[0] = int16s[0]
// for i := 1; i < len(int16s); i++ {
// current := alpha*float64(int16s[i]) + (1-alpha)*float64(filtered[i-1])
// filtered[i] = int16(current)
// }

// return int16sToBytes(filtered)
// }

// // technically copied too
// func downsample24kTo16kLinear(input []byte) []byte {
// int16s := bytesToInt16s(input)
// outputLength := (len(int16s) * 2) / 3
// output := make([]int16, outputLength)

// j := 0
// for i := 0; i < len(int16s)-2; i += 3 {
// first := (2*int32(int16s[i]) + int32(int16s[i+1])) / 3
// second := (int32(int16s[i+1]) + 2*int32(int16s[i+2])) / 3
// output[j] = int16(first)
// output[j+1] = int16(second)
// j += 2
// }

// return int16sToBytes(output)
// }

// // entry-point (soxr)
// func downsampleAudioSoxr(input []byte) [][]byte {
// newBytes := new(bytes.Buffer)
// dec, _ := resample.New(newBytes, 24000, 16000, 1, resample.I16, resample.VeryHighQ)
// dec.Write(input)
// var audioChunks [][]byte
// decodedBytes := newBytes.Bytes()
// filteredBytes := lowPassFilter(decodedBytes, 4000, 16000)
// iVolBytes := increaseVolume(filteredBytes, 5)
// for len(iVolBytes) >= 1024 {
// audioChunks = append(audioChunks, iVolBytes[:1024])
// iVolBytes = iVolBytes[1024:]
// }
// return audioChunks
// }
2 changes: 1 addition & 1 deletion chipper/pkg/wirepod/ttr/intentparam.go
Original file line number Diff line number Diff line change
Expand Up @@ -515,7 +515,7 @@ func ParamCheckerSlotsEnUS(req interface{}, intent string, slots map[string]stri
IntentPass(req, newIntent, intent, intentParams, isParam)
}

func prehistoricParamChecker(req interface{}, intent string, speechText string, botSerial string) {
func prehistoricParamChecker(req interface{}, intent string, speechText string) {
// intent.go detects if the stream uses opus or PCM.
// If the stream is PCM, it is likely a bot with 0.10.
// This accounts for the newer 0.10.1### builds.
Expand Down
Loading

0 comments on commit 739ded7

Please sign in to comment.