-
Notifications
You must be signed in to change notification settings - Fork 324
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into spenny/support-new-error-queries
- Loading branch information
Showing
29 changed files
with
1,720 additions
and
999 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| 'highlight.run': patch | ||
| --- | ||
|
|
||
| switch replay to using highlight backend for font cors proxying |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| package assets | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| log "github.com/sirupsen/logrus" | ||
| "io" | ||
| "net/http" | ||
| "net/url" | ||
| "strings" | ||
| ) | ||
|
|
||
| const ParamKey = "url" | ||
|
|
||
| func HandleAsset(w http.ResponseWriter, r *http.Request) { | ||
| qs := r.URL.Query() | ||
| urlStr := qs.Get(ParamKey) | ||
| u, err := url.Parse(urlStr) | ||
| if err != nil { | ||
| log.WithContext(r.Context()).WithError(err).Error("failed to parse url") | ||
| http.Error(w, "failed to parse url", http.StatusBadRequest) | ||
| return | ||
| } | ||
| log.WithContext(r.Context()).WithField(ParamKey, u).Debug("CORS worker request") | ||
|
|
||
| if len(u.Host) == 0 { | ||
| for n, h := range r.Header { | ||
| // get the origin from the request | ||
| if strings.Contains(n, "Origin") { | ||
| for _, h := range h { | ||
| u.Host = h | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // always allow access origin | ||
| w.Header().Add("Access-Control-Allow-Origin", u.Host) | ||
| w.Header().Add("Access-Control-Allow-Credentials", "true") | ||
| w.Header().Add("Access-Control-Allow-Methods", "GET") | ||
|
|
||
| // create the request to server | ||
| req, err := http.NewRequest(r.Method, u.String(), r.Body) | ||
| if err != nil { | ||
| log.WithContext(r.Context()).WithError(err).Error("failed to build request") | ||
| http.Error(w, "failed to build request", http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // add ALL headers to the connection | ||
| for n, h := range r.Header { | ||
| for _, h := range h { | ||
| req.Header.Add(n, h) | ||
| } | ||
| } | ||
|
|
||
| // use the host provided by the flag | ||
| if len(u.Host) > 0 { | ||
| req.Host = u.Host | ||
| } | ||
|
|
||
| // create a basic client to send the request | ||
| client := http.Client{} | ||
| if r.TLS != nil { | ||
| client.Transport = &http.Transport{ | ||
| TLSClientConfig: &tls.Config{}, | ||
| } | ||
| } | ||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| log.Println(err) | ||
| w.WriteHeader(http.StatusInternalServerError) | ||
| _, _ = w.Write([]byte(err.Error())) | ||
| return | ||
| } | ||
|
|
||
| for h, v := range resp.Header { | ||
| for _, v := range v { | ||
| w.Header().Add(h, v) | ||
| } | ||
| } | ||
| // copy the response from the server to the connected client request | ||
| w.WriteHeader(resp.StatusCode) | ||
|
|
||
| wr, err := io.Copy(w, resp.Body) | ||
| if err != nil { | ||
| log.WithContext(r.Context()).WithField("written", wr).WithError(err).Error("failed to write back body") | ||
| } else { | ||
| log.WithContext(r.Context()).WithField("written", wr).Debug("wrote back body") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package assets | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/stretchr/testify/assert" | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestCreateLogDrain(t *testing.T) { | ||
| server := httptest.NewServer(http.HandlerFunc(HandleAsset)) | ||
| defer server.Close() | ||
|
|
||
| resp, err := http.Get(fmt.Sprintf("%s?src=test&url=https://app.highlight.io/~r_app.webmanifest?~r_rid=knQ44NI79K8s_IOq_MbHMZ0JjqY", server.URL)) | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, resp.StatusCode, 200) | ||
| assert.Greater(t, resp.ContentLength, int64(256)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
100 changes: 100 additions & 0 deletions
100
backend/migrations/cmd/migrate-group-embeddings/main.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| log "github.com/sirupsen/logrus" | ||
| "gorm.io/gorm" | ||
|
|
||
| "github.com/highlight-run/highlight/backend/model" | ||
| ) | ||
|
|
||
| func main() { | ||
| ctx := context.Background() | ||
|
|
||
| db, err := model.SetupDB(ctx, os.Getenv("PSQL_DB")) | ||
| if err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
|
|
||
| if err := db.Exec(` | ||
| CREATE TABLE IF NOT EXISTS migrated_embeddings ( | ||
| project_id INTEGER PRIMARY KEY NOT NULL, | ||
| embedding_id INTEGER NOT NULL | ||
| )`).Error; err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
|
|
||
| var lastCreatedPart int | ||
| if err := db.Raw("select split_part(relname, '_', 5) from pg_stat_all_tables where relname like 'error_object_embeddings_partitioned%' order by relid desc limit 1"). | ||
| Scan(&lastCreatedPart).Error; err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
|
|
||
| // Only running this migration on project_id = 1 for now | ||
| for i := 1; i <= 1; i++ { | ||
| log.WithContext(ctx).Infof("beginning loop: %d", i) | ||
| tablename := fmt.Sprintf("error_object_embeddings_partitioned_%d", i) | ||
|
|
||
| if err := db.Exec(fmt.Sprintf("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_%s_id ON %s (id)", tablename, tablename)).Error; err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
| log.WithContext(ctx).Info("done creating index") | ||
|
|
||
| var prevEmbeddingId int | ||
| if err := db.Raw("select coalesce(max(embedding_id), 0) from migrated_embeddings where project_id = ?", i).Scan(&prevEmbeddingId).Error; err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
| log.WithContext(ctx).Infof("prevEmbeddingId: %d", prevEmbeddingId) | ||
|
|
||
| var maxEmbeddingId int | ||
| if err := db.Raw("select coalesce(max(id), 0) from error_object_embeddings_partitioned eoe where project_id = ?", i).Scan(&maxEmbeddingId).Error; err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
| log.WithContext(ctx).Infof("maxEmbeddingId: %d", maxEmbeddingId) | ||
|
|
||
| if err := db.Transaction(func(tx *gorm.DB) error { | ||
| if err := tx.Exec(` | ||
| insert into error_group_embeddings (project_id, error_group_id, count, gte_large_embedding) | ||
| select a.* from ( | ||
| select eo.project_id, eo.error_group_id, count(*) as count, AVG(eoe.gte_large_embedding) as gte_large_embedding | ||
| from error_object_embeddings_partitioned eoe | ||
| inner join error_objects eo | ||
| on eoe.error_object_id = eo.id | ||
| where eoe.gte_large_embedding is not null | ||
| and eoe.id > ? | ||
| and eoe.id <= ? | ||
| group by eo.project_id, eo.error_group_id) a | ||
| on conflict (project_id, error_group_id) | ||
| do update set | ||
| gte_large_embedding = | ||
| error_group_embeddings.gte_large_embedding * array_fill(error_group_embeddings.count::numeric / (error_group_embeddings.count + excluded.count), '{1024}')::vector | ||
| + excluded.gte_large_embedding * array_fill(excluded.count::numeric / (error_group_embeddings.count + excluded.count), '{1024}')::vector, | ||
| count = error_group_embeddings.count + excluded.count | ||
| `, prevEmbeddingId, maxEmbeddingId).Error; err != nil { | ||
| return err | ||
| } | ||
|
|
||
| log.WithContext(ctx).Info("done upserting new embeddings") | ||
|
|
||
| if err := tx.Exec(` | ||
| insert into migrated_embeddings (project_id, embedding_id) | ||
| values (?, ?) | ||
| on conflict (project_id) | ||
| do update set embedding_id = excluded.embedding_id | ||
| `, i, maxEmbeddingId).Error; err != nil { | ||
| return err | ||
| } | ||
|
|
||
| log.WithContext(ctx).Info("done updating maxEmbeddingId") | ||
|
|
||
| return nil | ||
| }); err != nil { | ||
| log.WithContext(ctx).Fatal(err) | ||
| } | ||
| log.WithContext(ctx).Infof("done loop: %d", i) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.