Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(warehouse): warehouse api to capture task run ID when calculating the pending uploads. #2435

Merged
merged 3 commits into from Sep 21, 2022
Merged

chore(warehouse): warehouse api to capture task run ID when calculating the pending uploads. #2435

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c1eadb9
Refactored the pending events function to include filtering on the ta…
abhimanyubabbar Sep 13, 2022
b019692
Fixed the query builder for filterby
abhimanyubabbar Sep 14, 2022
7afe7b1
PR Fixes
abhimanyubabbar Sep 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion warehouse/api.go
View file
Open in desktop
Expand Up @@ -194,7 +194,9 @@ func (uploadsReq *UploadsReqT) TriggerWhUploads() (response *proto.TriggerWhUplo
return
}
var pendingStagingFileCount int64
pendingUploadCount, err := getPendingUploadCount(uploadsReq.DestinationID, false)

filterBy := []warehouseutils.FilterBy{{Key: "destination_id", Value: uploadsReq.DestinationID}}
pendingUploadCount, err := getPendingUploadCount(filterBy...)
if err != nil {
return
}
Expand Down
5 changes: 5 additions & 0 deletions warehouse/utils/utils.go
View file
Open in desktop
Expand Up @@ -963,3 +963,8 @@ func GetDateRangeList(start, end time.Time, dateFormat string) (dateRange []stri
}
return
}

type FilterBy struct {
Key string
Value interface{}
}
56 changes: 33 additions & 23 deletions warehouse/warehouse.go
View file
Open in desktop
Expand Up @@ -1361,6 +1361,11 @@ func pendingEventsHandler(w http.ResponseWriter, r *http.Request) {
// TODO : respond with errors in a common way
pkgLogger.LogRequest(r)

if r.Method != "POST" {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of using this can we move this to the server mutex?
mux := mux.NewRouter()
mux.HandleFunc("/v1/warehouse/pending-events", pendingEventsHandler).Methods("POST")

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We would not need this right.

w.WriteHeader(http.StatusMethodNotAllowed)
return
}

// read body
body, err := io.ReadAll(r.Body)
if err != nil {
Expand Down Expand Up @@ -1398,19 +1403,21 @@ func pendingEventsHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
err := fmt.Errorf("Error getting pending staging file count : %v", err)
pkgLogger.Errorf("[WH]: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

// get pending uploads only if there are no pending staging files
if pendingStagingFileCount == 0 {
pendingUploadCount, err = getPendingUploadCount(sourceID, true)
if err != nil {
err := fmt.Errorf("Error getting pending uploads : %v", err)
pkgLogger.Errorf("[WH]: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
filterBy := []warehouseutils.FilterBy{{Key: "source_id", Value: sourceID}}
if pendingEventsReq.TaskRunID != "" {
filterBy = append(filterBy, warehouseutils.FilterBy{Key: "metadata->>'source_task_run_id'", Value: pendingEventsReq.TaskRunID})
}

pendingUploadCount, err = getPendingUploadCount(filterBy...)
if err != nil {
err := fmt.Errorf("Error getting pending uploads : %v", err)
pkgLogger.Errorf("[WH]: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

// if there are any pending staging files or uploads, set pending events as true
Expand Down Expand Up @@ -1465,7 +1472,7 @@ func pendingEventsHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
err := fmt.Errorf("Failed to marshall pending events response : %v", err)
pkgLogger.Errorf("[WH]: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

Expand Down Expand Up @@ -1506,21 +1513,24 @@ func getPendingStagingFileCount(sourceOrDestId string, isSourceId bool) (fileCou
return fileCount, nil
}

func getPendingUploadCount(sourceOrDestId string, isSourceId bool) (uploadCount int64, err error) {
sourceOrDestColumn := ""
if isSourceId {
sourceOrDestColumn = "source_id"
} else {
sourceOrDestColumn = "destination_id"
func getPendingUploadCount(filters ...warehouseutils.FilterBy) (uploadCount int64, err error) {
pkgLogger.Debugf("Fetching pending upload count with filters: %v", filters)

query := fmt.Sprintf(`
abhimanyubabbar marked this conversation as resolved.
Show resolved Hide resolved
SELECT COUNT(*)
FROM %[1]s
WHERE %[1]s.status NOT IN ('%[2]s', '%[3]s')
`, warehouseutils.WarehouseUploadsTable, ExportedData, Aborted)

args := make([]interface{}, 0)
for i, filter := range filters {
query += fmt.Sprintf(" AND %s=$%d", filter.Key, i+1)
args = append(args, filter.Value)
}
sqlStatement := fmt.Sprintf(`SELECT COUNT(*)
FROM %[1]s
WHERE %[1]s.status NOT IN ('%[2]s', '%[3]s') AND %[1]s.%[5]s='%[4]s'
`, warehouseutils.WarehouseUploadsTable, ExportedData, Aborted, sourceOrDestId, sourceOrDestColumn)

err = dbHandle.QueryRow(sqlStatement).Scan(&uploadCount)
err = dbHandle.QueryRow(query, args...).Scan(&uploadCount)
if err != nil && err != sql.ErrNoRows {
err = fmt.Errorf("Query: %s failed with Error : %w", sqlStatement, err)
err = fmt.Errorf("Query: %s failed with Error : %w", query, err)
return
}

Expand Down