Skip to content

Commit e0e6ccf

Browse files
thatportugueseguythatportugueseguy
committed
bk webhook: fetch commit author from GH api
1 parent 0dc4fa4 commit e0e6ccf

File tree

7 files changed

+98
-48
lines changed

7 files changed

+98
-48
lines changed

‎lib/action.ml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
616616
full_name = "";
617617
url = repo_url;
618618
commits_url = "";
619-
contents_url = Util.Build.git_ssh_to_contents_url n.pipeline.repository;
619+
contents_url = Util.Webhook.git_ssh_to_contents_url n.pipeline.repository;
620620
pulls_url = "";
621621
issues_url = "";
622622
compare_url = "";
@@ -674,7 +674,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
674674
{ steps = Common.FailedStepSet.empty; last_build = n.build.number };
675675
Lwt.return []
676676
| Ok (build : Buildkite_t.get_build_res) ->
677-
let build_steps = Util.Webhook.to_failed_step_set (Util.Build.filter_passed_jobs build.jobs) n in
677+
let build_steps = Util.Webhook.to_failed_step_set "" (Util.Build.filter_passed_jobs build.jobs) n in
678678
let state_failed_steps = FailedStepSet.diff state.steps build_steps in
679679
(match FailedStepSet.is_empty state_failed_steps with
680680
| true ->
@@ -705,6 +705,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
705705
(* repo state is updated upon fetching new failed steps *)
706706
new_failed_steps ~cfg ~repo_state
707707
~get_build:(Buildkite_api.get_build ~cache:`Refresh ~ctx)
708+
~get_commit:(Github_api.get_api_commit_webhook ~ctx)
708709
~db_update:(fun ~repo_state ~has_state_update n msg ->
709710
let%lwt () =
710711
Database.Failed_builds.update_state_after_notification ~repo_state ~has_state_update n

‎lib/api.ml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ open Slack_t
55
module type Github = sig
66
val get_config : ctx:Context.t -> repo:repository -> (Config_t.config, string) Result.t Lwt.t
77
val get_api_commit : ctx:Context.t -> repo:repository -> sha:string -> (api_commit, string) Result.t Lwt.t
8+
val get_api_commit_webhook :
9+
ctx:Context.t -> commits_url:string -> repo_url:string -> sha:string -> (api_commit, string) Result.t Lwt.t
810
val get_pull_request : ctx:Context.t -> repo:repository -> number:int -> (pull_request, string) Result.t Lwt.t
911
val get_issue : ctx:Context.t -> repo:repository -> number:int -> (issue, string) Result.t Lwt.t
1012
val get_compare : ctx:Context.t -> repo:repository -> basehead:Github.basehead -> (compare, string) Result.t Lwt.t

‎lib/api_local.ml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,22 @@ NB: please save the cache file in the same format *)
3737
let get_api_commit ~ctx:_ ~repo ~sha =
3838
get_repo_member_cache ~repo ~kind:"commit" ~ref_:sha ~of_string:Github_j.api_commit_of_string
3939

40+
let get_api_commit_webhook ~ctx:_ ~commits_url ~repo_url ~sha =
41+
(* creating a fake repository record to avoid making deep changes to get_repo_member_cache *)
42+
let repo : Github_t.repository =
43+
{
44+
name = "";
45+
full_name = "";
46+
url = repo_url;
47+
commits_url;
48+
contents_url = "";
49+
pulls_url = "";
50+
issues_url = "";
51+
compare_url = "";
52+
}
53+
in
54+
get_repo_member_cache ~repo ~kind:"commit" ~ref_:sha ~of_string:Github_j.api_commit_of_string
55+
4056
let get_pull_request ~ctx:_ ~(repo : Github_t.repository) ~number =
4157
get_repo_member_cache ~repo ~kind:"pull" ~ref_:(Int.to_string number) ~of_string:Github_j.pull_request_of_string
4258

‎lib/api_remote.ml‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ module Github : Api.Github = struct
2828
let headers = [ "Accept: application/vnd.github.v3+json" ] in
2929
Option.map_default (fun v -> sprintf "Authorization: token %s" v :: headers) headers token
3030

31-
let prepare_request ~secrets ~(repo : Github_t.repository) url =
32-
let token = Context.gh_token_of_secrets secrets repo.url in
31+
let prepare_request ~secrets ~repo_url url =
32+
let token = Context.gh_token_of_secrets secrets repo_url in
3333
let headers = build_headers ?token () in
3434
let url =
35-
match Context.gh_repo_of_secrets secrets repo.url with
35+
match Context.gh_repo_of_secrets secrets repo_url with
3636
| None -> url
3737
| Some repo_config ->
3838
(* The url might have been built based on information received through an untrusted source such as a slack message.
@@ -42,21 +42,21 @@ module Github : Api.Github = struct
4242
in
4343
headers, url
4444

45-
let get_resource ~secrets ~repo url =
46-
let headers, url = prepare_request ~secrets ~repo url in
45+
let get_resource ~secrets ~repo_url url =
46+
let headers, url = prepare_request ~secrets ~repo_url url in
4747
http_request ~headers `GET url
4848
|> Lwt_result.map_error (fun e -> sprintf "error while querying remote: %s\nfailed to get resource from %s" e url)
4949

50-
let post_resource ~secrets ~repo body url =
51-
let headers, url = prepare_request ~secrets ~repo url in
50+
let post_resource ~secrets ~repo_url body url =
51+
let headers, url = prepare_request ~secrets ~repo_url url in
5252
http_request ~headers ~body:(`Raw ("application/json; charset=utf-8", body)) `POST url
5353
|> Lwt_result.map_error (sprintf "POST to %s failed : %s" url)
5454

5555
let get_config ~(ctx : Context.t) ~repo =
5656
let secrets = Context.get_secrets_exn ctx in
5757
let url = contents_url ~repo ~path:ctx.config_filename in
5858
let* res =
59-
get_resource ~secrets ~repo url
59+
get_resource ~secrets ~repo_url:repo.url url
6060
|> Lwt_result.map_error (fun e ->
6161
sprintf "error while querying remote: %s\nfailed to get config from file %s" e url)
6262
in
@@ -77,26 +77,33 @@ module Github : Api.Github = struct
7777
@@ fmt_error "unexpected encoding '%s' in Github response\nfailed to get config from file %s" encoding url
7878

7979
let get_api_commit ~(ctx : Context.t) ~(repo : Github_t.repository) ~sha =
80-
let%lwt res = commits_url ~repo ~sha |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo in
80+
let%lwt res = commits_url ~repo ~sha |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo_url:repo.url in
81+
Lwt.return @@ Result.map Github_j.api_commit_of_string res
82+
83+
let get_api_commit_webhook ~(ctx : Context.t) ~commits_url ~repo_url ~sha =
84+
let _, commits_url = ExtLib.String.replace ~sub:"{/sha}" ~by:("/" ^ sha) ~str:commits_url in
85+
let%lwt res = get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo_url commits_url in
8186
Lwt.return @@ Result.map Github_j.api_commit_of_string res
8287

8388
let get_pull_request ~(ctx : Context.t) ~(repo : Github_t.repository) ~number =
84-
let%lwt res = pulls_url ~repo ~number |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo in
89+
let%lwt res = pulls_url ~repo ~number |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo_url:repo.url in
8590
Lwt.return @@ Result.map Github_j.pull_request_of_string res
8691

8792
let get_issue ~(ctx : Context.t) ~(repo : Github_t.repository) ~number =
88-
let%lwt res = issues_url ~repo ~number |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo in
93+
let%lwt res = issues_url ~repo ~number |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo_url:repo.url in
8994
Lwt.return @@ Result.map Github_j.issue_of_string res
9095

9196
let get_compare ~(ctx : Context.t) ~(repo : Github_t.repository) ~basehead =
92-
let%lwt res = compare_url ~repo ~basehead |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo in
97+
let%lwt res =
98+
compare_url ~repo ~basehead |> get_resource ~secrets:(Context.get_secrets_exn ctx) ~repo_url:repo.url
99+
in
93100
Lwt.return @@ Result.map Github_j.compare_of_string res
94101

95102
let request_reviewers ~(ctx : Context.t) ~(repo : Github_t.repository) ~number ~reviewers =
96103
let body = Github_j.string_of_request_reviewers_req reviewers in
97104
let%lwt res =
98105
pulls_url ~repo ~number ^ "/requested_reviewers"
99-
|> post_resource ~secrets:(Context.get_secrets_exn ctx) ~repo body
106+
|> post_resource ~secrets:(Context.get_secrets_exn ctx) ~repo_url:repo.url body
100107
in
101108
Lwt.return @@ Result.map ignore res
102109
end

‎lib/database.ml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ module Failed_builds = struct
136136
handle_create
137137
@@ with_db op_name (fun dbd ->
138138
let org, pipeline_name, _build_nr = Util.Build.get_org_pipeline_build' web_url in
139-
let repo_url = Util.Build.git_ssh_to_https pipeline.repository in
139+
let repo_url = Util.Webhook.git_ssh_to_https pipeline.repository in
140140
let repo_state = State.find_or_add_repo ctx.state repo_url in
141141
let%lwt (build : Buildkite_t.get_build_res) =
142142
(* TODO: review. Can we `Use cache here? *)

‎lib/slack.ml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@ let generate_status_notification ~(job_log : (string * string) list) ~(cfg : Con
458458
let generate_failed_build_notification ?(slack_ids = []) ?(is_fix_build_notification = false) ~(cfg : Config_t.config)
459459
~failed_steps (n : Util.Webhook.n) channel =
460460
let pipeline_name = Util.Webhook.pipeline_name n in
461-
let repo_url = Util.Build.git_ssh_to_https n.pipeline.repository in
461+
let repo_url = Util.Webhook.git_ssh_to_https n.pipeline.repository in
462462
let commit_url = sprintf "%s/commit/%s" repo_url n.build.sha in
463463
let summary =
464464
(* check if this is an escalation notification. if it's mixed, paciencia *)

‎lib/util.ml‎

Lines changed: 55 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -67,26 +67,6 @@ module Build = struct
6767
(* Gets the pipeline name from the buildkite context *)
6868
Re2.create_exn {|buildkite/([\w_-]+)|}
6969

70-
let git_ssh_re =
71-
(* matches git ssh clone links *)
72-
Re2.create_exn {|^git@([A-Za-z0-9.-]+):([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)\.git$|}
73-
74-
let git_ssh_to_https url =
75-
match Re2.find_submatches_exn git_ssh_re url with
76-
| exception exn -> util_error ~exn "failed to parse git ssh link %s" url
77-
| [| Some _; Some url; Some user; Some repo |] -> sprintf "https://%s/%s/%s" url user repo
78-
| _ -> util_error "failed to get repo details from the ssh link."
79-
80-
let git_ssh_to_contents_url url =
81-
match Re2.find_submatches_exn git_ssh_re url with
82-
| exception exn -> util_error ~exn "failed to parse git ssh link %s" url
83-
| [| Some _; Some "github.com"; Some user; Some repo |] ->
84-
sprintf "https://api.github.com/repos/%s/%s/contents/{+path}" user repo
85-
| [| Some _; Some url; Some user; Some repo |] ->
86-
(* GHE links *)
87-
sprintf "https://%s/api/v3/repos/%s/%s/contents/{+path}" url user repo
88-
| _ -> util_error "failed to get repo details from the ssh link."
89-
9070
let is_pipeline_step context = Re2.matches buildkite_is_step_re context
9171

9272
(** For now we only care about buildkite pipelines and steps. Other CI systems are not supported yet. *)
@@ -258,6 +238,47 @@ end
258238
module Webhook = struct
259239
type n = Buildkite_t.webhook_build_payload
260240

241+
type repo_details = {
242+
url : string;
243+
user : string;
244+
repo : string;
245+
}
246+
247+
type repo =
248+
| Github of repo_details
249+
| GHE of repo_details
250+
251+
type failed_passed_steps = {
252+
failed_steps : FailedStepSet.t;
253+
passed_steps : FailedStepSet.t;
254+
}
255+
256+
let git_ssh_re =
257+
(* matches git ssh clone links *)
258+
Re2.create_exn {|^git@([A-Za-z0-9.-]+):([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)\.git$|}
259+
260+
let git_ssh_to_repo url =
261+
match Re2.find_submatches_exn git_ssh_re url with
262+
| exception exn -> util_error ~exn "failed to parse git ssh link %s" url
263+
| [| Some _; Some "github.com"; Some user; Some repo |] -> Github { url = "github.com"; user; repo }
264+
| [| Some _; Some url; Some user; Some repo |] ->
265+
(* GHE links *)
266+
GHE { url; user; repo }
267+
| _ -> util_error "failed to get repo details from the ssh link."
268+
269+
let git_ssh_to_api_url ?(resource = "") url =
270+
match git_ssh_to_repo url with
271+
| Github { user; repo; _ } -> sprintf "https://api.github.com/repos/%s/%s%s" user repo resource
272+
| GHE { url; user; repo } -> sprintf "https://%s/api/v3/repos/%s/%s%s" url user repo resource
273+
274+
let git_ssh_to_https url =
275+
match git_ssh_to_repo url with
276+
| Github { url; user; repo } | GHE { url; user; repo } -> sprintf "https://%s/%s/%s" url user repo
277+
278+
let git_ssh_to_contents_url ssh_url = git_ssh_to_api_url ~resource:"/contents/{+path}" ssh_url
279+
280+
let git_ssh_to_commits_url ssh_url = git_ssh_to_api_url ~resource:"/commits{/sha}" ssh_url
281+
261282
let parse_signature_header header =
262283
let timestamp, signature =
263284
String.split_on_char ',' header
@@ -292,7 +313,7 @@ module Webhook = struct
292313
| Ok (timestamp, signature) -> is_valid_signature ~secret ~timestamp ~signature body
293314

294315
let validate_repo_url (secrets : Config_t.secrets) (n : Buildkite_t.webhook_build_payload) =
295-
let repo_url = Build.git_ssh_to_https n.pipeline.repository in
316+
let repo_url = git_ssh_to_https n.pipeline.repository in
296317
match List.exists (fun (r : Config_t.repo_config) -> String.equal r.url repo_url) secrets.repos with
297318
| true -> repo_url
298319
| false -> util_error "unsupported repository %s" repo_url
@@ -382,33 +403,36 @@ module Webhook = struct
382403
let time_since t = Ptime.(Span.abs (diff (Ptime_clock.now ()) t))
383404
let is_past_span time span = Ptime.Span.compare (time_since time) span > 0
384405

385-
type failed_passed_steps = {
386-
failed_steps : FailedStepSet.t;
387-
passed_steps : FailedStepSet.t;
388-
}
406+
let get_commit_author ~get_commit (n : n) sha =
407+
let repo_url = git_ssh_to_https n.pipeline.repository in
408+
let commits_url = git_ssh_to_commits_url n.pipeline.repository in
409+
let* (gh_commit : Github_t.api_commit) = get_commit ~commits_url ~repo_url ~sha in
410+
Lwt.return_ok gh_commit.commit.author.email
389411

390-
let to_failed_step ~(n : n) (job : Buildkite_t.job) =
412+
let to_failed_step ~(n : n) ~author (job : Buildkite_t.job) =
391413
{
392414
Buildkite_t.id = Option.default job.name job.step_key;
393415
name = job.name;
394416
build_url = job.web_url;
395417
created_at = Timestamp.wrap_with_fallback n.build.created_at;
396-
author = extract_metadata_email n.build.meta_data |> Option.default "";
418+
author;
397419
escalated_at = None;
398420
}
399421

400-
let to_failed_step_set jobs n = List.map (to_failed_step ~n) jobs |> FailedStepSet.of_list
422+
let to_failed_step_set author jobs n = List.map (to_failed_step ~n ~author) jobs |> FailedStepSet.of_list
401423

402-
let new_failed_steps ~(cfg : Config_t.config) ~(repo_state : State_t.repo_state) ~get_build ~db_update (n : n) =
424+
let new_failed_steps ~(cfg : Config_t.config) ~(repo_state : State_t.repo_state) ~get_build ~get_commit ~db_update
425+
(n : n) =
403426
let org, pipeline, build_nr = Build.get_org_pipeline_build' n.build.web_url in
404427
let repo_key = repo_key org pipeline in
405428
let partition_build_steps () =
406429
log#info "Fetching failed steps for build %s/%s/%s" org pipeline build_nr;
407430
let* (build : Buildkite_t.get_build_res) = get_build n.build.web_url in
431+
let* author = get_commit_author ~get_commit n build.sha in
408432
Lwt.return_ok
409433
{
410-
failed_steps = to_failed_step_set (Build.filter_failed_jobs build.jobs) n;
411-
passed_steps = to_failed_step_set (Build.filter_passed_jobs build.jobs) n;
434+
failed_steps = to_failed_step_set author (Build.filter_failed_jobs build.jobs) n;
435+
passed_steps = to_failed_step_set author (Build.filter_passed_jobs build.jobs) n;
412436
}
413437
in
414438
match Stringtbl.find_opt repo_state.failed_steps repo_key with

0 commit comments

Comments
 (0)