From ea44f5014b979cc2b823cfdbf81198757c1996d0 Mon Sep 17 00:00:00 2001 From: Ross MacArthur Date: Mon, 6 Jun 2022 18:59:26 +0200 Subject: [PATCH] Refactor API queries, add pagination --- Cargo.lock | 148 +++++++++++++++++++++++++++++++++++--------------- Cargo.toml | 1 + src/gitlab.rs | 143 +++++++++++++++++++++++++++++++++++------------- 3 files changed, 209 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b772182..2856b22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "autocfg" @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.53+curl-7.82.0" +version = "0.4.55+curl-7.83.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8092905a5a9502c312f223b2775f57ec5c5b715f9a15ee9d2a8591d1364a0352" +checksum = "23734ec77368ec583c2e61dd3f0b0e5c98b93abe6d2a004ca06b91dd7e3e2762" dependencies = [ "cc", "libc", @@ -141,6 +141,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "upon", ] [[package]] @@ -154,9 +155,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "lazy_static" @@ -166,15 +167,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", "libc", @@ -184,18 +185,18 @@ dependencies = [ [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -203,18 +204,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "openssl-probe" @@ -224,9 +225,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "835363342df5fba8354c5b453325b110ffd54044e588c539cf2f20a8014e4cb1" dependencies = [ "autocfg", "cc", @@ -237,9 +238,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "powerpack" @@ -264,52 +265,52 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "quote" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "schannel" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "winapi", + "windows-sys", ] [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -318,9 +319,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", @@ -350,13 +351,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.89" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -377,10 +378,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] -name = "unicode-xid" -version = "0.2.2" +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "upon" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "c70579f216a758dcb695ec3a44cedbe439478522318354a80caf540967ad3b2d" +dependencies = [ + "serde", + "unicode-width", +] [[package]] name = "vcpkg" @@ -421,3 +438,46 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" diff --git a/Cargo.toml b/Cargo.toml index b0ee84e..73502ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,4 @@ powerpack = { version = "0.4.1", features = ["detach"] } serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.79" sha1 = "0.10.1" +upon = "0.1.0" diff --git a/src/gitlab.rs b/src/gitlab.rs index 014e722..1859dd9 100644 --- a/src/gitlab.rs +++ b/src/gitlab.rs @@ -2,14 +2,85 @@ use std::io::prelude::*; use anyhow::{anyhow, Context, Result}; use chrono::DateTime; +use once_cell::sync::Lazy; use serde::de::DeserializeOwned; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json as json; use crate::config::CONFIG; use crate::{Issue, MergeRequest}; -fn fetch(query: &str, token: &str) -> Result { +type ParseFn = fn(json::Value) -> Result; + +struct Query<'a, T> { + name: &'a str, + project: &'a str, + template: &'a str, + page_info_ptr: &'a str, + nodes_ptr: &'a str, + parse_fn: ParseFn, +} + +#[derive(Deserialize)] +struct PageInfo { + #[serde(rename = "endCursor")] + cursor: String, + #[serde(rename = "hasNextPage")] + has_next: bool, +} + +impl Query<'_, T> { + fn checksum(&self) -> [u8; 20] { + use sha1::*; + let mut hasher = Sha1::new(); + hasher.update(self.name.as_bytes()); + hasher.update(self.project.as_bytes()); + hasher.update(self.template.as_bytes()); + hasher.finalize().try_into().unwrap() + } +} + +fn fetch_and_parse(q: Query<'_, T>) -> Result> { + let token = CONFIG + .token + .as_ref() + .ok_or_else(|| anyhow!("GITLAB_TOKEN environment variable is not set!"))?; + + let mut r = crate::cache::load(q.name, q.checksum(), || fetch_all(&q, token))?; + let resps = r + .as_array_mut() + .context("cache value is not an array")? + .drain(..); + + let mut nodes = Vec::new(); + for resp in resps { + let ns: Vec = lookup(&resp, q.nodes_ptr)?; + nodes.extend(ns); + } + nodes.into_iter().map(q.parse_fn).collect() +} + +fn fetch_all(q: &Query<'_, T>, token: &str) -> Result { + static ENGINE: Lazy = + Lazy::new(|| upon::Engine::with_delims("", "<", ">")); + + let template = ENGINE.compile(q.template)?; + + let mut array = Vec::new(); + let mut args = None; + loop { + let query = template.render(upon::value! { project: q.project, args: args })?; + let resp = fetch(&query, token)?; + let page_info: PageInfo = lookup(&resp, q.page_info_ptr)?; + array.push(resp); + if !page_info.has_next { + break Ok(json::Value::Array(array)); + } + args = Some(format!(", after:\"{}\"", page_info.cursor)); + } +} + +fn fetch(query: &str, token: &str) -> Result { #[derive(Debug, Serialize)] struct Query<'a> { query: &'a str, @@ -43,27 +114,11 @@ fn fetch(query: &str, token: &str) -> Result { Ok(serde_json::from_slice(&buf)?) } -fn fetch_and_parse( - name: &str, - query: &str, - checksum: [u8; 20], - ptr: &str, - parse_fn: fn(json::Value) -> Result, -) -> Result> { - let token = CONFIG - .token - .as_ref() - .ok_or_else(|| anyhow!("GITLAB_TOKEN environment variable is not set!"))?; - let resp = crate::cache::load(name, checksum, || fetch(query, token))?; - let nodes: Vec = lookup(&resp, ptr)?; - nodes.into_iter().map(parse_fn).collect() -} - pub fn issues(name: &str, project: &str) -> Result> { - let query = r#" + let template = r#" query { project(fullPath: "") { - issues(state: opened) { + issues(state: opened ) { nodes { title author { @@ -84,21 +139,29 @@ query { } } } + pageInfo { + endCursor + hasNextPage + } } } } -"# - .replace("", project); - let ptr = "/data/project/issues/nodes"; - let checksum = checksum(&query); - fetch_and_parse(name, &query, checksum, ptr, parse_issue) +"#; + fetch_and_parse(Query { + name, + project, + template, + page_info_ptr: "/data/project/issues/pageInfo", + nodes_ptr: "/data/project/issues/nodes", + parse_fn: parse_issue, + }) } pub fn merge_requests(name: &str, project: &str) -> Result> { - let query = r#" + let template = r#" query { project(fullPath: "") { - mergeRequests(state: opened) { + mergeRequests(state: opened ) { nodes { title author { @@ -113,14 +176,23 @@ query { } } } + + pageInfo { + endCursor + hasNextPage + } } } } -"# - .replace("", project); - let ptr = "/data/project/mergeRequests/nodes"; - let checksum = checksum(&query); - fetch_and_parse(name, &query, checksum, ptr, parse_merge_request) +"#; + fetch_and_parse(Query { + name, + project, + template, + page_info_ptr: "/data/project/mergeRequests/pageInfo", + nodes_ptr: "/data/project/mergeRequests/nodes", + parse_fn: parse_merge_request, + }) } fn parse_issue(value: json::Value) -> Result { @@ -172,10 +244,3 @@ where let list: Vec = lookup(value, ptr)?; list.into_iter().map(|v| lookup(&v, sub_ptr)).collect() } - -fn checksum(query: &str) -> [u8; 20] { - use sha1::*; - let mut hasher = Sha1::new(); - hasher.update(query.as_bytes()); - hasher.finalize().try_into().unwrap() -}