add cfclient

src/api/cfapi.nim

@@ -8,7 +8,7 @@
 ## Docs for the official API are available at https://docs.curseforge.com.
 ## Requests to the proxy stay the same, except the base URL is switched out.
 
-import std/[asyncdispatch, json, options, sequtils, strutils, sugar, tables]
+import std/[asyncdispatch, json, options, strutils]
 import uri except Url
 import cfcore, http
 
@@ -22,69 +22,33 @@ const
 type
   CfApiError* = object of HttpRequestError
 
-proc sortTo[T, X](s: seq[T], x: seq[X], pred: proc (x: T): X): seq[T] =
-  ## sort `s` so that the order of its items matches `x`.
-  ## `pred` should be a function that returns a unique value to which `s` is sorted.
-  assert s.len == x.len
-
-  var table = initTable[X, T]()
-  for sItem in s:
-    table[pred(sItem)] = sItem
-  for xItem in x:
-    result.add(table[xItem])
-
-proc fetchAddonsByQuery*(query: string, category: Option[CfAddonGameCategory]): Future[seq[CfAddon]] {.async.} =
+proc fetchAddonsByQuery*(query: string, category: Option[CfAddonGameCategory]): Future[JsonNode] {.async.} =
   ## retrieves all addons that match the given `query` search and `category`.
   let encodedQuery = encodeUrl(query, usePlus = false)
   var url = addonsBaseUrl & "/v1/mods/search?gameId=432&pageSize=50&sortField=6&sortOrder=desc&searchFilter=" & encodedQuery
   if category.isSome:
     url = url & "&classId=" & $ord(category.get())
-  return get(url.Url).await.parseJson["data"].addonsFromForgeSvc
-
-proc fetchAddonsByQuery*(query: string, category: CfAddonGameCategory): Future[seq[CfAddon]] =
-  ## retrieves all addons that match the given `query` search and `category`.
-  return fetchAddonsByQuery(query, category = some(category))
+  return get(url.Url).await.parseJson["data"]
 
-proc fetchAddonsByQuery*(query: string): Future[seq[CfAddon]] =
-  ## retrieves all addons that match the given `query` search.
-  return fetchAddonsByQuery(query, category = none[CfAddonGameCategory]())
-
-proc fetchAddon*(projectId: int): Future[CfAddon] {.async.} =
+proc fetchAddon*(projectId: int): Future[JsonNode] {.async.} =
   ## get the addon with the given `projectId`.
   let url = addonsBaseUrl & "/v1/mods/" & $projectId
   try:
-    return get(url.Url).await.parseJson["data"].addonFromForgeSvc
+    return get(url.Url).await.parseJson["data"]
   except HttpRequestError:
     raise newException(CfApiError, "addon with project id '" & $projectId & "' not found.")
 
-proc fetchAddons*(projectIds: seq[int], chunk = true): Future[seq[CfAddon]] {.async.} =
+proc fetchAddons*(projectIds: seq[int]): Future[JsonNode] {.async.} =
   ## get all addons with their given `projectId`.
-  ## 
-  ## chunks the projectIds to minimize request size and to pinpoint errors better.
-  if projectIds.len > 10 and chunk:
-    let futures: seq[Future[seq[CfAddon]]] = collect:
-      for chunkedIds in projectIds.distribute(int(projectIds.len / 10), spread = true):
-        fetchAddons(chunkedIds, chunk = false)
-    let addons: seq[seq[CfAddon]] = await all(futures)
-    return collect:
-      for addonSeq in addons:
-        for addon in addonSeq:
-          addon
-  else:
-    let url = addonsBaseUrl & "/v1/mods/"
-    let body = %* { "modIds": projectIds }
-    try:
-      let addons = post(url.Url, $body).await.parseJson["data"].addonsFromForgeSvc
-      if addons.len != projectIds.len:
-        raise newException(CfApiError, "one of the addons of project ids '" & $projectIds & "' was not found.")
-      return addons.sortTo(projectIds, (x) => x.projectId)
-    except HttpRequestError:
-      let futures: seq[Future[CfAddon]] = collect:
-        for projectId in projectIds:
-          fetchAddon(projectId)
-      return await all(futures)
+  let url = addonsBaseUrl & "/v1/mods/"
+  let body = %* { "modIds": projectIds }
+  try:
+    let addons = post(url.Url, $body).await.parseJson["data"]
+    return addons
+  except HttpRequestError:
+    raise newException(CfApiError, "one of the addons of project ids '" & $projectIds & "' was not found.")
 
-proc fetchAddon*(slug: string): Future[CfAddon] {.async.} =
+proc fetchAddon*(slug: string): Future[JsonNode] {.async.} =
   ## get the addon matching the `slug`.
   let reqBody = %* {
     "query": "{ addons(slug: \"" & slug & "\") { id }}"
@@ -96,30 +60,27 @@ proc fetchAddon*(slug: string): Future[CfAddon] {.async.} =
   let projectId = addons[0]["id"].getInt()
   return await fetchAddon(projectId)
 
-proc fetchAddonFiles*(projectId: int): Future[seq[CfAddonFile]] {.async.} =
+proc fetchAddonFiles*(projectId: int): Future[JsonNode] {.async.} =
   ## get all addon files associated with the given `projectId`.
   let url = addonsBaseUrl & "/v1/mods/" & $projectId & "/files?pageSize=10000"
   try:
-    return get(url.Url).await.parseJson["data"].addonFilesFromForgeSvc
+    return get(url.Url).await.parseJson["data"]
   except HttpRequestError:
     raise newException(CfApiError, "addon with project id '" & $projectId & "' not found.")
 
-proc fetchAddonFiles*(fileIds: seq[int]): Future[seq[CfAddonFile]] {.async.} =
+proc fetchAddonFiles*(fileIds: seq[int]): Future[JsonNode] {.async.} =
   ## get all addon files with their given `fileIds`.
   let url = addonsBaseUrl & "/v1/mods/files"
   let body = %* { "fileIds": fileIds }
   try:
-    let addonFiles = post(url.Url, $body).await.parseJson["data"].addonFilesFromForgeSvc
-    if addonFiles.len != fileIds.len:
-      raise newException(CfApiError, "one of the addon files of file ids '" & $fileIds & "' was not found.")
-    return addonFiles.sortTo(fileIds, (x) => x.fileId)
+    return post(url.Url, $body).await.parseJson["data"]
   except HttpRequestError:
     raise newException(CfApiError, "one of the addon files of file ids '" & $fileIds & "' was not found.")
 
-proc fetchAddonFile*(projectId: int, fileId: int): Future[CfAddonFile] {.async.} =
+proc fetchAddonFile*(projectId: int, fileId: int): Future[JsonNode] {.async.} =
   ## get the addon file with the given `fileId` & `projectId`.
   let url = addonsBaseUrl & "/v1/mods/" & $projectId & "/files/" & $fileId
   try:
-    return get(url.Url).await.parseJson["data"].addonFileFromForgeSvc
+    return get(url.Url).await.parseJson["data"]
   except HttpRequestError:
     raise newException(CfApiError, "addon with project & file id  '" & $projectId & ':' & $fileId & "' not found.")

src/api/cfclient.nim

@@ -0,0 +1,166 @@
+## Provides functions for retrieving information about addons and addon files.
+## 
+## uses the cfcache to reduce the number of requests sent to to the cfapi.
+
+import std/[asyncdispatch, asyncfutures, options, sequtils, sugar, tables]
+import cfapi, cfcache, cfcore
+export CfApiError
+
+const
+  chunkSize = 10 ## how many ids a request should be chunked to.
+
+proc sortTo[T, X](s: seq[T], x: seq[X], pred: proc (x: T): X): seq[T] =
+  ## sort `s` so that the order of its items matches `x`.
+  ## `pred` should be a function that returns a unique value to which `s` is sorted.
+  assert s.len == x.len
+
+  result = newSeq[T]()
+  var table = initTable[X, T]()
+  for sItem in s:
+    table[pred(sItem)] = sItem
+  for xItem in x:
+    result.add(table[xItem])
+
+proc flatten[T](s: seq[seq[T]]): seq[T] =
+  ## flatten `s`.
+  result = newSeq[T]()
+  for arr in s:
+    result = result.concat arr
+
+proc fetchAddonsByQuery*(query: string, category: Option[CfAddonGameCategory]): Future[seq[CfAddon]] {.async.} =
+  ## retrieves all addons that match the given `query` search and `category`.
+  let data = await cfapi.fetchAddonsByQuery(query, category)
+  cfcache.putAddons(data)
+  return data.addonsFromForgeSvc
+
+proc fetchAddonsByQuery*(query: string, category: CfAddonGameCategory): Future[seq[CfAddon]] =
+  ## retrieves all addons that match the given `query` search and `category`.
+  return fetchAddonsByQuery(query, category = some(category))
+
+proc fetchAddonsByQuery*(query: string): Future[seq[CfAddon]] =
+  ## retrieves all addons that match the given `query` search.
+  return fetchAddonsByQuery(query, category = none[CfAddonGameCategory]())
+
+proc fetchAddon(projectId: int, lookupCache: bool): Future[CfAddon] {.async.} =
+  ## get the addon with the given `projectId`.
+  if lookupCache:
+    withCachedAddon(addon, projectId):
+      return addon.addonFromForgeSvc
+  let data = await cfapi.fetchAddon(projectId)
+  cfcache.putAddon(data)
+  return data.addonFromForgeSvc
+
+proc fetchAddon*(projectId: int): Future[CfAddon] =
+  ## get the addon with the given `projectId`.
+  return fetchAddon(projectId, lookupCache = true)
+
+proc fetchAddonsChunks(projectIds: seq[int]): Future[seq[CfAddon]] {.async.} =
+  ## get all addons with their given `projectId`.
+  if projectIds.len == 0:
+    return @[]
+  try:
+    let data = await cfapi.fetchAddons(projectIds)
+    cfcache.putAddons(data)
+    return data.addonsFromForgeSvc
+  except CfApiError:
+    # fallback to looking up the ids individually
+    return await all(projectIds.map((x) => fetchAddon(x, lookupCache = false)))
+
+proc fetchAddons*(projectIds: seq[int], chunk = true): Future[seq[CfAddon]] {.async.} =
+  ## get all addons with their given `projectId`.
+  ## 
+  ## chunks the projectIds to minimize request size and to pinpoint errors better.
+
+  # load all addons already in cache
+  result = newSeq[CfAddon]()
+  var missingIds = newSeq[int]()
+  for projectId in projectIds:
+    let addon = getAddon(projectId)
+    if addon.isSome:
+      result.add addon.get()
+    else:
+      missingIds.add projectId
+
+  if chunk and missingIds.len > chunkSize:
+    # if chunking is enabled, chunk the missing ids and fetch the chunks individually
+    let futures: seq[Future[seq[CfAddon]]] = collect:
+      for chunkedIds in missingIds.distribute(int(missingIds.len / chunkSize), spread = true):
+        fetchAddonsChunks(chunkedIds)
+    let addons: seq[seq[CfAddon]] = await all(futures)
+    result = result.concat(addons.flatten())
+  elif missingIds.len > 0:
+    # otherwise just fetch them
+    result = result.concat(await fetchAddonsChunks(missingIds))
+
+  # check that all addons have been retrieved
+  if projectIds.len != result.len:
+    let currentIds = result.map((x) => x.projectId)
+    let missingIds = projectIds.filter((x) => x notin currentIds)
+    raise newException(CfApiError, "one of the addons of project ids '" & $missingIds & "' was not found.")
+  # sort so the output is deterministic
+  result = result.sortTo(projectIds, (x) => x.projectId)
+
+proc fetchAddon*(slug: string): Future[CfAddon] {.async.} =
+  ## get the addon matching the `slug`.
+  let data = await cfapi.fetchAddon(slug)
+  cfcache.putAddon(data)
+  return data.addonFromForgeSvc
+
+proc fetchAddonFiles*(projectId: int): Future[seq[CfAddonFile]] {.async.} =
+  ## get all addon files associated with the given `projectId`.
+  let data = await cfapi.fetchAddonFiles(projectId)
+  cfcache.putAddonFiles(data)
+  return data.addonFilesFromForgeSvc
+
+proc fetchAddonFilesChunks(fileIds: seq[int]): Future[seq[CfAddonFile]] {.async.} =
+  ## get all addons with their given `projectId`.
+  if fileIds.len == 0:
+    return @[]
+  try:
+    let data = await cfapi.fetchAddonFiles(fileIds)
+    cfcache.putAddonFiles(data)
+    return data.addonFilesFromForgeSvc
+  except CfApiError:
+    # fallback to looking up the ids individually
+    return all(fileIds.map((x) => fetchAddonFiles(x))).await.flatten()
+
+proc fetchAddonFiles*(fileIds: seq[int], chunk = true): Future[seq[CfAddonFile]] {.async.} =
+  ## get all addon files with their given `fileIds`.
+
+  # load all files already in cache
+  result = newSeq[CfAddonFile]()
+  var missingIds = newSeq[int]()
+  for fileId in fileIds:
+    let addonFile = getAddonFile(fileId)
+    if addonFile.isSome:
+      result.add addonFile.get()
+    else:
+      missingIds.add fileId
+
+  if chunk and missingIds.len > chunkSize:
+    # if chunking is enabled, chunk the missing ids and fetch the chunks individually
+    let futures: seq[Future[seq[CfAddonFile]]] = collect:
+      for chunkedIds in missingIds.distribute(int(missingIds.len / chunkSize), spread = true):
+        fetchAddonFilesChunks(chunkedIds)
+    let addons: seq[seq[CfAddonFile]] = await all(futures)
+    result = result.concat(addons.flatten())
+  elif missingIds.len > 0:
+    # otherwise just fetch them
+    result = result.concat(await fetchAddonFilesChunks(missingIds))
+
+  # check that all addons have been retrieved & fetch missing ones
+  if fileIds.len != result.len:
+    let currentIds = result.map((x) => x.fileId)
+    let missingIds = fileIds.filter((x) => x notin currentIds)
+    result = result.concat(all(missingIds.map((x) => fetchAddonFiles(x))).await.flatten())
+  # sort so the output is deterministic
+  result = result.sortTo(fileIds, (x) => x.fileId)
+
+proc fetchAddonFile*(projectId: int, fileId: int): Future[CfAddonFile] {.async.} =
+  ## get the addon file with the given `fileId` & `projectId`.
+  withCachedAddonFile(addonFile, fileId):
+    return addonFile.addonFileFromForgeSvc
+
+  let data = await cfapi.fetchAddonFile(projectId, fileId)
+  cfcache.putAddonFile(data)
+  return data.addonFileFromForgeSvc

src/cmd/add.nim

@@ -1,6 +1,6 @@
 import std/[asyncdispatch, options, strscans]
 import common
-import ../api/[cfapi, cfcore]
+import ../api/[cfclient, cfcore]
 import ../modpack/[install, manifest]
 import ../term/[log, prompt]
 import ../util/flow

src/cmd/list.nim

@@ -1,6 +1,6 @@
 import std/[algorithm, asyncdispatch, sequtils, strutils, os, sugar]
 import common
-import ../api/[cfapi, cfcore]
+import ../api/[cfclient, cfcore]
 import ../modpack/[manifest, modinfo]
 import ../term/log
 

src/cmd/pin.nim

@@ -1,6 +1,6 @@
 import std/[asyncdispatch, os]
 import common
-import ../api/cfapi
+import ../api/cfclient
 import ../modpack/manifest
 import ../term/[log, prompt]
 import ../util/flow

src/cmd/remove.nim

@@ -1,6 +1,6 @@
 import std/[asyncdispatch, os]
 import common
-import ../api/cfapi
+import ../api/cfclient
 import ../modpack/manifest
 import ../term/[log, prompt]
 import ../util/flow

src/cmd/update.nim

@@ -1,6 +1,6 @@
 import std/[asyncdispatch, os]
 import common
-import ../api/[cfapi, cfcore]
+import ../api/[cfclient, cfcore]
 import ../modpack/[install, manifest]
 import ../term/[log, prompt]
 import ../util/flow

src/cmd/upgrade.nim

@@ -1,5 +1,5 @@
 import std/[asyncdispatch, asyncfutures, sequtils, sugar, os]
-import ../api/[cfapi, cfcore]
+import ../api/[cfclient, cfcore]
 import ../modpack/[manifest, install]
 import ../term/[log, prompt]
 import ../util/flow

src/modpack/manifest.nim

@@ -6,7 +6,7 @@
 
 import std/[algorithm, asyncdispatch, json, options, os, sequtils, sugar]
 import loader
-import ../api/[cfapi, cfcore]
+import ../api/[cfclient, cfcore]
 import ../modpack/version
 import ../term/[color, log]
 export color, log

tests/api/tcfapi.nim → tests/api/tcfclient.nim

@@ -1,5 +1,5 @@
 import std/[asyncdispatch, options, sequtils, strutils, sugar]
-import api/[cfapi, cfcore]
+import api/[cfclient, cfcore]
 import ../tutils
 
 asyncBlock: # fetch by query
@@ -73,9 +73,9 @@ asyncBlock: # fetch mod file by project & file id
 
 asyncBlock: # fetch mod files by non-existing project & file id
   doAssertRaises(CfApiError):
-    discard await fetchAddonFile(306770, 99999999)
+    discard await fetchAddonFile(0, 99999999)
   doAssertRaises(CfApiError):
-    discard await fetchAddonFile(99999999, 2992184)
+    discard await fetchAddonFile(99999999, 0)
 
 asyncBlock: # check if dependencies are tracked
   let modFile = await fetchAddonFile(243121, 3366626)