Skip to content

crawlconfig api: add 'currCrawlState' and 'currCrawlTimeStart' to crawlconfig list api (already queried on backend) #770

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 18, 2023
Merged

crawlconfig api: add 'currCrawlState' and 'currCrawlTimeStart' to crawlconfig list api (already queried on backend) #770

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
30b5745
crawlconfig api: add 'currCrawlState' and 'currCrawlTimeStart' to cra…
ikreymer Apr 17, 2023
036ea52
add missing null check
ikreymer Apr 17, 2023
65176bd
move to shared method
ikreymer Apr 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions backend/btrixcloud/crawlconfigs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,9 @@ class CrawlConfigOut(CrawlConfig):
"""Crawl Config Output, includes currCrawlId of running crawl"""

currCrawlId: Optional[str]
currCrawlStartTime: Optional[datetime]
currCrawlState: Optional[str]

profileName: Optional[str]

createdByName: Optional[str]
Expand Down Expand Up @@ -468,7 +471,7 @@ async def get_crawl_configs(
sort_direction: int = -1,
):
"""Get all crawl configs for an organization is a member of"""
# pylint: disable=too-many-locals
# pylint: disable=too-many-locals,too-many-branches
# Zero-index page for query
page = page - 1
skip = page * page_size
Expand Down Expand Up @@ -612,13 +615,13 @@ async def get_crawl_configs(
)
running = {}
for crawl in crawls:
running[crawl.cid] = crawl.id
running[crawl.cid] = crawl

configs = []
for res in items:
config = CrawlConfigOut.from_dict(res)
# pylint: disable=invalid-name
config.currCrawlId = running.get(config.id)
self._add_curr_crawl_stats(config, running.get(config.id))
configs.append(config)

return configs, total
Expand All @@ -644,7 +647,7 @@ async def get_running_crawl(self, crawlconfig: CrawlConfig):
)

if len(crawls) == 1:
return crawls[0].id
return crawls[0]

return None

Expand All @@ -659,6 +662,15 @@ async def _annotate_with_crawl_stats(self, crawlconfig: CrawlConfigOut):
crawlconfig.lastCrawlState = crawl_stats["last_crawl_state"]
return crawlconfig

async def _add_curr_crawl_stats(self, crawlconfig, crawl):
"""Add stats from current running crawl, if any"""
if not crawl:
return

crawlconfig.currCrawlId = crawl.id
crawlconfig.currCrawlStartTime = crawl.started
crawlconfig.currCrawlState = crawl.state

async def get_crawl_config_out(self, cid: uuid.UUID, org: Organization):
"""Return CrawlConfigOut, including state of currently running crawl, if active
also include inactive crawl configs"""
Expand All @@ -672,7 +684,9 @@ async def get_crawl_config_out(self, cid: uuid.UUID, org: Organization):
)

if not crawlconfig.inactive:
crawlconfig.currCrawlId = await self.get_running_crawl(crawlconfig)
self._add_curr_crawl_stats(
crawlconfig, await self.get_running_crawl(crawlconfig)
)

user = await self.user_manager.get(crawlconfig.createdBy)
# pylint: disable=invalid-name
Expand Down