Implement caching for slack user info

pyslackers · Jun 23, 2019 · a4eef5e · a4eef5e
1 parent 9abca40
commit a4eef5e
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 39 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -8,7 +8,8 @@ services:
       - redis
     environment:
       REDIS_URL: "redis://redis:6379/0"
-      SLACK_OAUTH_TOKEN: "${SLACK_OAUTH_TOKEN}"
+      SLACK_INVITE_TOKEN: "${SLACK_INVITE_TOKEN}"
+      SLACK_TOKEN: "${SLACK_TOKEN}"
     ports:
       - "8000:8000"
     volumes:

diff --git a/pyslackersweb/website/contexts.py b/pyslackersweb/website/contexts.py
@@ -30,11 +30,12 @@ async def background_jobs(app: web.Application) -> None:
     )
 
     scheduler.add_job(
-        tasks.sync_slack_users(app),
+        tasks.sync_slack_users,
         "cron",
         minute=0,
         jitter=30,
         next_run_time=datetime.utcnow() + timedelta(seconds=30),
+        args=(app["slack_client"], app["redis"]),
     )
 
     scheduler.add_job(

diff --git a/pyslackersweb/website/tasks.py b/pyslackersweb/website/tasks.py
@@ -7,12 +7,19 @@
 import slack
 from aiohttp import ClientSession, web
 from aioredis.abc import AbcConnection
+from slack.io.abc import SlackAPI
 
 from pyslackersweb.util.log import ContextAwareLoggerAdapter
 
 
 logger = ContextAwareLoggerAdapter(logging.getLogger(__name__))
 
+GITHUB_REPO_CACHE_KEY = "github:repos"
+
+SLACK_COUNT_CACHE_KEY = "slack:user:count"
+
+SLACK_TZ_CACHE_KEY = "slack:user:timezones"
+
 
 @dataclasses.dataclass(frozen=True)  # pylint: disable=too-few-public-methods
 class Repository:
@@ -33,7 +40,7 @@ class Channel:
 
 
 async def sync_github_repositories(
-    session: ClientSession, redis: AbcConnection, *, cache_key: str = "github:repos"
+    session: ClientSession, redis: AbcConnection, *, cache_key: str = GITHUB_REPO_CACHE_KEY
 ):
     logger.debug("Refreshing GitHub cache")
     try:
@@ -69,40 +76,35 @@ async def sync_github_repositories(
         raise
 
 
-def sync_slack_users(app: web.Application):
-    client = app["slack_client"]
-
-    async def _sync_slack_users():
-        logger.debug("Refreshing slack users cache.")
-        oauth_token = app["slack_token"]
-
-        if oauth_token is None:
-            logger.error("No slack oauth token set, unable to sync slack users.")
-            return
-
-        try:
-            counter = Counter()
-            async for user in client.iter(slack.methods.USERS_LIST, minimum_time=3):
-                if user["deleted"] or user["is_bot"] or not user["tz"]:
-                    continue
+async def sync_slack_users(
+    slack_client: SlackAPI,
+    redis: AbcConnection,
+    *,
+    cache_key_tz: str = SLACK_TZ_CACHE_KEY,
+    cache_key_count: str = SLACK_COUNT_CACHE_KEY,
+):
+    logger.debug("Refreshing slack users cache.")
 
-                counter[user["tz"]] += 1
+    try:
+        counter = Counter()
+        async for user in slack_client.iter(slack.methods.USERS_LIST, minimum_time=3):
+            if user["deleted"] or user["is_bot"] or not user["tz"]:
+                continue
 
-            logger.debug(
-                "Found %s users across %s timezones",
-                sum(counter.values()),
-                len(list(counter.keys())),
-            )
+            counter[user["tz"]] += 1
 
-            app.update(
-                slack_timezones=dict(counter.most_common(100)),
-                slack_user_count=sum(counter.values()),
-            )
-        except Exception:  # pylint: disable=broad-except
-            logger.exception("Error refreshing slack user cache")
-            return
+        logger.debug(
+            "Found %s users across %s timezones", sum(counter.values()), len(list(counter.keys()))
+        )
 
-    return _sync_slack_users
+        tx = redis.multi_exec()
+        tx.delete(cache_key_tz)
+        tx.hmset_dict(cache_key_tz, dict(counter.most_common(100)))
+        tx.set(cache_key_count, str(sum(counter.values())))
+        await tx.execute()
+    except Exception:  # pylint: disable=broad-except
+        logger.exception("Error refreshing slack user cache")
+        return
 
 
 def sync_slack_channels(app: web.Application):

diff --git a/pyslackersweb/website/views.py b/pyslackersweb/website/views.py
@@ -5,6 +5,7 @@
 from marshmallow.exceptions import ValidationError
 
 from .models import InviteSchema
+from .tasks import GITHUB_REPO_CACHE_KEY, SLACK_COUNT_CACHE_KEY, SLACK_TZ_CACHE_KEY
 
 routes = web.RouteTableDef()
 
@@ -13,11 +14,11 @@
 class Index(web.View):
     @template("index.html")
     async def get(self):
+        redis = self.request.app["redis"]
+
         return {
-            "member_count": self.request.app["slack_user_count"],
-            "projects": json.loads(
-                await self.request.app["redis"].get("github:repos", encoding="utf-8")
-            ),
+            "member_count": int((await redis.get(SLACK_COUNT_CACHE_KEY, encoding="utf-8")) or 0),
+            "projects": json.loads(await redis.get(GITHUB_REPO_CACHE_KEY, encoding="utf-8")),
             "sponsors": [
                 {
                     "image": self.request.app.router["static"].url_for(
@@ -40,9 +41,11 @@ class SlackView(web.View):
     schema = InviteSchema()
 
     async def shared_response(self):
+        redis = self.request.app["redis"]
+
         return {
-            "member_count": self.request.app["slack_user_count"],
-            "member_timezones": self.request.app["slack_timezones"],
+            "member_count": int((await redis.get(SLACK_COUNT_CACHE_KEY, encoding="utf-8")) or 0),
+            "member_timezones": await redis.hgetall(SLACK_TZ_CACHE_KEY, encoding="utf-8"),
             "errors": {},
         }