Merge pull request #78 from ugyballoons/table-creation

Add table of metadata to listing pages

.gitignore

@@ -141,4 +141,10 @@ dmypy.json
 
 # jupyter notebooks
 *.ipynb
-.ipynb_checkpoints
+
+# npm and eslint
+*/jsconfig.json
+.eslintrc.*
+node_modules/
+package-lock.json
+package.json

requirements/dev.in

@@ -17,3 +17,4 @@ pytest
 pytest-aiohttp
 python-dateutil
 types-python-dateutil
+types-requests

requirements/dev.txt

@@ -671,6 +671,14 @@ types-python-dateutil==2.8.18 \
     --hash=sha256:8695c7d7a5b1aef4002f3ab4e1247e23b1d41cd7cc1286d4594c2d8c5593c991 \
     --hash=sha256:fd5ed97262b76ae684695ea38ace8dd7c1bc9491aba7eb4edf6654b7ecabc870
     # via -r requirements/dev.in
+types-requests==2.28.0 \
+    --hash=sha256:85383b4ef0535f639c3f06c5bbb6494bbf59570c4cd88bbcf540f0b2ac1b49ab \
+    --hash=sha256:9863d16dfbb3fa55dcda64fa3b989e76e8859033b26c1e1623e30465cfe294d3
+    # via -r requirements/dev.in
+types-urllib3==1.26.15 \
+    --hash=sha256:6011befa13f901fc934f59bb1fd6973be6f3acf4ebfce427593a27e7f492918f \
+    --hash=sha256:c89283541ef92e344b7f59f83ea9b5a295b16366ceee3f25ecfc5593c79f794e
+    # via types-requests
 typing-extensions==4.2.0 \
     --hash=sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708 \
     --hash=sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376

requirements/main.txt

@@ -477,38 +477,38 @@ pyasn1-modules==0.2.8 \
     --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \
     --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74
     # via google-auth
-pycares==4.2.0 \
-    --hash=sha256:1857631b448925aabc2b306baa27fbedb27ca49036ce5c083ae21eef7850adfa \
-    --hash=sha256:23d3f272b9f6ea80e488de76ea93e0edde418d545e86562b9e9094952a6ce658 \
-    --hash=sha256:2539b9e20c52d9daedfa5f52f84ba6ea8cb351236fcda6034ab5422f0be1d171 \
-    --hash=sha256:2707044e62ae32d816141e99bb2cfbba02700f4b5ffebbd7ba3fb2070af9a5ff \
-    --hash=sha256:2c4a6410e01f401982b7bc9d20756a9e1549bab1cc431eb9a7a26193168895b0 \
-    --hash=sha256:3178be9e269795216376220d8a847bf30497fcf415821d58659b07dc2873e9b2 \
-    --hash=sha256:45cc3feae3335b70cd4ec7447cd6ef9c11c7d9aa6731153cb87bae4db69fed8d \
-    --hash=sha256:48c733b8087f618d64b05c5807264ea94987b599cf37688ba59d23c57e197ff1 \
-    --hash=sha256:496cc499980d9be6b151f5028aef9e735a4881d8c2eaa32b2562678dce463787 \
-    --hash=sha256:4a491bb6c9c6420f3547e89898902e6f67a11ba42ff39026cc94aa238bcc2e67 \
-    --hash=sha256:50a50b6c55919563745b0b242aa54eeb727696351ac187156d2d0ab9614fd726 \
-    --hash=sha256:67b9495aebd575c18cf0371636057fdaf198f8b60e7bf1ec7fdd511154094f94 \
-    --hash=sha256:8401526f945c96210f81887ad38d5415113f149719893006fc148862fbf4c128 \
-    --hash=sha256:84d6572dff420631758ebdffc13283895e7fb04a54823641fa3830cdcb6f4743 \
-    --hash=sha256:88709762179daa858b494f66f5ad35cf9e4be39893a565aa24148a237d05f3cd \
-    --hash=sha256:9070d507349d8b75c3f3e81b7d519555cd009391ae2b8fce7505753058e6cd08 \
-    --hash=sha256:a1c0cc79c442eecd0beeaf2bcd91541a0ff366b43df5bb061d57dbe8b5d04c38 \
-    --hash=sha256:a3cf407fba0ab0f21417676d480926dd438e12a0134d2e50e6164b0b4822ff63 \
-    --hash=sha256:a56eea0cf117edffb51479e2acda15b1b02a33a225838708828a5c1010333ba1 \
-    --hash=sha256:ad97e6db69a8b825fa4590cddbd00b00441fc16bfbebf3a37f6f9b6f72d9380d \
-    --hash=sha256:ae732c15c5bb5a381c521e3eb4daf56175e890f4fa1fddf2fc2349a778a94ed0 \
-    --hash=sha256:b076d5d8a3f94bc38efeb7a5ac9772190e13515a510f96d600d58b12d25b2ae7 \
-    --hash=sha256:b286649597791cd53072b2f3383cc38fc14a8ab016b78cb04bdcaa6ecce3b8ce \
-    --hash=sha256:bc24ee402697d4babe3a3652af6ede7fdd18f8d33a691d62c1e33131435f66e2 \
-    --hash=sha256:bce6e64b85aad08dbeb6f92aecd53d856ac534fb70e6d3a5baa7cda8d2301c16 \
-    --hash=sha256:de97ae19883b3f49f7c75d60292d49a21d06e64113ed69b5c6d5eba8ae4ef60a \
-    --hash=sha256:e0cffbf241c0384a6b56e2cddfb7bbd7b9830954edd756f685ad2b36e54ee107 \
-    --hash=sha256:e6bbf663d89a93df0386050db430a26d98155d1c6d9f1e522ac8b01d38016c10 \
-    --hash=sha256:e7b396e074efca974b211378f2bfb864f9be0465ba75f6a8851d8b115d928d79 \
-    --hash=sha256:f1aadb7ebaa6da649ba3e665898aebd9f1ab57ece9af6c0d1c089074bf9a822a \
-    --hash=sha256:f9d057445ebc381398fb26fb87e6b9d2ed416e58074710a3e7feb6115b8a28ac
+pycares==4.2.1 \
+    --hash=sha256:061dd4c80fec73feb150455b159704cd51a122f20d36790033bd6375d4198579 \
+    --hash=sha256:15dd5cf21bc73ad539e8aabf7afe370d1df8af7bc6944cd7298f3bfef0c1a27c \
+    --hash=sha256:1a9506d496efeb809a1b63647cb2f3f33c67fcf62bf80a2359af692fef2c1755 \
+    --hash=sha256:1f37f762414680063b4dfec5be809a84f74cd8e203d939aaf3ba9c807a9e7013 \
+    --hash=sha256:2113529004df4894783eaa61e9abc3a680756b6f033d942f2800301ae8c71c29 \
+    --hash=sha256:2fd53eb5b441c4f6f9c78d7900e05883e9998b34a14b804be4fc4c6f9fea89f3 \
+    --hash=sha256:3636fccf643c5192c34ee0183c514a2d09419e3a76ca2717cef626638027cb21 \
+    --hash=sha256:396ee487178e9de06ca4122a35a157474db3ce0a0db6038a31c831ebb9863315 \
+    --hash=sha256:3b78bdee2f2f1351d5fccc2d1b667aea2d15a55d74d52cb9fd5bea8b5e74c4dc \
+    --hash=sha256:4ee625d7571039038bca51ae049b047cbfcfc024b302aae6cc53d5d9aa8648a8 \
+    --hash=sha256:5333b51ef4ff3e8973b4a1b57cad5ada13e15552445ee3cd74bd77407dec9d44 \
+    --hash=sha256:66b5390a4885a578e687d3f2683689c35e1d4573f4d0ecf217431f7bb55c49a0 \
+    --hash=sha256:6724573e830ea2345f4bcf0f968af64cc6d491dc2133e9c617f603445dcdfa58 \
+    --hash=sha256:735b4f75fd0f595c4e9184da18cd87737f46bc81a64ea41f4edce2b6b68d46d2 \
+    --hash=sha256:7a901776163a04de5d67c42bd63a287cff9cb05fc041668ad1681fe3daa36445 \
+    --hash=sha256:8bd6ed3ad3a5358a635c1acf5d0f46be9afb095772b84427ff22283d2f31db1b \
+    --hash=sha256:99e00e397d07a79c9f43e4303e67f4f97bcabd013bda0d8f2d430509b7aef8a0 \
+    --hash=sha256:9b05c2cec644a6c66b55bcf6c24d4dfdaf2f7205b16e5c4ceee31db104fac958 \
+    --hash=sha256:a521d7f54f3e52ded4d34c306ba05cfe9eb5aaa2e5aaf83c96564b9369495588 \
+    --hash=sha256:b03f69df69f0ab3bfb8dbe54444afddff6ff9389561a08aade96b4f91207a655 \
+    --hash=sha256:c8a46839da642b281ac5f56d3c6336528e128b3c41eab9c5330d250f22325e9d \
+    --hash=sha256:d2e8ec4c8e07c986b70a3cc8f5b297c53b08ac755e5b9797512002a466e2de86 \
+    --hash=sha256:d83f193563b42360528167705b1c7bb91e2a09f990b98e3d6378835b72cd5c96 \
+    --hash=sha256:d9cd826d8e0c270059450709bff994bfeb072f79d82fd3f11c701690ff65d0e7 \
+    --hash=sha256:e4dc37f732f7110ca6368e0128cbbd0a54f5211515a061b2add64da2ddb8e5ca \
+    --hash=sha256:e75cbd4d3b3d9b02bba6e170846e39893a825e7a5fb1b96728fc6d7b964f8945 \
+    --hash=sha256:e7a95763cdc20cf9ec357066e656ea30b8de6b03de6175cbb50890e22aa01868 \
+    --hash=sha256:e9dbfcacbde6c21380c412c13d53ea44b257dea3f7b9d80be2c873bb20e21fee \
+    --hash=sha256:f05223de13467bb26f9a1594a1799ce2d08ad8ea241489fecd9d8ed3bbbfc672 \
+    --hash=sha256:f8e6942965465ca98e212376c4afb9aec501d8129054929744b2f4a487c8c14b \
+    --hash=sha256:fbd53728d798d07811898e11991e22209229c090eab265a53d12270b95d70d1a
     # via aiodns
 pycparser==2.21 \
     --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \

src/rubintv/handlers/external/endpoints.py

@@ -11,6 +11,7 @@
 from datetime import date, datetime, timedelta
 from typing import Any, Dict, Iterator, List, Optional
 
+import requests
 from aiohttp import web
 from google.cloud.storage import Bucket
 from jinja2 import Environment, PackageLoader, select_autoescape
@@ -137,27 +138,26 @@ async def get_recent_table(request: web.Request) -> web.Response:
             bucket = request.config_dict["rubintv/gcs_bucket"]
             events = get_most_recent_day_events(bucket, camera)
             channels = camera.channels
-            grid_columns = build_grid_columns_css(channels)
+            the_date = events[0].cleanDate()
+
+            metadata_json = get_metadata_json(
+                bucket.name, camera.slug, the_date, logger
+            )
+
             page = get_formatted_page(
                 "cameras/camera.jinja",
                 title=title,
                 camera=camera,
                 channels=channels,
-                date=events[0].cleanDate(),
+                date=the_date,
                 events=events,
-                grid_columns=grid_columns,
+                metadata=metadata_json,
             )
+
     logger.info("get_recent_table", duration=timer.seconds)
     return web.Response(text=page, content_type="text/html")
 
 
-def build_grid_columns_css(channels: Dict[str, Channel]) -> str:
-    grid_columns = "1fr"
-    for channel in channels:
-        grid_columns += " 40px"
-    return grid_columns
-
-
 @routes.get("/{camera}/update/{date}")
 async def update_todays_table(request: web.Request) -> web.Response:
     logger = request["safir/logger"]
@@ -191,13 +191,17 @@ async def update_todays_table(request: web.Request) -> web.Response:
             bucket, camera, recent_events, the_date
         )
         events = flatten_events_dict_into_list(camera, events_dict)
-        grid_columns = build_grid_columns_css(camera.channels)
+
+        metadata_json = get_metadata_json(
+            bucket.name, camera.slug, date_str, logger
+        )
+
         page = get_formatted_page(
             "cameras/data-table-header.jinja",
             camera=camera,
             date=the_date,
             events=events,
-            grid_columns=grid_columns,
+            metadata=metadata_json,
         )
 
     logger.info("update_todays_table", duration=timer.seconds)
@@ -208,6 +212,7 @@ async def update_todays_table(request: web.Request) -> web.Response:
 async def get_historical(request: web.Request) -> web.Response:
     logger = request["safir/logger"]
     with Timer() as timer:
+        bucket = request.config_dict["rubintv/gcs_bucket"]
         camera = cameras[request.match_info["camera"]]
         if not camera.has_historical:
             raise web.HTTPNotFound()
@@ -230,7 +235,9 @@ async def get_historical(request: web.Request) -> web.Response:
         smrd_dict = historical.get_events_for_date(camera, smrd)
         smrd_events = flatten_events_dict_into_list(camera, smrd_dict)
 
-        grid_columns = build_grid_columns_css(camera.channels)
+        metadata_json = get_metadata_json(
+            bucket.name, camera.slug, smrd.strftime("%Y-%m-%d"), logger
+        )
 
         page = get_formatted_page(
             "cameras/historical.jinja",
@@ -241,7 +248,7 @@ async def get_historical(request: web.Request) -> web.Response:
             month_names=month_names(),
             date=smrd,
             events=smrd_events,
-            grid_columns=grid_columns,
+            metadata=metadata_json,
         )
 
     logger.info("get_historical", duration=timer.seconds)
@@ -250,6 +257,8 @@ async def get_historical(request: web.Request) -> web.Response:
 
 @routes.get("/{camera}/historical/{date_str}")
 async def get_historical_day_data(request: web.Request) -> web.Response:
+    logger = request["safir/logger"]
+    bucket = request.config_dict["rubintv/gcs_bucket"]
     camera = cameras[request.match_info["camera"]]
     if not camera.has_historical:
         return web.Response(
@@ -261,17 +270,44 @@ async def get_historical_day_data(request: web.Request) -> web.Response:
     the_date = date(year, month, day)
     day_dict = historical.get_events_for_date(camera, the_date)
     day_events = flatten_events_dict_into_list(camera, day_dict)
-    grid_columns = build_grid_columns_css(camera.channels)
+
+    metadata_json = get_metadata_json(
+        bucket.name, camera.slug, date_str, logger
+    )
+
     page = get_formatted_page(
         "cameras/data-table-header-with-day-channels.jinja",
         camera=camera,
         date=the_date,
         events=day_events,
-        grid_columns=grid_columns,
+        metadata=metadata_json,
     )
     return web.Response(text=page, content_type="text/html")
 
 
+def get_metadata_json(
+    bucket_name: str, camera_slug: str, date_str: str, logger: Any
+) -> str:
+    metadata_json = "{}"
+    metadata_url = get_metadata_url(bucket_name, camera_slug, date_str)
+    try:
+        metadata_res = requests.get(metadata_url)
+        if metadata_res.status_code == 200:
+            metadata_json = metadata_res.text
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Error retrieving from {metadata_url} with error: {e}")
+    return metadata_json
+
+
+def get_metadata_url(bucket_name: str, camera_slug: str, date_str: str) -> str:
+    #  reformat the date string from YYYY-m-d to YYYYmmdd
+    date_str = "".join([f"{int(x):02}" for x in date_str.split("-")])
+
+    url = f"https://storage.googleapis.com/{bucket_name}/"
+    url += f"{camera_slug}_metadata/dayObs_{date_str}.json"
+    return url
+
+
 def month_names() -> List[str]:
     return [date(2000, m, 1).strftime("%B") for m in list(range(1, 13))]
 

src/rubintv/models.py

@@ -43,7 +43,7 @@ def parse_filename(self, delimiter: str = "_") -> Tuple:
         date = nList[2]
         seq_str = nList[4][:-4]  # Strip extension
         if seq_str == "final":
-            seq = 0
+            seq = 99999
         else:
             seq = int(seq_str)
         return (name, prefix, datetime.strptime(date, "%Y-%m-%d"), seq)

src/rubintv/static/js/allsky-calendar.js

@@ -1,14 +1,16 @@
-(function($){
+/* global jQuery */
+
+(function ($) {
   // click to retrieve & display data for day:
-  $('.day').click(function(){
-    let date = this.dataset.date;
-    let url_path = document.location.pathname;
-    $('.current-movie').load(url_path + "/" + date);
-  });
-  $('.year-title').click(function(){
-    $year_to_open = $(this).parent('.year');
-    if ($year_to_open.hasClass('open')) return;
+  $('.day').click(function () {
+    const date = this.dataset.date
+    const urlPath = document.location.pathname
+    $('.current-movie').load(urlPath + '/' + date)
+  })
+  $('.year-title').click(function () {
+    const $yearToOpen = $(this).parent('.year')
+    if ($yearToOpen.hasClass('open')) return
     $('.year.open').removeClass('open').addClass('closed')
-    $year_to_open.removeClass('closed').addClass('open');
-  });
+    $yearToOpen.removeClass('closed').addClass('open')
+  })
 })(jQuery)

src/rubintv/static/js/allsky-refresh.js

@@ -1,33 +1,34 @@
-(function($){
-  let url_path = document.location.pathname;
-  let currentImage = $('.current-still');
-  let currentMovie = $('.current-movie')
+/* global jQuery */
 
-  setInterval(function refresh(){
-    $.get(url_path + "/update/image", function(data){
-      if (data.channel == "image"){
-        currentImage.find('img').attr({'src': data.url})
-        currentImage.find('a').attr({'href': data.url})
-        currentImage.find('.subheader h3').text(`${data.date} : Image ${data.seq}`);
+(function ($) {
+  const urlPath = document.location.pathname
+  const currentImage = $('.current-still')
+  const currentMovie = $('.current-movie')
+
+  setInterval(function refresh () {
+    $.get(urlPath + '/update/image', function (data) {
+      if (data.channel === 'image') {
+        currentImage.find('img').attr({ src: data.url })
+        currentImage.find('a').attr({ href: data.url })
+        currentImage.find('.subheader h3').text(`${data.date} : Image ${data.seq}`)
         currentImage.find('.desc').text(data.name)
       }
-    });
-  }, 5000);
+    })
+  }, 5000)
 
-  let videoCheckLatest = function() {
-    video = currentMovie.find("video")[0];
-    $.get(url_path + "/update/monitor", function(data){
-    let currentMovieUrl = $(video).find('source').attr('src');
-    if (data.channel == "monitor" && data.url != currentMovieUrl) {
-      $(video).find('source').attr({'src': data.url});
-      currentMovie.find('.movie-date').text(data.date);
-      currentMovie.find('.movie-number').text(data.seq)
-      currentMovie.find('.desc').text(data.name);
-      video.load();
+  const videoCheckLatest = function () {
+    const video = currentMovie.find('video')[0]
+    $.get(urlPath + '/update/monitor', function (data) {
+      const currentMovieUrl = $(video).find('source').attr('src')
+      if (data.channel === 'monitor' && data.url !== currentMovieUrl) {
+        $(video).find('source').attr({ src: data.url })
+        currentMovie.find('.movie-date').text(data.date)
+        currentMovie.find('.movie-number').text(data.seq)
+        currentMovie.find('.desc').text(data.name)
+        video.load()
       }
-    });
+    })
   }
 
-  setInterval(videoCheckLatest, 5000);
-
+  setInterval(videoCheckLatest, 5000)
 })(jQuery)