[Cal-5] Calendar API

calendar/README.md

@@ -17,14 +17,14 @@ This is separated from the server because:
 For getting started, there are some system dependencys which you will need.
 Please follow the [system dependencys docs](resources/documentation/Dependencys.md) before trying to run this part of our project.
 
-### How to Set up the Sqlite Database (needed for the `get`, `legacy_redirect` and `preview` endpoints)
+### How to Set up the Sqlite Database
 
 #### Getting the data
 
 To populate the database, you will need to get said data.
 There are multiple ways to do this, but the easiest way is to download the data from our [website](https://nav.tum.de/).
 
-(Assuming you are in the `server` directory)
+(Assuming you are in the `calendar` directory)
 
 ```bash
 mkdir -p data
@@ -73,7 +73,7 @@ To make sure that this specification is up-to-date and without holes, we run [sc
 python -m venv venv
 source venv/bin/activate
 pip install schemathesis
-st run --workers=auto --base-url=http://localhost:8081 --checks=all ../openapi.yaml
+st run --workers=auto --base-url=http://localhost:8060 --checks=all ../openapi.yaml
 ```
 
 Some fuzzing-goals may not be available for you locally, as they require prefix-routing (f.ex.`/cdn` to the CDN) and some fuzzing-goals are automatically tested in our CI.  

calendar/src/calendar.rs

@@ -0,0 +1,117 @@
+use crate::scraping::tumonline_calendar::XMLEvent;
+use crate::utils;
+use actix_web::web::Data;
+use actix_web::{get, web, HttpResponse};
+use chrono::NaiveDateTime;
+use diesel::prelude::*;
+use serde::{Deserialize, Serialize};
+use std::fmt::Debug;
+use tokio::sync::Mutex;
+
+#[derive(Deserialize, Debug)]
+pub struct CalendarQueryArgs {
+    start: NaiveDateTime, // eg. 2022-01-01T00:00:00
+    end: NaiveDateTime,   // eg. 2022-01-07T00:00:00
+}
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(calendar_handler);
+}
+
+#[get("/{id}")]
+pub async fn calendar_handler(
+    params: web::Path<String>,
+    web::Query(args): web::Query<CalendarQueryArgs>,
+    last_sync: Data<Mutex<Option<NaiveDateTime>>>,
+) -> HttpResponse {
+    let last_sync = match *last_sync.lock().await {
+        Some(last_sync) => last_sync,
+        None => {
+            return HttpResponse::ServiceUnavailable().body("Waiting for first sync with TUMonline")
+        }
+    };
+
+    let id = params.into_inner();
+    let conn = &mut utils::establish_connection();
+    use crate::schema::calendar::dsl::*;
+    let results = calendar
+        .filter(key.eq(&id))
+        .filter(dtstart.ge(&args.start))
+        .filter(dtend.le(&args.end))
+        .load::<XMLEvent>(conn);
+    match results {
+        Ok(result) => {
+            let events = result.into_iter().map(Event::from).collect();
+            HttpResponse::Ok().json(Events { events, last_sync })
+        }
+        Err(_) => HttpResponse::NotFound()
+            .content_type("text/plain")
+            .body("Not found"),
+    }
+}
+
+#[derive(Serialize, Debug)]
+struct Events {
+    events: Vec<Event>,
+    last_sync: NaiveDateTime,
+}
+
+#[derive(Serialize, Debug)]
+struct Event {
+    title: String,
+    start: NaiveDateTime,
+    end: NaiveDateTime,
+    entry_type: EventType,
+    detailed_entry_type: String,
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "lowercase")]
+enum EventType {
+    Lecture,
+    Exercise,
+    Exam,
+    Barred,
+    Other,
+}
+impl EventType {
+    fn from(xml_event: &XMLEvent) -> (Self, String) {
+        // only used for the lecture type
+        let course_type_name = xml_event
+            .course_type_name
+            .clone()
+            .unwrap_or_else(|| "Course type is unknown".to_string());
+        match xml_event.single_event_type_id.as_str() {
+            "SPERRE" => return (EventType::Barred, "".to_string()),
+            "PT" => return (EventType::Exam, "".to_string()),
+            "P" => return (EventType::Lecture, course_type_name), // Prüfung (geplant) is sometimes used for lectures
+            _ => {}
+        }
+        match xml_event.event_type_id.as_str() {
+            "LV" => (EventType::Lecture, course_type_name),
+            "PT" => (EventType::Exam, "".to_string()),
+            "EX" => (EventType::Exercise, "".to_string()),
+            _ => match &xml_event.event_type_name {
+                Some(event_type_name) => (
+                    EventType::Other,
+                    format!("{}: {}", xml_event.single_event_type_name, event_type_name),
+                ),
+                None => (EventType::Other, xml_event.single_event_type_name.clone()),
+            },
+        }
+    }
+}
+
+impl From<XMLEvent> for Event {
+    fn from(xml_event: XMLEvent) -> Self {
+        let (entry_type, detailed_entry_type) = EventType::from(&xml_event);
+        let title = xml_event.event_title;
+        Self {
+            title,
+            start: xml_event.dtstart,
+            end: xml_event.dtend,
+            entry_type,
+            detailed_entry_type,
+        }
+    }
+}

calendar/src/main.rs

@@ -1,9 +1,11 @@
+mod calendar;
 mod schema;
 mod scraping;
 mod utils;
 
 use actix_cors::Cors;
 use actix_web::{get, middleware, web, App, HttpResponse, HttpServer};
+use tokio::sync::Mutex;
 
 const MAX_JSON_PAYLOAD: usize = 1024 * 1024; // 1 MB
 
@@ -31,8 +33,10 @@ async fn health_handler() -> HttpResponse {
 async fn main() -> std::io::Result<()> {
     env_logger::init_from_env(env_logger::Env::default().default_filter_or("info"));
 
+    let last_sync = web::Data::new(Mutex::new(None));
+    let cloned_last_sync = last_sync.clone();
     actix_rt::spawn(async move {
-        scraping::continous_scraping::start_scraping().await;
+        scraping::continous_scraping::start_scraping(cloned_last_sync).await;
     });
     HttpServer::new(move || {
         let cors = Cors::default()
@@ -43,11 +47,16 @@ async fn main() -> std::io::Result<()> {
 
         App::new()
             .wrap(cors)
-            .wrap(middleware::Logger::default())
+            .wrap(middleware::Logger::default().exclude("/api/calendar/health"))
             .wrap(middleware::Compress::default())
             .app_data(web::JsonConfig::default().limit(MAX_JSON_PAYLOAD))
             .service(source_code_handler)
             .service(health_handler)
+            .service(
+                web::scope("/api/calendar")
+                    .configure(calendar::configure)
+                    .app_data(last_sync.clone()),
+            )
     })
     .bind(std::env::var("BIND_ADDRESS").unwrap_or_else(|_| "0.0.0.0:8060".to_string()))?
     .run()

calendar/src/scraping/continous_scraping.rs

@@ -2,22 +2,28 @@ use crate::scraping::task::ScrapeRoomTask;
 use crate::scraping::tumonline_calendar::{Strategy, XMLEvents};
 use crate::utils;
 use crate::utils::statistics::Statistic;
+use actix_web::web::Data;
 use awc::{Client, Connector};
-use chrono::{NaiveDate, Utc};
+use chrono::{NaiveDate, NaiveDateTime, Utc};
 use diesel::prelude::*;
 use futures::future::join_all;
 use log::{error, info, warn};
 use std::time::{Duration, Instant};
+use tokio::sync::Mutex;
 use tokio::time::sleep;
 
 const SECONDS_PER_DAY: u64 = 60 * 60 * 24;
-pub async fn start_scraping() {
+pub async fn start_scraping(last_sync: Data<Mutex<Option<NaiveDateTime>>>) {
     let mut interval = actix_rt::time::interval(Duration::from_secs(SECONDS_PER_DAY)); //24h
     loop {
         interval.tick().await;
         delete_scraped_results();
         scrape_to_db(chrono::Duration::days(30 * 4)).await;
         promote_scraped_results_to_prod();
+        {
+            let mut last_scrape = last_sync.lock().await;
+            *last_scrape = Some(Utc::now().naive_utc());
+        }
     }
 }
 

calendar/src/scraping/mod.rs

@@ -1,3 +1,3 @@
 pub mod continous_scraping;
 mod task;
-mod tumonline_calendar;
+pub mod tumonline_calendar;

deployment/k3s/templates/ingress/ingress.yaml

@@ -29,6 +29,12 @@ spec:
       services:
         - name: calendar-svc
           port: 8060
+    - kind: Rule
+      match: Host(`{{ .Values.url }}`) && PathPrefix(`/api/calendar/`)
+      priority: 13
+      services:
+        - name: calendar-svc
+          port: 8082
     - kind: Rule
       match: Host(`{{ .Values.url }}`) && PathPrefix(`/api/`)
       priority: 12

deployment/k3s/templates/service.yaml

@@ -57,6 +57,24 @@ spec:
 ---
 apiVersion: v1
 kind: Service
+metadata:
+  name: calendar-svc
+  labels:
+    app: navigatum
+    deployment: calendar
+  namespace: {{ .Values.namespace }}
+spec:
+  type: ClusterIP
+  selector:
+    app: navigatum
+    deployment: calendar
+  ports:
+    - name: calendar
+      port: 8082
+      targetPort: 8082
+---
+apiVersion: v1
+kind: Service
 metadata:
   name: maps-svc
   labels: