Merge pull request #122 from HannesOberreiter/beta

Beta
HannesOberreiter · Sep 7, 2023 · d33d346 · d33d346
2 parents d9e1e3c + 75c8a0c
commit d33d346
Show file tree

Hide file tree

Showing 4 changed files with 158 additions and 27 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -61,6 +61,7 @@
     "openai": "^4.1.0",
     "pino": "^8.14.1",
     "pino-abstract-transport": "^1.0.0",
+    "proj4": "^2.9.0",
     "query-string": "^8.1.0",
     "rotating-file-stream": "^3.0.4",
     "stripe": "^13.2.0",
@@ -72,6 +73,7 @@
     "@types/node": "^20.3.1",
     "@types/node-schedule": "^2.1.0",
     "@types/nodemailer": "^6.4.7",
+    "@types/proj4": "^2.5.2",
     "@typescript-eslint/eslint-plugin": "^6.3.0",
     "@typescript-eslint/parser": "^6.3.0",
     "chai": "^4.3.6",

diff --git a/src/api/utils/velutina.util.ts b/src/api/utils/velutina.util.ts
@@ -1,10 +1,20 @@
 import { Observation } from '../models/observation.model.js';
+import proj4 from 'proj4';
 
 export async function fetchObservations() {
   const inat = fetchInat();
   const patriNat = fetchPatriNat();
-  const [inatResult, patriNatResult] = await Promise.all([inat, patriNat]);
-  return { iNaturalist: inatResult, patriNat: patriNatResult };
+  const artenfinderNet = fetchArtenfinderNet();
+  const [inatResult, patriNatResult, artenfinderNetResult] = await Promise.all([
+    inat,
+    patriNat,
+    artenfinderNet,
+  ]);
+  return {
+    iNaturalist: inatResult,
+    patriNat: patriNatResult,
+    artenfinderNet: artenfinderNetResult,
+  };
 }
 
 type ObservationPatriNat = {
@@ -16,14 +26,30 @@ type ObservationPatriNat = {
   commentaires_validation: string;
 };
 
-async function fetchPatriNat() {
+export async function fetchPatriNat() {
   // Example Input: Rome07072015033744
   function dateExtract(t: string) {
     const e = t.substring(t.length - 14),
       day = e.substring(0, 2),
       month = e.substring(2, 4),
-      year = e.substring(4, 8);
-    return new Date(year + '-' + month + '-' + day);
+      year = e.substring(4, 8),
+      hour = e.substring(8, 10),
+      minute = e.substring(10, 12),
+      second = e.substring(12, 14);
+    return new Date(
+      year +
+        '-' +
+        month +
+        '-' +
+        day +
+        'T' +
+        hour +
+        ':' +
+        minute +
+        ':' +
+        second +
+        'Z',
+    );
   }
   const result = await fetch(
     'https://frelonasiatique.mnhn.fr/wp-content/plugins/spn_csv_exporter/widgetVisualisateur/visuaMapDisplayController.php',
@@ -43,12 +69,15 @@ async function fetchPatriNat() {
   const oldRecords = await Observation.query()
     .select('external_uuid')
     .where('external_service', 'PatriNat');
+  const searchArray = oldRecords.map((o) => o.external_uuid);
 
-  for (const observation of data.records as ObservationPatriNat[]) {
+  for (let i = 0; i < data.records.length; i++) {
+    const observation: ObservationPatriNat = data.records[i];
     if (observation.validation != 'Validé') continue;
-    const date = dateExtract(observation.nom_station);
-    if (oldRecords.find((o) => o.external_uuid === observation.nom_station))
+    if (searchArray.includes(observation.nom_station)) {
       continue;
+    }
+    const date = dateExtract(observation.nom_station);
     observation['uri'] = 'https://frelonasiatique.mnhn.fr/visualisateur';
     observations.push({
       external_uuid: observation.nom_station,
@@ -67,35 +96,41 @@ async function fetchPatriNat() {
 
   await Observation.query().insertGraph(observations);
 
-  //await Observation.query().insertGraph(observations);
   return { newObservations: observations.length };
 }
 
-async function fetchInat() {
+export async function fetchInat(monthsBefore = 6) {
   const fields =
     '(id:!t,uri:!t,quality_grade:!t,time_observed_at:!t,location:!t,taxon:(id:!t))';
 
-  const latestId = await Observation.query()
-    .max('external_id as external_id')
-    .where('external_service', 'iNaturalist')
-    .first();
-
-  let idAbove = +(latestId.external_id ?? 1);
-
+  let idAbove = 1;
   let newObservations = 0;
+  let createdAtD1: Date | string = new Date();
+  createdAtD1.setMonth(createdAtD1.getMonth() - monthsBefore);
+  createdAtD1 = createdAtD1.toISOString().split('T')[0];
+  const createdAtD2 = new Date().toISOString().split('T')[0];
 
-  while (idAbove > 0) {
-    const url = `https://api.inaturalist.org/v2/observations?taxon_id=119019&verifiable=true&quality_grade=research&order=asc&order_by=id&per_page=200&fields=${fields}&id_above=${idAbove}`;
+  const oldRecords = await Observation.query()
+    .select('external_id')
+    .where('external_service', 'iNaturalist');
+  const searchArray = oldRecords.map((o) => o.external_id);
 
+  while (idAbove > 0) {
+    const url = `https://api.inaturalist.org/v2/observations?taxon_id=119019&verifiable=true&quality_grade=research&order=asc&order_by=id&per_page=200&fields=${fields}&created_d1=${createdAtD1}&created_d2=${createdAtD2}&id_above=${idAbove}`;
     const response = await fetch(url);
-    const data = await response.json();
+    const res = await response.json();
     let observations = [];
-    if (data.results.length === 0) break;
-    idAbove = data.results[data.results.length - 1].id;
-    newObservations += data.results.length;
-    for (const observation of data.results) {
+    if (res.results.length === 0) break;
+    idAbove = res.results[res.results.length - 1].id;
+    // newObservations += data.results.length;
+    for (let i = 0; i < res.results.length; i++) {
+      const observation = res.results[i];
       if (!observation.time_observed_at) continue;
       if (!observation.location) continue;
+      if (searchArray.includes(observation.id)) {
+        continue;
+      }
+      newObservations++;
       const data = { ...observation };
       delete data.location;
       observations.push({
@@ -114,3 +149,72 @@ async function fetchInat() {
   }
   return { newObservations: newObservations };
 }
+
+export async function fetchArtenfinderNet() {
+  const oldRecords = await Observation.query()
+    .select('external_id')
+    .where('external_service', 'Artenfinder.net');
+  const searchArray = oldRecords.map((o) => o.external_id);
+
+  let url =
+    'https://www.artenfinder.net/api/v2/sichtbeobachtungen?titel_wissenschaftlich=vespa%velutina';
+
+  let newObservations = 0;
+
+  // Coordination system is ETRS89/UTM 32N (EPSG:25832), but we need WGS84 (EPSG:4326)
+  proj4.defs(
+    'EPSG:25832',
+    '+proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs +type=crs',
+  );
+
+  while (url) {
+    const response = await fetch(url);
+    const res = await response.json();
+    if (res.length === 0) break;
+    if (res?.result?.length === 0) break;
+    url = res.next || undefined;
+    let observations = [];
+    for (let i = 0; i < res.result.length; i++) {
+      const observation = res.result[i];
+      if (!observation.lat || !observation.lon) continue;
+      if (searchArray.includes(observation.id)) continue;
+
+      newObservations++;
+      const data = { ...observation };
+      delete data.lat;
+      delete data.lon;
+
+      data.uri = `https://www.artenfinder.net/artenfinder-pwa/#/beobachtung/${observation.id}`;
+
+      const observed_at = observation.datum.split('.');
+
+      const coordinates = proj4('EPSG:25832', 'EPSG:4326', [
+        Number(observation.lon),
+        Number(observation.lat),
+      ]);
+
+      observations.push({
+        external_id: Number(observation.id),
+        external_uuid: observation.guid,
+        external_service: 'Artenfinder.net',
+        observed_at: new Date(
+          observed_at[2],
+          observed_at[1] - 1,
+          observed_at[0],
+        ).toISOString(),
+        location: {
+          lat: coordinates[1],
+          lng: coordinates[0],
+        },
+        taxa: 'Vespa velutina',
+        data: data,
+      });
+    }
+
+    await Observation.query().insertGraph(observations);
+  }
+
+  if (newObservations === 0) return { newObservations: 0 };
+
+  return { newObservations: newObservations };
+}
diff --git a/src/config/environment.config.ts b/src/config/environment.config.ts
@@ -97,9 +97,7 @@ const totalLimit = {
   scale: parseInt(process.env.TOTAL_LIMIT_SCALE),
 };
 
-const cronjobTimer = process.env.CRONJOB
-  ? process.env.CRONJOB
-  : '0 11 ./1 * *.js';
+const cronjobTimer = process.env.CRONJOB ? process.env.CRONJOB : '0 11 */1 * *';
 
 const redisConfig = {
   host: process.env.REDIS_HOSTNAME,