Merge pull request #61 from bioimage-io/update_to_collection

Switch to new collection backend; MVP achieved

.gitignore

@@ -5,6 +5,7 @@ node_modules
 /package
 .env
 .env.*
+*.local
 __pycache__
 !.env.example
 vite.config.js.timestamp-*

functions/generate_name.js

@@ -40,19 +40,19 @@ function sample(array){
 
 function generate_name(starts, animals){
     const animal = sample(animals);
-    return {id: `${sample(starts)}-${animal[0]}`, icon: animal[1]};
+    return {id: `${sample(starts)}-${animal[0]}`, emoji: animal[1]};
 }
 
 
 export default async () => {
     console.log("Generating nickname from allowed lists...");
-    const [adjectives, animals_with_icons, taken_names] = await Promise.all([
+    const [adjectives, animals_with_emojis, taken_names] = await Promise.all([
         get_adjectives(),
         get_animals(),
         get_taken_names()]);
-    let name = generate_name(adjectives, animals_with_icons);
+    let name = generate_name(adjectives, animals_with_emojis);
     while(name in taken_names){
-        name = generate_name(adjectives, animals_with_icons);
+        name = generate_name(adjectives, animals_with_emojis);
     }
     console.log("Name generated:");
     console.log(name);

functions/get_chats.js

@@ -0,0 +1,6 @@
+export default async (event) => {
+    const data = await event.json();
+    const url = data.url;
+    return await fetch(url);
+}
+

functions/get_status.js

@@ -13,10 +13,6 @@ export default async (event) => {
     }catch(err){
         obj = {last_message: "Status file not present yet", messages:[]};
     }
-    console.log("Got status for");
-    console.log(url);
-    console.log("Answer:");
-    console.log(obj);
     const res = Response.json(obj);
     res.headers.set("Access-Control-Allow-Origin", "*");
     res.headers.append("Access-Control-Allow-Headers", "*");

functions/get_versions.ts

@@ -0,0 +1,23 @@
+export default async (event:  Request) => {
+    const data = await event.json();
+    const url = data.url;
+    let obj = {};
+
+    try{
+        const resp = await fetch(url);
+        try{
+            obj = await resp.json();
+        }catch(err){
+            obj = {'error': "Versions file not present", 'err': err};
+        }
+    }catch(err){
+        obj = {'error': "Versions file not present", 'err': err};
+    }
+    console.log("Got versions for");
+    console.log(url);
+    const res = Response.json(obj);
+    res.headers.set("Access-Control-Allow-Origin", "*");
+    res.headers.append("Access-Control-Allow-Headers", "*");
+    res.headers.append("Access-Control-Allow-Methods", "*");
+    return res;
+}

functions/notify_ci.js

@@ -38,7 +38,7 @@ export default async (event, context) => {
         body: JSON.stringify({
             'ref': GITHUB_BRANCH,
             'inputs': {
-                'resource_path': data.resource_path,
+                'resource_id': data.resource_path,
                 'package_url': data.package_url,
             }
         })
@@ -51,7 +51,8 @@ export default async (event, context) => {
             // According to API docs, just expect a 204
             resp_obj = { 'status': resp.status };
         } else {
-            console.error("Bad response from CI: ${resp.status}");
+            console.error(`Bad response from CI: ${resp.status}`);
+            console.error(`Bad response from CI: ${resp.body}`);
             let text = "";
             try {
                 text = await resp.text()

functions/update_resource.ts

@@ -0,0 +1,136 @@
+import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import * as Minio from 'minio';
+
+const { S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY } = process.env;
+
+export default async (event, context) => {
+
+    const data = await event.json();
+    const root_folder = "sandbox.bioimage.io";
+    const bucket = "public-datasets";
+    const resource_id = data.resource_id;
+    const folder = data.resource_id; 
+    const update_type = data.type;
+    const filename = "chat.json";
+    console.log("Creating client...");
+    const key = `${root_folder}/${folder}/${filename}`;
+    const host = "uk1s3.embassy.ebi.ac.uk";
+    const hest_url = `https://${host}`;
+    //console.log("Using")
+    //console.log(S3_ACCESS_KEY_ID);
+    //console.log(S3_SECRET_ACCESS_KEY);
+    //console.log(`Using object path ${key}`);
+
+    if(true){
+    const minioClient = new Minio.Client({
+      endPoint: host,
+      useSSL: true,
+      accessKey: S3_ACCESS_KEY_ID,
+      secretKey: S3_SECRET_ACCESS_KEY,
+    })
+    console.log(key);
+
+    //const objectsStream = minioClient.listObjectsV2(bucket, '', true, '')
+    //objectsStream.on('data', function (obj) {
+        //console.log(obj)
+    //})
+    //objectsStream.on('error', function (e) {
+        //console.log(e)
+    //})
+    let obj;
+    let stream = await minioClient.getObject(bucket, key);
+    stream.setEncoding('utf8');
+    let data = '';
+    for await (const chunk of stream) {
+        data += chunk;
+    }
+    console.log(data);
+    //await minioClient.getObject(bucket, key, function (err, dataStream) {
+        //let data = [];
+        //if (err) {
+            //return console.log(err)
+        //}
+        //dataStream.on('data', function (chunk) {
+            //console.log(`Streaming...: ${String.fromCharCode(...chunk)}`);
+            //data.push(chunk); 
+        //})
+        //dataStream.on('end', function () {
+            //let str = String.fromCharCode(...data);
+            //console.log(`END - GOT: ${str}`);
+            //obj = JSON.parse(str);
+        //})
+        //dataStream.on('error', function (err) {
+            //console.log(err)
+        //})
+    //});
+
+    //let obj = await new Promise(async resolve => {
+        //let str = "";
+        //let dataStream = await minioClient.getObject(bucket, key);
+        //let obj;
+        //let data = [];
+        //console.log("Getting data...");
+        //dataStream.on('data', function (chunk) {
+            //console.log("Push...");
+            //data.push(chunk);
+        //})
+        //dataStream.on('end', function () {
+            //str = String.fromCharCode(...data);
+            //console.log(`END - GOT: ${str}`);
+            //obj = JSON.parse(str);
+            //resolve(obj);
+        //})
+        //dataStream.on('error', function (err) {
+            //console.log(err)
+        //})
+    //});
+
+    //await new Promise(resolve => dataStream.on("end", resolve));
+    console.log(obj);
+    console.log("Done");
+    const res = Response.json({ "message": "Success" });
+    res.headers.set("Access-Control-Allow-Origin", "*");
+    res.headers.append("Access-Control-Allow-Headers", "*");
+    res.headers.append("Access-Control-Allow-Methods", "*");
+    return res;
+    }else{
+    const client = new S3Client({
+        endpoint: host, // S3_ENDPOINT, // "uk1s3.embassy.ebi.ac.uk",
+        //region: 'eu-west-1',
+        region: 'us-east-1',
+        credentials: {
+            accessKeyId: S3_ACCESS_KEY_ID, // from env variables
+            secretAccessKey: S3_SECRET_ACCESS_KEY,
+        },
+        s3ForcePathStyle: true,
+    });
+
+    console.log(`Get chat from ${bucket} using ${key}`);
+    const get_command = new GetObjectCommand({
+        Bucket: bucket,
+        Key: key,
+    });
+    let resp_get = await client.send(get_command);
+    let chats = resp_get.json();
+    chats.chats.push({
+        timestamp: new Date().toUTCString(),
+        author: data.user,
+        text: data.text,
+    })
+
+    console.log("Put chat");
+    const put_command = new PutObjectCommand({
+        Bucket: bucket,
+        Key: key,
+        Body: JSON.stringify(chats),
+    });
+    const resp = await client.send(put_command);
+    console.log(resp);
+
+    const res = Response.json({ "message": "Success" });
+    res.headers.set("Access-Control-Allow-Origin", "*");
+    res.headers.append("Access-Control-Allow-Headers", "*");
+    res.headers.append("Access-Control-Allow-Methods", "*");
+    return res;
+    }
+}

functions/update_status.js

@@ -0,0 +1,41 @@
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+const { S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY } = process.env;
+
+export default async (event, context) => {
+
+
+    console.log(S3_ENDPOINT);
+    console.log(S3_ACCESS_KEY_ID);
+    console.log(S3_SECRET_ACCESS_KEY);
+    console.log(`Connecting to S3: ${S3_ENDPOINT}`);
+
+    const data = await event.json();
+    console.log(data);
+    const client = new S3Client({
+        endpoint: S3_ENDPOINT, // from env "http://localhost:9001"
+        region: 'eu-west-1',
+        credentials: {
+            accessKeyId: S3_ACCESS_KEY_ID, // from env variables
+            secretAccessKey: S3_SECRET_ACCESS_KEY,
+        },
+        s3ForcePathStyle: true,
+    });
+
+    const root_folder = "sandbox.bioimage.io";
+    const folder = "jm-test";
+    const filename = "ntl-functest.json";
+
+    const command = new PutObjectCommand({
+        Bucket: "public-datasets",
+        Key: `${root_folder}/${folder}/${filename}`,
+        Body: JSON.stringify({ "status": "This works too" }),
+    });
+    const resp = await client.send(command);
+    console.log(resp);
+
+    const res = Response.json({ "message": "Success" });
+    res.headers.set("Access-Control-Allow-Origin", "*");
+    res.headers.append("Access-Control-Allow-Headers", "*");
+    res.headers.append("Access-Control-Allow-Methods", "*");
+    return res;
+}

index.html

@@ -3,9 +3,10 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Uploader</title>
+    <title>BioimageIO</title>
   </head>
   <body id="app">
-    <script type="module" src="/src/main.js"></script>
+    <!--<div id="app"></div>-->
+    <script type="module" src="/src/main.ts"></script>
   </body>
 </html>

netlify.toml

@@ -1,5 +1,5 @@
 [dev] 
-  command = "npm run dev" 
+  command = "yarn run dev" 
   autoLaunch = false
   targetPort = 38283
   port = 9520