server.js

/*
BIM360 Zip extractor and file transfer for BIM360 and large zip files

Transfer only the minimal set of data required, by taking advantage of the zip structure and GET RANGE network requests:
https://users.cs.jmu.edu/buchhofp/forensics/formats/pkzip.html

*/
const fetch = require('node-fetch');
const StreamZip = require('node-stream-zip');
const fs = require('fs');
const path = require('path');
const fastify = require('fastify')({ logger: true })

let ze = null, bm = null;

fastify.register(require('fastify-static'), {
  root: path.join(__dirname, 'ui'),
  prefix: '/', // optional: default '/'
})

fastify.get('/dir', async (request, reply) => {
    const dirlist = fs.readdirSync('.', {withFileTypes: true})
        .filter(item => !item.isDirectory())
        .map(item => item.name);
    return dirlist;
});

function setCORS(reply) {
    reply.header("Access-Control-Allow-Origin", "*");
    reply.header("Access-Control-Allow-Methods", "POST");    
}

// INPUT: projectID, folderID, AccessToken
// OUTPUT: list of BIM360 files in the folder
fastify.get('/bim/list', async (request, reply) => {
    if (!request.query.project) return "INPUT: project, folder, token";
    setCORS(reply);
    bm = new BIM360utils(request.query.project, request.query.folder, request.query.token);
    const res = await bm.getFolderContents();
    return res;
});

// INPUT: zipURL, size (file-size)
// OUTPUT: list of zip file contents
fastify.get('/listZipContents', async (request, reply) => {
    if (!request.query.zipURL) return "INPUT: zipURL, size (file-size)";
    setCORS(reply);
    try {
        ze = new netZipExtract(request.query.zipURL, request.query.size, bm.token);
        const contents = await ze.getContents();
        return contents;
    } catch(err) { return err }
})

// INPUT: filename
// OUTPUT: status of result (timeout after 30seconds)
fastify.get('/transfer', async (request, reply) => {
    if (!ze && !bm) return {status:`not-ready.  Use 'listcontents' first`};
    setCORS(reply);
    if (!request.query.filename) return "INPUT: filename";
    if (request.query.destFolder) bm.folder = request.query.destFolder;
    if (request.query.destProject) bm.project = request.query.destProject;

    try {
        const filename = request.query.filename;
        const resObj = await bm.createEmptyFile(filename);
        const status = await ze.extractFile(filename, resObj.destURL);
        let ress = null;
        if (request.query.lineage)
          ress = await bm.bumpVersion(resObj, request.query.lineage);
        else
          ress = await bm.createVersion(resObj);
    } catch(err) {
        return {status: err};
    }
})

// INPUT: filename
// OUTPUT: status of job
fastify.get('/status', async (request, reply) => {
    if (!request.query.filename) return "INPUT: filename";
    setCORS(reply);
    const id = request.query.filename;
    if (!id) return;
    return (id) ?  `{ "${id}" : "${ze.session[id]}" }` : "missing filename= parameter"
})

fastify.listen(process.env.PORT || 3000, "0.0.0.0", (err, address) => {
    if (err) throw err
    fastify.log.info(`server listening on ${address}`)
})


/*
fastify.get('/bim/empty', async (request, reply) => {
    const res = await bm.createEmptyFile(request.query.filename);
    return res;
})

fastify.get('/bim/upload', async (request, reply) => {
    const res = await bm.createVersion();
    return res;
})

fastify.get('/extract2', async (request, reply) => {
    try {
        if (!ze) return {status:`not-ready.  Use 'listcontents' first`};
        const status = await ze.extractFile(request.query.outfile, request.query.destURL);
        return {status: status};
    } catch(err) {
        return {status: err};
    }
})


    async _createTempZip2(offset, size) {
        const tmpfile = fs.openSync(this.tmpFn, 'w');
        const chunksize = 16 * 1024; // only need 16k bytes of data
        await this._fetchWrite(tmpfile, 0, chunksize); // fetch/write header            
        await this._fetchWrite(tmpfile, this.fileLength - chunksize, chunksize); // fetch/write footer
        fs.closeSync(tmpfile);        
        if (offset && size) {
          // Write bytes to file
          const zipHeaderOffset = 128;
          const res = await fetch( this.URL, { headers: {
            'range': `bytes=${offset}-${offset+size+zipHeaderOffset}`,
            'Authorization': `Bearer ${this.token}`
          }});
          const dest = fs.createWriteStream(this.tmpFn, {flags:'a+', start:offset, highWaterMark: 2 * 1024 * 1024});
          await new Promise((resolve) => {
              res.body.pipe(dest);
              dest.on("finish", () => resolve());
          })
        }
    }

*/

String.prototype.hashCode = function() {
  var hash = 0;
  if (this.length == 0) {
      return hash;
  }
  for (var i = 0; i < this.length; i++) {
      var char = this.charCodeAt(i);
      hash = ((hash<<5)-hash)+char;
      hash = hash & hash; // Convert to 32bit integer
  }
  return hash;
}

//
// Main
//
class netZipExtract {
    constructor(URN, fileLength, token) {
        this.URL = `${URN}`;
        this.token = token;
        this.fileLength = fileLength;
        this.tmpFn = `tmp-${URN.hashCode()}.zip`;
        this.session = [];
    }

    //
    // fetch a chunk of bytes from BIM360 and write to 'temp' file on fs
    //
    async _fetchWrite( fd, offset, length ) { 
        const res = await fetch( this.URL, { headers: {
            'range': `bytes=${offset}-${offset+length}`,
            'Authorization': `Bearer ${this.token}`
        }});
        if (res.status != 206) 
            throw(`error:${res.statusText}, bytes=${offset}-${offset+length}`)
        const buff = await res.buffer();
        fs.writeSync(fd, buff, 0, buff.length, offset);
        return res.status;
    }

    async _createTempZip(offset, size) {
        const tmpfile = fs.openSync(this.tmpFn, 'w');
        const chunksize = 4 * 1024; // only need 16k bytes of data
        await this._fetchWrite(tmpfile, 0, chunksize); // fetch/write header            
        await this._fetchWrite(tmpfile, this.fileLength - chunksize, chunksize); // fetch/write footer
        const zipHeaderOffset = 128;
        if (size)
            await this._fetchWrite(tmpfile, offset, size + zipHeaderOffset); // fetch/write our filename within the zip
        fs.closeSync(tmpfile);        
    }

    //
    // get directory-list inside zip (that's hosted on bim360)
    //
    async getContents() { return new Promise(async resolve => {
        this._log(`fetch/extract Contents: ${this.URL} size ${this.fileLength}...`)
        try {
            await this._createTempZip();
            // now, extract content directory
            this.zip = new StreamZip({ file: this.tmpFn, storeEntries: true });
        }
        catch(err) {resolve(err);return}
        this.zip.on('error', (err) => { resolve(`error:${err}`) });
        this.zip.on('ready', () => { 
            this.entries = this.zip.entries();
            this.zip.close();
            resolve(this.entries);
        });
    })};

    //
    // extract a file from inside the zip file, then post it to bim360 destURL
    //
    async extractFile( filename, destURL ) { return new Promise(async resolve => {
        // get filename's offset and byte-length, located inside the zip file
        if (!this.entries) return;
        const offset = this.entries[filename].offset;
        const size = this.entries[filename].compressedSize;
        this.filename = filename;

        // now, fetch the exact bytes from bim360, and write to our temp file
        const MBytes = Math.round(size / 100000) / 10;
        this._log(`(downloading ${MBytes} MB) ${filename} , zip offset: ${offset}`)
        await this._createTempZip(offset, size);

        // now, use StreamZip to do it's magic.
        this._log(`Extracting ${filename} from ${this.tmpFn}...`)
        this.zip = new StreamZip({ file: this.tmpFn, storeEntries: true });
        this.zip.on('error', err => { throw(`error:${err}`) });
        this.zip.on('ready', async () => { 
            this.entries = this.zip.entries();

            this.zip.extract( filename, filename, async err => {
            //if (err) throw(`Zip-Extract error: ${err}`);

            this._log(`Uploading (0 KB) ${filename} to ${destURL}...`)

            // upload file to forge signedURL
            let bytes = size;
            const stream = fs.createReadStream(filename,{highWaterMark: 1024 * 1024 });
            stream.on('data', b => { 
                this._log(`Upload progress ${96-Math.round((bytes/size)*96)}%`)
                bytes-=b.length;
            });
            const rs2 = await fetch(destURL, { 
                method: 'PUT',
                headers: { Authorization: `Bearer ${this.token}` },
                body: stream });
            const res2 = await rs2.json();
            console.log(res2);
            // this.zip.close();  // don't close zip before upload complete. this seems buggy for slow networks
            this._log(`Upload complete: ${filename} to ${destURL}.${JSON.stringify(res2)}`)
            resolve(`Upload complete: ${filename} to ${destURL}. ${res2}`)
            });
        });
    })}

    _log(m) {
        console.log(m);
        if (this.filename) this.session[this.filename] = m;
    }
}


class BIM360utils {
    constructor(project, folder, token) {
        this.folder = folder;
        this.project = project;
        this.token = token;
        this.filename = "";
        this.displayName = "";
        this.objid = "";
    }

    async getFolderContents() {
        const res = await fetch( `https://developer.api.autodesk.com/data/v1/projects/${this.project}/folders/urn:adsk.wipprod:fs.folder:${this.folder}/contents`, 
        { headers: { Authorization: `Bearer ${this.token}` }});
        const jres = await res.json();
        if (!jres.included) return jres;
        let lista = jres.included.map(i => { return ({ 
            filename: i.attributes.displayName, 
            size:i.attributes.storageSize,
            lastModifiedTime: i.attributes.lastModifiedTime,
            lineage: i.relationships.item.data.id,
            version: i.attributes.versionNumber,
            datetime: (new Date(i.attributes.lastModifiedTime)).toLocaleDateString('en-GB', { day: '2-digit', month: 'short', year: 'numeric' }).replace(/ /g, '-'),
            url: `https://developer.api.autodesk.com/oss/v2/buckets/wip.dm.prod/objects/${i.relationships.storage.data.id.split('/')[1]}`
        })});
        lista.sort((a,b)=>{return (b.lastModifiedTime < a.lastModifiedTime) ? -1:1});
        return lista;
    }

    

    // token2 (optional) is a second ACCESS_TOKEN of a 2nd BIM360 HUB.
    // Use this to save the resulting output file to a secondary BIM360 Hub
    async createEmptyFile(filename) {
        //const id = `${Math.random()}`.slice(2,5);
        this.filename = filename;//`_${filename.slice(0,-4)}_${id}${filename.slice(-4)}`;
        const res = await fetch( `https://developer.api.autodesk.com/data/v1/projects/${this.project}/storage`, 
        {
            method: 'POST', 
            headers: {
                'Content-Type': 'application/vnd.api+json',
                Accept : 'application/vnd.api+json',
                Authorization: `Bearer ${this.token}`
            },
            body: `{
                "jsonapi": { "version": "1.0" },
                "data": {
                  "type": "objects",
                  "attributes": {
                    "name": "${this.filename}"
                  },
                  "relationships": {
                    "target": {
                      "data": { "type": "folders", "id": "urn:adsk.wipprod:fs.folder:${this.folder}" }
                    }
                  }
                }
          }`
        });
        const obj = await res.json();
        this.objid = obj.data.id.split("/")[1];
        return {
          objid: this.objid,
          filename: this.filename,
          destURL: `https://developer.api.autodesk.com/oss/v2/buckets/wip.dm.prod/objects/${this.objid}`
        }
    }

    async createVersion( ob ) {
        const res = await fetch( `https://developer.api.autodesk.com/data/v1/projects/${this.project}/items`, 
        {
            method: 'POST', 
            headers: {
                'Content-Type': 'application/vnd.api+json',
                Accept : 'application/vnd.api+json',
                Authorization: `Bearer ${this.token}`
            },
            body: `{
                "jsonapi": { "version": "1.0" },
                "data": {
                  "type": "items",
                  "attributes": {
                    "displayName": "${ob.filename}",
                    "extension": {
                      "type": "items:autodesk.bim360:File",
                      "version": "1.0"
                    }
                  },
                  "relationships": {
                    "tip": {
                      "data": {
                        "type": "versions", "id": "1"
                      }
                    },
                    "parent": {
                      "data": {
                        "type": "folders",
                        "id": "urn:adsk.wipprod:fs.folder:${this.folder}"
                      }
                    }
                  }
                },
                "included": [
                  {
                    "type": "versions",
                    "id": "1",
                    "attributes": {
                      "name": "${ob.filename}",
                      "extension": {
                        "type": "versions:autodesk.bim360:File",
                        "version": "1.0"
                      }
                    },
                    "relationships": {
                      "storage": {
                        "data": {
                          "type": "objects",
                          "id": "urn:adsk.objects:os.object:wip.dm.prod/${ob.objid}"
                        }
                      }
                    }
                  }
                ]
              }`
        });
        const obj = await res.json();
        return obj;
    }

    async getTip( ob, lineageId ) {
      
    }

    async bumpVersion( ob, lineageId ) {
      const res = await fetch( `https://developer.api.autodesk.com/data/v1/projects/${this.project}/versions`, 
      {
          method: 'POST', 
          headers: {
              'Content-Type': 'application/vnd.api+json',
              Accept : 'application/vnd.api+json',
              Authorization: `Bearer ${this.token}`
          },
          body: `{
            "jsonapi": { "version": "1.0" },
            "data": {
               "type": "versions",
               "attributes": {
                  "name": "${ob.filename}",
                  "extension": { "type": "versions:autodesk.bim360:File", "version": "1.0"}
               },
               "relationships": {
                  "item": { "data": { "type": "items", "id": "${lineageId}" } },
                  "storage": { "data": { "type": "objects", "id": "urn:adsk.objects:os.object:wip.dm.prod/${ob.objid}" } }
               }
            }
         }`
      });
      const obj = await res.json();
      return obj;
  }
}