not for merging: playing with encoding DF as arrow table

[kylebrandt]

(function () {
  if (typeof window.R === 'undefined') {
    var s = document.createElement('script');
    s.setAttribute('src', 'https://cdn.jsdelivr.net/npm/apache-arrow@0.14.1/Arrow.es5.min.js');
    document.body.appendChild(s);
  }
}());

foo = JSON.parse(`{"results":{"":{"refId":"","meta":{"GC":"QVJST1cxAAB8AAAAEAAAAAAACgAMAAoACQAEAAoAAAAQAAAAAAEDAAgACAAAAAQACAAAAAQAAAABAAAAFAAAABAAFAAQAAAADwAIAAAABAAQAAAAEAAAABgAAAAAAAADGAAAAAAAAAAAAAYACAAGAAYAAAAAAAIABgAAAFNjYWxhcgAAAAAAAIwAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAAAIAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAA4AAAAAQAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAhAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAAIgAAAAAAAAAkAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAgACAAAAAQACAAAAAQAAAABAAAAFAAAABAAFAAQAAAADwAIAAAABAAQAAAAEAAAABgAAAAAAAADGAAAAAAAAAAAAAYACAAGAAYAAAAAAAIABgAAAFNjYWxhcgAAqAAAAEFSUk9XMQ=="},"series":[],"tables":null,"frames":null}}}`);

b64 = atob((foo["results"][""]["meta"]["GC"]));

b64AsArray = Uint8Array.from(b64, function(c) {return c.charCodeAt(0);});

t = Arrow.Table.from(b64AsArray);

t.getColumn('Scalar').toArray();

[kylebrandt]

Return is now a slice of encoded tables, and has metadata in it:

foo = JSON.parse(`{"results":{"":{"refId":"","meta":["QVJST1cxAADsAQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAfAAAAAMAAABMAAAAKAAAAAQAAACg/v//CAAAAAwAAAACAAAAR0IAAAUAAAByZWZJZAAAAMD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAA4P7//wgAAAAUAAAACQAAAHRlc3Q9dGVzdAAAAAYAAABsYWJlbHMAAAIAAACsAAAABAAAAG7///8UAAAAaAAAAHAAAAAAAAMBcAAAAAIAAAAsAAAABAAAADj///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAFz///8IAAAAFAAAAAkAAABHQi1zZXJpZXMAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAACAAkAAABHQi1zZXJpZXMAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAcAAAAAAABQFsAAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAVGltZQAAAAAAAAAAzAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAMgAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAGgAAAAGAAAAAAAAAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAHgAAAAAAAAAmAAAAAAAAAAAAAAAAAAAAJgAAAAAAAAAMAAAAAAAAAAAAAAAAgAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATAAAAJgAAADkAAABMAAAAXwAAAHIAAAAAAAAAMTU2ODk4NjgwMDAwMDAwMDAwMDE1Njg5ODY4NjAwMDAwMDAwMDAxNTY4OTg2OTIwMDAwMDAwMDAwMTU2ODk4Njk4MDAwMDAwMDAwMDE1Njg5ODcwNDAwMDAwMDAwMDAxNTY4OTg3MTAwMDAwMDAwMDAwAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/EAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAAPgBAAAAAAAA0AAAAAAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAfAAAAAMAAABMAAAAKAAAAAQAAACg/v//CAAAAAwAAAACAAAAR0IAAAUAAAByZWZJZAAAAMD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAA4P7//wgAAAAUAAAACQAAAHRlc3Q9dGVzdAAAAAYAAABsYWJlbHMAAAIAAACsAAAABAAAAG7///8UAAAAaAAAAHAAAAAAAAMBcAAAAAIAAAAsAAAABAAAADj///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAFz///8IAAAAFAAAAAkAAABHQi1zZXJpZXMAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAACAAkAAABHQi1zZXJpZXMAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAcAAAAAAABQFsAAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAVGltZQAAAAAYAgAAQVJST1cx","QVJST1cxAADsAQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAfAAAAAMAAABMAAAAKAAAAAQAAACg/v//CAAAAAwAAAACAAAAR0MAAAUAAAByZWZJZAAAAMD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAA4P7//wgAAAAUAAAACQAAAHRlc3Q9dGVzdAAAAAYAAABsYWJlbHMAAAIAAACsAAAABAAAAG7///8UAAAAaAAAAHAAAAAAAAMBcAAAAAIAAAAsAAAABAAAADj///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAFz///8IAAAAFAAAAAkAAAB0ZXN0PXRlc3QAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAACAAkAAAB0ZXN0PXRlc3QAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAcAAAAAAABQFsAAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAVGltZQAAAAAAAAAAzAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAMgAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAGgAAAAGAAAAAAAAAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAHgAAAAAAAAAmAAAAAAAAAAAAAAAAAAAAJgAAAAAAAAAMAAAAAAAAAAAAAAAAgAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATAAAAJgAAADkAAABMAAAAXwAAAHIAAAAAAAAAMTU2ODk4NjgwMDAwMDAwMDAwMDE1Njg5ODY4NjAwMDAwMDAwMDAxNTY4OTg2OTIwMDAwMDAwMDAwMTU2ODk4Njk4MDAwMDAwMDAwMDE1Njg5ODcwNDAwMDAwMDAwMDAxNTY4OTg3MTAwMDAwMDAwMDAwAAAAAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAABBAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAAPgBAAAAAAAA0AAAAAAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAfAAAAAMAAABMAAAAKAAAAAQAAACg/v//CAAAAAwAAAACAAAAR0MAAAUAAAByZWZJZAAAAMD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAA4P7//wgAAAAUAAAACQAAAHRlc3Q9dGVzdAAAAAYAAABsYWJlbHMAAAIAAACsAAAABAAAAG7///8UAAAAaAAAAHAAAAAAAAMBcAAAAAIAAAAsAAAABAAAADj///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAFz///8IAAAAFAAAAAkAAAB0ZXN0PXRlc3QAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAACAAkAAAB0ZXN0PXRlc3QAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAcAAAAAAABQFsAAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAVGltZQAAAAAYAgAAQVJST1cx"],"series":[],"tables":null,"frames":null}}}`);

tables = [];

foo["results"][""]["meta"].forEach(function(encTable) {
    b64 = atob(encTable);
    b64AsArray = Uint8Array.from(b64, function(c) {return c.charCodeAt(0);});
    tables.push(Arrow.Table.from(b64AsArray))
});

tables.forEach(function(table) {
    console.log(table.schema.metadata);
    table.schema.fields.forEach(function(field) {
        console.log(field.metadata);
        console.log(table.getColumn(field.name).toArray());
    });
});

[ryantxu]

@kylebrandt -- I just pushed changes so this can convert bytes to DataFrame on the frontend. It all seems to work ok :)

Since frontend only deals with dates as ms... I added a hack to convert everything with name "Time" -- this is obviously something we need to sync up on representation

[ryantxu]

NOTE: make sure to clear node_modules and yarn install for this to work. I had to go back to the canary version to get arrow column to directly implement frontend Vector