Data Manager

This module allows users to fetch data (JSON, Text, ArrayBuffer, blob) and
cache the result for future use. Additional pattern based URL requests can
be achieved. Images can also be retrieved as a blob and can be display using
a generated browser url.

A new instance is created with:

import DataManager from 'paraviewweb/src/IO/Core/DataManager';

dataManager = new DataManager({cacheSize : 1000000000}); // 1 GB by default

All listening is managed using monolog.js
and might not be fully described here, consult their full documentation
here.

fetchURL(url, type[, mimeType, topicForNotification])

Fetches a data object from a static URL. The set of possible
data type are: [ ‘json’, ‘text’, ‘blob’, ‘array’ ]

For the blob type, an extra argument is required which is the mimetype of
the blob.

var url = '/data/myJsonFile.json';
dataManager.on(url, function(data, envelope) {
    console.log("Got " + data.data.type + " object from " + data.requestedURL + " - last access time: " + data.ts);
    console.log(data.data);
});

dataManager.fetchURL(url, 'json');

Then you can optionally provide a topic name that you want your request to be notified on.

fetch(key, options)

Downloads a resource based on a previously registered pattern
with specific key-value pair replacement.

Here is a full example using that method:

function onJsonData(data, envelope) {

    var jsonObj = data.data;

    // Print additional cache meta data
    console.log(" - Last read time: " + data.ts);
    console.log(" - Data Type: " + data.type);
    console.log(" - Requested URL: " + data.requestedURL);

    // Access data from JSON object
    console.log(" - str: " + jsonObj.str);
    console.log(" - array: " + jsonObj.array);
    console.log(" - nestedObject.a: " + jsonObj.nestedObject.a);
}

dataManager.registerURL('jsonDataModel', '/data/{name}.json', 'json');
var subscription = dataManager.on('jsonDataModel', onJsonData);
dataManager.fetch('jsonDataModel', { name: 'info'});

// Then to stop listening: subscription.unsubscribe();

get(url[, freeCache])

Returns the downloaded data object if available or an undefined
object if unavailable.

The freeCache argument is optional and should be true if you want to remove
the given resource from the cache.

The standard returned object will look like the following:

{
    ts: 23423452, // Last access time in milliseconds.
    data: "str" | { json: 'data'} | Blob() | Uint8Array(), // Raw data depending of the fetch data type.
    type: 'text' | 'json' | 'blob' | 'arraybuffer',
    url: ...internal browser url to point to the data..., // This can be use to render images
    requestedURL: '/origin/requested/url'
}

free(url)

Removes the entry from the cache based on the requestedURL of a cache entry.

registerURL(key, urlPattern, type, mimeType)

Registers a pattern based URL to ease data fetching from it.

var pattern = '/data/{ds}/image_{idx}.png';
var key = 'image_ds'
dataManager.registerURL(key, pattern, 'blob', 'image/png');

dataManager.on(key, function(data, envelope) {
    console.log(
        "Got " + data.type + " object from " + data.requestedURL
        + " - last access time: " + data.ts
        + " - usable url: " + data.url);
});

dataManager.fetch(key, { idx: 0, ds: 'temperature' });
dataManager.fetch(key, { idx: 1, ds: 'temperature' });
dataManager.fetch(key, { idx: 2, ds: 'temperature' });
dataManager.fetch(key, { idx: 0, ds: 'pressure' });
dataManager.fetch(key, { idx: 1, ds: 'pressure' });
dataManager.fetch(key, { idx: 2, ds: 'pressure' });

unregisterURL(key)

Removes the pattern from the registry.

clear()

Empties the content of the cache.

on(event, listener)

Attaches a listener to a url or a pattern key.

Here is a list of possible listener functions:

function onJsonData(data, envelope) {
    var jsonObj = data.data;

    // Print additional cache meta data
    console.log(" - Last read time: " + data.ts);
    console.log(" - Data Type: " + data.type);
    console.log(" - Requested URL: " + data.requestedURL);

    // Access data from JSON object
    console.log(" - str: " + jsonObj.str);
    console.log(" - array: " + jsonObj.array);
    console.log(" - nestedObject.a: " + jsonObj.nestedObject.a);
}

function onTxtData(data, envelope) {
    // Print additional cache meta data
    console.log(" - Last read time: " + data.ts);
    console.log(" - Data Type: " + data.type);
    console.log(" - Requested URL: " + data.requestedURL);

    // Replace content inside your DOM
    var strHTML = data.data;
    $('.help').html(strHTML);
}

function onBlobData(data, envelope) {
    var blob = data.data;

    // Print additional cache meta data
    console.log(" - Last read time: " + data.ts);
    console.log(" - Data Type: " + data.type);
    console.log(" - Requested URL: " + data.requestedURL);

    // The URL let you provide a link to the blob
    console.log(" - Usable URL: " + data.url);
}

function onArrayData(data, envelope) {
    // Print additional cache meta data
    console.log(" - Last read time: " + data.ts);
    console.log(" - Data Type: " + data.type);
    console.log(" - Requested URL: " + data.requestedURL);

    // Replace content inside your DOM
    var Uint8ArrayObj = data.data;
    // [...]
}

function onImage(data, envelope) {
    var blob = data.data;

    // Print additional cache meta data
    console.log(" - Last read time: " + data.ts);
    console.log(" - Data Type: " + data.type);
    console.log(" - Requested URL: " + data.requestedURL);

    // The URL let you provide a link to the blob
    console.log(" - Usable URL: " + data.url);

    // Update the image in the DOM
    $('.image-to-refresh').attr('src', data.url);
}

getCacheSize()

Return the maximum size that the cache can contain before the gc() take any action.

setCacheSize( maxSizeInBytes )

Update the maximum cache size to adjust when the garbage collector should empty it.

getMemoryUsage()

Return the current memory usage of the cache data in Bytes.

Source

htmlRequest.js

import { toByteArray } from 'base64-js';

function getContent(url) {
  const el = document.querySelector(`.webResource[data-url="${url}"]`);
  return el ? el.innerHTML : null;
}

function text(url, cb) {
  const txt = getContent(url);
  if (txt === null) {
    cb(`No such resource ${url}`);
  } else {
    cb(null, txt);
  }
}

function json(url, cb) {
  const txt = getContent(url);
  if (txt === null) {
    cb(`No such resource ${url}`);
  } else {
    cb(null, JSON.parse(txt));
  }
}

function array(url, cb) {
  const txt = getContent(url);
  if (txt === null) {
    cb(`No such resource ${url}`);
  } else {
    const uint8array = toByteArray(txt);
    const buffer = new ArrayBuffer(uint8array.length);
    const view = new Uint8Array(buffer);
    view.set(uint8array);
    cb(null, buffer);
  }
}

function blob(url, mimeType, cb) {
  const txt = getContent(url);
  if (txt === null) {
    cb(`No such resource ${url}`);
  } else {
    const buffer = toByteArray(txt);
    cb(null, new Blob([buffer], { type: mimeType }));
  }
}

// Export fetch methods
export default {
  json,
  text,
  blob,
  array,
};

index.js

import JSZip from 'jszip';
import Monologue from 'monologue.js';

// Module dependencies and constants
import request from 'paraviewweb/src/IO/Core/DataManager/request';
import htmlRequest from 'paraviewweb/src/IO/Core/DataManager/htmlRequest';
import PatternMap from 'paraviewweb/src/IO/Core/DataManager/pattern';

const typeFnMap = {
  json: request.fetchJSON,
  text: request.fetchTxt,
  blob: request.fetchBlob,
  arraybuffer: request.fetchArray,
  array: request.fetchArray,
};

// Internal helper that return the current time
function ts() {
  return new Date().getTime();
}

function updateDataSize(data) {
  if (data.type === 'json') {
    data.size = JSON.stringify(data.data).length;
  } else if (data.type === 'blob') {
    data.size = data.data.size;
  } else {
    data.size = data.data.length;
  }
  return data.size;
}

// Should use converter
// flipArrayEndianness = function(array) {
//   var u8 = new Uint8Array(array.buffer, array.byteOffset, array.byteLength);
//   for (var i=0; i<array.byteLength; i+=array.BYTES_PER_ELEMENT) {
//     for (var j=i+array.BYTES_PER_ELEMENT-1, k=i; j>k; j--, k++) {
//       var tmp = u8[k];
//       u8[k] = u8[j];
//       u8[j] = tmp;
//     }
//   }
//   return array;
// }

export default class DataManager {
  constructor(cacheSize = 1000000000) {
    this.pattern = new PatternMap();
    this.keyToTypeMap = {};
    this.cacheSize = cacheSize;
    this.cacheData = {
      cache: {},
      modified: 0,
      ts: 0,
      size: 0,
    };
  }

  destroy() {
    this.off();
    this.clear();
  }

  // Fetch data in an asynchronous manner
  // This will trigger an event using the key as the type
  fetch(key, options, notificationTopic = null) {
    const url = options ? this.pattern.getValue(key, options) : key;
    let dataCached = this.cacheData.cache[url];

    if (dataCached) {
      if (!dataCached.pending) {
        dataCached.ts = ts();
        this.cacheData.ts = dataCached.ts;

        // Trigger the event after the return
        setTimeout(() => {
          const array = dataCached.keysToNotify || [key];
          let count = array.length;

          delete dataCached.keysToNotify;

          while (count) {
            count -= 1;
            this.emit(array[count], dataCached);
          }

          if (notificationTopic) {
            this.emit(notificationTopic, dataCached);
          }
        }, 0);
      } else {
        dataCached.keysToNotify.push(key);
        if (notificationTopic) {
          dataCached.keysToNotify.push(notificationTopic);
        }
      }
    } else {
      // Run Garbage collector to free memory if need be
      this.gc();

      // Prevent double fetch
      this.cacheData.cache[url] = {
        pending: true,
        keysToNotify: [key],
      };

      if (notificationTopic) {
        this.cacheData.cache[url].keysToNotify.push(notificationTopic);
      }

      // Need to fetch the data on the web
      const self = this;
      const typeFnMime = this.keyToTypeMap[key];
      const type = typeFnMime[0];
      const fn = typeFnMime[1];
      const mimeType = typeFnMime[2];
      const callback = (error, data) => {
        if (error) {
          delete self.cacheData.cache[url];
          self.emit(key, {
            error,
            data: {
              key,
              options,
              url,
              typeFnMime,
            },
          });
          return;
        }

        dataCached = {
          data,
          type,
          requestedURL: url,
          pending: false,
        };

        // Handle internal url for image blob
        if (mimeType && mimeType.indexOf('image') !== -1) {
          dataCached.url = window.URL.createObjectURL(data);
        }

        // Update memory usage
        self.cacheData.size += updateDataSize(dataCached);

        // Update ts
        dataCached.ts = ts();
        self.cacheData.ts = dataCached.ts;
        self.cacheData.modified = self.cacheData.ts;

        // Trigger the event
        const array = self.cacheData.cache[url].keysToNotify;
        let count = array.length;

        // Store it in the cache
        self.cacheData.cache[url] = dataCached;

        while (count) {
          count -= 1;
          self.emit(array[count], dataCached);
        }
      };

      if (mimeType) {
        fn(url, mimeType, callback);
      } else {
        fn(url, callback);
      }
    }

    return url;
  }

  // Fetch data from URL
  fetchURL(url, type, mimeType, notificationTopic = null) {
    this.keyToTypeMap[url] = [type, typeFnMap[type], mimeType];
    return this.fetch(url, null, notificationTopic);
  }

  // Get data in cache
  get(url, freeCache) {
    const dataObj = this.cacheData.cache[url];
    if (freeCache) {
      this.free(url);
    }
    return dataObj;
  }

  // Free a fetched data
  free(url) {
    const dataCached = this.cacheData.cache[url];
    if (dataCached && dataCached.url) {
      window.URL.revokeObjectURL(dataCached.url);
      delete dataCached.url;
    }

    delete this.cacheData.cache[url];
    this.off(url);
  }

  // Register a key/pattern for future use
  // Type can only be ['json', 'text', 'blob', 'array']
  // mimeType is only required for blob
  registerURL(key, filePattern, type, mimeType) {
    this.pattern.registerPattern(key, filePattern);
    this.keyToTypeMap[key] = [type, typeFnMap[type], mimeType];
  }

  // Free previously registered URL
  unregisterURL(key) {
    this.pattern.unregisterPattern(key);
    delete this.keyToTypeMap[key];
    this.off(key);
  }

  // Empty cache
  clear() {
    const urlToDelete = [];
    Object.keys(this.cacheData.cache).forEach((url) => {
      urlToDelete.push(url);
    });

    let count = urlToDelete.length;
    while (count) {
      count -= 1;
      this.free(urlToDelete[count]);
    }
    this.cacheData.size = 0;
  }

  gc() {
    if (this.cacheData.size > this.cacheSize) {
      console.log('Free cache memory', this.cacheData.size);
      this.clear();
    }
  }

  setCacheSize(sizeBeforeGC) {
    this.cacheSize = sizeBeforeGC;
  }

  getCacheSize() {
    return this.cacheSize;
  }

  getMemoryUsage() {
    return this.cacheData.size;
  }

  useHttpRequest() {
    typeFnMap.json = request.fetchJSON;
    typeFnMap.text = request.fetchTxt;
    typeFnMap.blob = request.fetchBlob;
    typeFnMap.arraybuffer = request.fetchArray;
    typeFnMap.array = request.fetchArray;
    return this;
  }

  useZipContent(zipContent, options) {
    return new Promise((accept, reject) => {
      const zip = new JSZip();
      let zipRoot = zip;
      zip.loadAsync(zipContent, options).then(() => {
        // Find root index.json
        const metaFiles = [];
        zip.forEach((relativePath, zipEntry) => {
          if (relativePath.indexOf('index.json') !== -1) {
            metaFiles.push(relativePath);
          }
        });
        metaFiles.sort((a, b) => a.length > b.length);
        const fullRootPath = metaFiles[0].split('/');
        while (fullRootPath.length > 1) {
          const dirName = fullRootPath.shift();
          zipRoot = zipRoot.folder(dirName);
        }

        // Replace access method
        typeFnMap.json = (url, cb) => {
          zipRoot
            .file(url)
            .async('string')
            .then(
              (str) => {
                cb(null, JSON.parse(str));
              },
              (err) => {
                cb(err);
              }
            );
        };

        typeFnMap.text = (url, cb) => {
          zipRoot
            .file(url)
            .async('string')
            .then(
              (str) => {
                cb(null, str);
              },
              (err) => {
                cb(err);
              }
            );
        };

        typeFnMap.blob = (url, mimeType, cb) => {
          zipRoot
            .file(url)
            .async('uint8array')
            .then(
              (uint8array) => {
                const buffer = new ArrayBuffer(uint8array.length);
                const view = new Uint8Array(buffer);
                view.set(uint8array);
                cb(null, new Blob([buffer], { type: mimeType }));
              },
              (err) => {
                cb(err);
              }
            );
        };

        typeFnMap.arraybuffer = (url, cb) => {
          zipRoot
            .file(url)
            .async('uint8array')
            .then(
              (uint8array) => {
                const buffer = new ArrayBuffer(uint8array.length);
                const view = new Uint8Array(buffer);
                view.set(uint8array);
                cb(null, buffer);
              },
              (err) => {
                cb(err);
              }
            );
        };

        typeFnMap.array = typeFnMap.arraybuffer;

        // Fix any previously registered pattern
        Object.keys(this.keyToTypeMap).forEach((key) => {
          const array = this.keyToTypeMap[key];
          array[1] = typeFnMap[array[0]];
        });

        accept(this);
      });
    });
  }

  useHtmlContent() {
    typeFnMap.json = htmlRequest.json;
    typeFnMap.text = htmlRequest.text;
    typeFnMap.blob = htmlRequest.blob;
    typeFnMap.arraybuffer = htmlRequest.array;
    typeFnMap.array = htmlRequest.array;

    // Fix any previously registered pattern
    Object.keys(this.keyToTypeMap).forEach((key) => {
      const array = this.keyToTypeMap[key];
      array[1] = typeFnMap[array[0]];
    });

    return this;
  }
}

Monologue.mixInto(DataManager);

pattern.js

// The goal of that module is to be able to register a set of String pattern
// and have a simple way to evaluate that pattern from an object.
// Here is an example on how the following module can be used.
//
//     var m = new PatternMap();
//     m.registerPattern('imagesURL', '{time}/{pressure}/{phi}_{theta}.png');
//     m.registerPattern('jsonURL', '{time}/{pressure}/data.json');
//     var time = [1, 2, 3, 4, 5, 6],
//         pressure = [34, 35, 36],
//         phi = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90],
//         theta = [0, 20, 40, 60, 80];
//
//     timeCount =  time.length;
//     var options = {};
//     while(timeCount--) {
//        options.time = time[timeCount];
//        pressureCount = pressure.length;
//        while(pressureCount--) {
//           options.pressure = pressure[pressureCount];
//           phiCount = phi.length;
//           while(phiCount--) {
//              options.phi = phi[phiCount];
//              thetaCount = theta.length;
//              while(thetaCount--) {
//                 options.theta = theta[thetaCount];
//                 console.log(" => Image: " + m.getValue('imageURL', options));
//              }
//           }
//           console.log(" => JSON: " + m.getValue('jsonURL', options));
//        }
//     }
//     m.unregisterPattern('imageURL');

export default class PatternMap {
  constructor() {
    this.keyPatternMap = {};
  }

  // Register a pattern to a given key
  registerPattern(key, pattern) {
    this.keyPatternMap[key] = pattern;
  }

  // Unregister a key
  unregisterPattern(key) {
    delete this.keyPatternMap[key];
  }

  // Evaluate the pattern base on its registered key and set of key to be replaced
  getValue(key, options) {
    let result = this.keyPatternMap[key];
    const keyPattern = ['{', '}'];

    Object.keys(options).forEach((opt) => {
      result = result.replace(
        new RegExp(keyPattern.join(opt), 'g'),
        options[opt]
      );
    });

    return result;
  }
}

request.js

// Generic request handler
function makeRequest(url, handler) {
  const xhr = new XMLHttpRequest();

  xhr.open('GET', url, true);
  xhr.responseType = handler.type;

  xhr.onload = function onLoad(e) {
    if (this.status === 200 || this.status === 0) {
      handler.fn(null, xhr);
      return;
    }
    handler.fn(e, xhr);
  };
  xhr.onerror = function onError(e) {
    handler.fn(e, xhr);
  };
  xhr.send();
}

// Array buffer handler
function arraybufferHandler(callback) {
  return {
    type: 'arraybuffer',
    fn: (error, xhrObject) => {
      if (error) {
        callback(error);
        return;
      }
      callback(null, xhrObject.response);
    },
  };
}

// Text handler
function textHandler(callback) {
  return {
    type: 'text',
    fn: (error, xhrObject) => {
      if (error) {
        callback(error);
        return;
      }
      callback(null, xhrObject.response);
    },
  };
}

// JSON handler
function jsonHandler(callback) {
  return {
    type: 'text',
    fn: (error, xhrObject) => {
      if (error) {
        callback(error);
        return;
      }
      callback(null, JSON.parse(xhrObject.response));
    },
  };
}

// Blob handler
function blobHandler(mimeType, callback) {
  return {
    type: 'blob',
    fn: (error, xhrObject) => {
      if (error) {
        callback(error);
        return;
      }

      const blob = new Blob([xhrObject.response], {
        type: mimeType,
      });
      callback(null, blob);
    },
  };
}

// Fetch methods

function fetchJSON(url, callback) {
  makeRequest(url, jsonHandler(callback));
}

function fetchTxt(url, callback) {
  makeRequest(url, textHandler(callback));
}

function fetchBlob(url, mimeType, callback) {
  makeRequest(url, blobHandler(mimeType, callback));
}

function fetchArray(url, callback) {
  makeRequest(url, arraybufferHandler(callback));
}

// Export fetch methods
export default {
  fetchJSON,
  fetchTxt,
  fetchBlob,
  fetchArray,
};