HttpDataSetReader

Introduction

The vtkHttpDataSetReader is using a custom format that only exist in vtk.js
which aims to simplify data fetching in an HTTP context. Basically the format
is composed of a JSON metadata file referencing all the required data array
as side binary files along with all the dataset configuration (i.e.: type,
extent…).

Usage

import vtkHttpDataSetReader from '@kitware/vtk.js/IO/Core/HttpDataSetReader';

const reader = vtkHttpDataSetReader.newInstance();
reader.setURL('/Data/can.ex2/index.json').then((reader, dataset) => {
  console.log('Metadata loaded with the geometry', dataset);

  reader.getArrays().forEach(array => {
    console.log('-', array.name, array.location, ':', array.enable);
  });

  reader.update()
    .then((reader, dataset) => {
      console.log('dataset fully loaded', dataset);
    });
});

Methods

enableArray

Enable or disable a given array.

reader.enableArray('pointData', 'Temperature');
reader.enableArray('pointData', 'Pressure', false);
reader.enableArray('cellData', 'CellId', true);
reader.enableArray('fieldData', 'labels', true);

Argument	Type	Required	Description
location	String	Yes
name	String	Yes
enable	Boolean	No

extend

Method used to decorate a given object (publicAPI+model) with vtkHttpDataSetReader characteristics.

Argument	Type	Required	Description
publicAPI		Yes	object on which methods will be bounds (public)
model		Yes	object on which data structure will be bounds (protected)
initialValues	IHttpDataSetReaderInitialValues	No	(default: {})

getArrays

Get the list of available array with their location and if they are
enable or not for download using the update() method.

getArraysByReference

getBaseURL

Get the base url to use to download arrays or other data from the given
dataset.

reader.setURL('/Data/can.ex2/index.json');

if (reader.getBaseURL() === '/Data/can.ex2') {
  console.log('Good guess...');
}

getDataAccessHelper

getEnableArray

getFetchGzip

getUrl

Get the url of the object to load.

invokeBusy

Argument	Type	Required	Description
busy	Boolean	Yes

isBusy

Get the current status of the reader. True means busy and False means
idle.

loadData

newInstance

Method used to create a new instance of vtkHttpDataSetReader while enabling a
default behavior regarding the data array and the way they should be fetched
from the server.

The enableArray argument allow you to choose if you want to activate all
data array by default or if you will have to manually enable them before
downloading them.

Argument	Type	Required	Description
initialValues	IHttpDataSetReaderInitialValues	No	for pre-setting some of its content

onBusy

Attach listener to monitor when the reader is downloading data or not.

const subscription = reader.onBusy(busy => {
  console.log('Reader is', busy ? 'downloading' : 'idle');
})

reader.update();
// much later
subscription.unsubscribe();

Argument	Type	Required	Description
callback		Yes

parseObject

Set the dataset object to use for data fetching.

Argument	Type	Required	Description
manifest	IDatasetManifest	Yes	The dataset manifest object
options	IParseObjectOptions	Yes

requestData

Argument	Type	Required	Description
inData		Yes
outData		Yes

setDataAccessHelper

Argument	Type	Required	Description
dataAccessHelper		Yes

setProgressCallback

Argument	Type	Required	Description
progressCallback		Yes

setUrl

Set the url for the dataset to load.

const reader = HttpDataSetReader.newInstance();
isReady = reader.setURL('/Data/can.ex2/index.json');

// Same as 
const reader = HttpDataSetReader.newInstance({ url: '/Data/can.ex2/index.json' });
isReady = reader.updateMetadata();

Argument	Type	Required	Description
url	String	Yes	the url of the object to load.
option	IHttpDataSetReaderOptions	No	The Draco reader options.

updateMetadata

Source

index.d.ts

import { vtkAlgorithm, vtkObject, vtkSubscription } from "../../../interfaces";
import HtmlDataAccessHelper from "../DataAccessHelper/HtmlDataAccessHelper";
import HttpDataAccessHelper from "../DataAccessHelper/HttpDataAccessHelper";
import JSZipDataAccessHelper from "../DataAccessHelper/JSZipDataAccessHelper";
import LiteHttpDataAccessHelper from "../DataAccessHelper/LiteHttpDataAccessHelper";

/**
 *
 */
export interface IHttpDataSetReaderInitialValues {
	enableArray?: boolean;
	fetchGzip?: boolean;
	arrays?: any[];
	url?: string;
	baseURL?: string;
	requestCount?: number;
}

export interface IHttpDataSetReaderOptions {
	fullpath?: string,
	compression?: string,
	loadData?: boolean;
}

export interface IHttpDataSetReaderArray {
	location: string;
	name: string;
	enable: boolean;
}

export interface IRange {
	max: number,
	component: unknown,
	min: number
}

export interface IPointDataArray {
	data: {
		numberOfComponents: number,
		name: string,
		vtkClass: string,
		dataType: string,
		ranges: Array<IRange>,
		ref: {
			registration: string,
			encode: string,
			basepath: string,
			id: string
		},
		size: number
	}
}

export interface IDatasetManifest {
	origin: [number, number, number],
	cellData: {
		arrays: Array<unknown>,
		vtkClass: string
	},
	FieldData: {
		arrays: Array<unknown>,
		vtkClass: string,
	},
	vtkClass: string,	
	pointData: {
		arrays: Array<IPointDataArray>,
		vtkClass: string
	},
	spacing: [number, number, number],
	extent: [number, number, number, number, number, number],
	direction: [number, number, number, number, number, number, number, number, number],
	metadata?: Record<string, unknown>
}

export interface IParseObjectOptions {
	loadData: boolean,
	baseUrl: string,
	deepCopy: boolean
}

type vtkHttpDataSetReaderBase = vtkObject & Omit<vtkAlgorithm,
	| 'getInputData'
	| 'setInputData'
	| 'setInputConnection'
	| 'getInputConnection'
	| 'addInputConnection'
	| 'addInputData'>;

export interface vtkHttpDataSetReader extends vtkHttpDataSetReaderBase {

	/**
	 * Enable or disable a given array.
	 * 
	 * ```js
	 * reader.enableArray('pointData', 'Temperature');
	 * reader.enableArray('pointData', 'Pressure', false);
	 * reader.enableArray('cellData', 'CellId', true);
	 * reader.enableArray('fieldData', 'labels', true);
	 * ```
	 * @param {String} location 
	 * @param {String} name 
	 * @param {Boolean} [enable] 
	 */
	enableArray(location: string, name: string, enable?: boolean): void;

	/**
	 * Get the list of available array with their location and if they are
	 * enable or not for download using the __update()__ method.
	 */
	getArrays(): IHttpDataSetReaderArray[];

	/**
	 * 
	 */
	getArraysByReference(): IHttpDataSetReaderArray[];

	/**
	 * Get the base url to use to download arrays or other data from the given
	 * dataset.
	 * 
	 * ```js
	 * reader.setURL('/Data/can.ex2/index.json');
	 * 
	 * if (reader.getBaseURL() === '/Data/can.ex2') {
	 *   console.log('Good guess...');
	 * }
	 * ```
	 */
	getBaseURL(): string;

	/**
	 * 
	 */
	getDataAccessHelper(): HtmlDataAccessHelper | HttpDataAccessHelper | JSZipDataAccessHelper | LiteHttpDataAccessHelper;

	/**
	 * 
	 */
	getEnableArray(): boolean;

	/**
	 * 
	 */
	getFetchGzip(): boolean;

	/**
	 * Get the url of the object to load.
	 */
	getUrl(): string;

	/**
	 * 
	 * @param {Boolean} busy 
	 */
	invokeBusy(busy: boolean): void;

	/**
	 * Get the current status of the reader. True means busy and False means
	 * idle.
	 */
	isBusy(): boolean;

	/**
	 * 
	 */
	loadData(): string;

	/**
	 * Attach listener to monitor when the reader is downloading data or not.
	 * 
	 * ```js
	 * const subscription = reader.onBusy(busy => {
	 *   console.log('Reader is', busy ? 'downloading' : 'idle');
	 * })
	 * 
	 * reader.update();
	 * // much later
	 * subscription.unsubscribe();
	 * ```
	 * @param callback 
	 */
	onBusy(callback: (busy: boolean) => any): vtkSubscription;

	/**
	 *
	 * @param inData 
	 * @param outData 
	 */
	requestData(inData: any, outData: any): void;

	/**
	 * 
	 * @param dataAccessHelper 
	 */
	setDataAccessHelper(dataAccessHelper: HtmlDataAccessHelper | HttpDataAccessHelper | JSZipDataAccessHelper | LiteHttpDataAccessHelper): boolean;

	/**
	 * 
	 * @param progressCallback 
	 */
	setProgressCallback(progressCallback: any): boolean;

	/**
	 * Set the url for the dataset to load.
	 * 
	 * ```js
	 * const reader = HttpDataSetReader.newInstance();
	 * isReady = reader.setURL('/Data/can.ex2/index.json');
	 * 
	 * // Same as 
	 * const reader = HttpDataSetReader.newInstance({ url: '/Data/can.ex2/index.json' });
	 * isReady = reader.updateMetadata();
	 * ```
	 * @param {String} url the url of the object to load.
	 * @param {IHttpDataSetReaderOptions} [option] The Draco reader options.
	 */
	setUrl(url: string, option?: IHttpDataSetReaderOptions): Promise<any>;

	/**
	 * Set the dataset object to use for data fetching.
	 * 
	 * @param {IDatasetManifest} manifest The dataset manifest object
	 * @param {IParseObjectOptions} options
	 */
	parseObject(manifest: IDatasetManifest, options: IParseObjectOptions): Promise<void>;

	/**
	 * 
	 */
	updateMetadata(): Promise<any>;
}

/**
 * Method used to decorate a given object (publicAPI+model) with vtkHttpDataSetReader characteristics.
 *
 * @param publicAPI object on which methods will be bounds (public) 
 * @param model object on which data structure will be bounds (protected) 
 * @param {IHttpDataSetReaderInitialValues} [initialValues] (default: {}) 
 */
export function extend(publicAPI: object, model: object, initialValues?: IHttpDataSetReaderInitialValues): void;

/**
 * Method used to create a new instance of vtkHttpDataSetReader while enabling a
 * default behavior regarding the data array and the way they should be fetched
 * from the server.
 *
 * The __enableArray__ argument allow you to choose if you want to activate all
 * data array by default or if you will have to manually enable them before
 * downloading them.
 * @param {IHttpDataSetReaderInitialValues} [initialValues] for pre-setting some of its content 
 */
export function newInstance(initialValues?: IHttpDataSetReaderInitialValues): vtkHttpDataSetReader;

/**
 * The vtkHttpDataSetReader is using a custom format that only exist in vtk.js
 * which aims to simplify data fetching in an HTTP context. Basically the format
 * is composed of a JSON metadata file referencing all the required data array
 * as side binary files along with all the dataset configuration (i.e.: type,
 * extent...). 
 *
 * @example
 * ```js
 * import vtkHttpDataSetReader from '@kitware/vtk.js/IO/Core/HttpDataSetReader';
 *
 * const reader = vtkHttpDataSetReader.newInstance();
 * reader.setURL('/Data/can.ex2/index.json').then((reader, dataset) => {
 *   console.log('Metadata loaded with the geometry', dataset);
 * 
 *   reader.getArrays().forEach(array => {
 *     console.log('-', array.name, array.location, ':', array.enable);
 *   });
 * 
 *   reader.update()
 *     .then((reader, dataset) => {
 *       console.log('dataset fully loaded', dataset);
 *     });
 * });
 * ```
 */
export declare const vtkHttpDataSetReader: {
	newInstance: typeof newInstance,
	extend: typeof extend,
};
export default vtkHttpDataSetReader;

index.js

// For vtk factory
import 'vtk.js/Sources/Common/DataModel/ImageData';
import 'vtk.js/Sources/Common/DataModel/PolyData';

import vtk from 'vtk.js/Sources/vtk';
import macro from 'vtk.js/Sources/macros';
import DataAccessHelper from 'vtk.js/Sources/IO/Core/DataAccessHelper';
import vtkDataArray from 'vtk.js/Sources/Common/Core/DataArray';
import vtkStringArray from 'vtk.js/Sources/Common/Core/StringArray';

// Enable data soure for DataAccessHelper
import 'vtk.js/Sources/IO/Core/DataAccessHelper/LiteHttpDataAccessHelper'; // Just need HTTP
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/HttpDataAccessHelper'; // HTTP + zip
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/HtmlDataAccessHelper'; // html + base64 + zip
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/JSZipDataAccessHelper'; // zip

const fieldDataLocations = ['pointData', 'cellData', 'fieldData'];
const ARRAY_BUILDERS = {
  vtkDataArray,
  vtkStringArray,
};

// ----------------------------------------------------------------------------
// Global methods
// ----------------------------------------------------------------------------

const cachedArrays = {};

const GEOMETRY_ARRAYS = {
  vtkPolyData(dataset) {
    const arrayToDownload = [];
    arrayToDownload.push(dataset.points);
    ['verts', 'lines', 'polys', 'strips'].forEach((cellName) => {
      if (dataset[cellName]) {
        arrayToDownload.push(dataset[cellName]);
      }
    });

    return arrayToDownload;
  },

  vtkImageData(dataset) {
    return [];
  },

  vtkUnstructuredGrid(dataset) {
    const arrayToDownload = [];
    arrayToDownload.push(dataset.points);
    arrayToDownload.push(dataset.cells);
    arrayToDownload.push(dataset.cellTypes);

    return arrayToDownload;
  },

  vtkRectilinearGrid(dataset) {
    const arrayToDownload = [];
    arrayToDownload.push(dataset.xCoordinates);
    arrayToDownload.push(dataset.yCoordinates);
    arrayToDownload.push(dataset.zCoordinates);

    return arrayToDownload;
  },
};

function processDataSet(
  publicAPI,
  model,
  dataset,
  fetchArray,
  resolve,
  reject,
  loadData
) {
  const enable = model.enableArray;

  // Generate array list
  model.arrays = [];

  fieldDataLocations.forEach((location) => {
    if (dataset[location]) {
      dataset[location].arrays
        .map((i) => i.data)
        .forEach((array) => {
          model.arrays.push({
            name: array.name,
            enable,
            location,
            array,
            registration: array.ref.registration || 'addArray',
          });
        });

      // Reset data arrays
      dataset[location].arrays = [];
    }
  });

  // Fetch geometry arrays
  const pendingPromises = [];
  const { progressCallback } = model;
  const compression = model.fetchGzip ? 'gz' : null;
  GEOMETRY_ARRAYS[dataset.vtkClass](dataset).forEach((array) => {
    pendingPromises.push(fetchArray(array, { compression, progressCallback }));
  });

  function success() {
    model.dataset = vtk(dataset);
    if (!loadData) {
      model.output[0] = model.dataset;
      resolve(publicAPI, model.output[0]);
    } else {
      publicAPI.loadData().then(() => {
        model.output[0] = model.dataset;
        resolve(publicAPI, model.output[0]);
      });
    }
  }

  // Wait for all geometry array to be fetched
  if (pendingPromises.length) {
    Promise.all(pendingPromises).then(success, (err) => {
      reject(err);
    });
  } else {
    success();
  }
}

// ----------------------------------------------------------------------------
// vtkHttpDataSetReader methods
// ----------------------------------------------------------------------------

function vtkHttpDataSetReader(publicAPI, model) {
  // Set our className
  model.classHierarchy.push('vtkHttpDataSetReader');

  // Empty output by default
  model.output[0] = vtk({ vtkClass: 'vtkPolyData' });

  // Create default dataAccessHelper if not available
  if (!model.dataAccessHelper) {
    model.dataAccessHelper = DataAccessHelper.get('http');
  }

  // Internal method to fetch Array
  function fetchArray(array, options = {}) {
    const arrayId = `${array.ref.id}|${array.vtkClass}`;
    if (!cachedArrays[arrayId]) {
      // Cache the promise while fetching
      cachedArrays[arrayId] = model.dataAccessHelper
        .fetchArray(publicAPI, model.baseURL, array, options)
        .then((newArray) => {
          // Replace the promise with the array in cache once downloaded
          cachedArrays[arrayId] = newArray;
          return newArray;
        });
    } else {
      // cacheArrays[arrayId] can be a promise or value
      Promise.resolve(cachedArrays[arrayId]).then((cachedArray) => {
        if (array !== cachedArray) {
          Object.assign(array, cachedArray);
          delete array.ref;
        }
      });
    }

    return Promise.resolve(cachedArrays[arrayId]);
  }

  // Fetch dataset (metadata)
  publicAPI.updateMetadata = (loadData = false) => {
    if (model.compression === 'zip') {
      return new Promise((resolve, reject) => {
        DataAccessHelper.get('http')
          .fetchBinary(model.url)
          .then(
            (zipContent) => {
              model.dataAccessHelper = DataAccessHelper.get('zip', {
                zipContent,
                callback: (zip) => {
                  model.baseURL = '';
                  model.dataAccessHelper
                    .fetchJSON(publicAPI, 'index.json')
                    .then(
                      (dataset) => {
                        publicAPI
                          .parseObject(dataset, {
                            loadData,
                            deepCopy: false,
                          })
                          .then(resolve, reject);
                      },
                      (error) => {
                        reject(error);
                      }
                    );
                },
              });
            },
            (error) => {
              reject(error);
            }
          );
      });
    }

    return new Promise((resolve, reject) => {
      model.dataAccessHelper.fetchJSON(publicAPI, model.url).then(
        (dataset) => {
          publicAPI
            .parseObject(dataset, { loadData, deepCopy: false })
            .then(resolve, reject);
        },
        (error) => {
          reject(error);
        }
      );
    });
  };

  // Set DataSet url
  publicAPI.setUrl = (url, options = {}) => {
    if (url.indexOf('index.json') === -1 && !options.fullpath) {
      model.baseURL = url;
      model.url = `${url}/index.json`;
    } else {
      model.url = url;

      // Remove the file in the URL
      const path = url.split('/');
      path.pop();
      model.baseURL = path.join('/');
    }

    model.compression = options.compression;

    // Fetch metadata
    return publicAPI.updateMetadata(!!options.loadData);
  };

  publicAPI.parseObject = (
    manifest,
    { loadData, baseUrl, deepCopy = true }
  ) => {
    if (baseUrl) {
      model.baseURL = baseUrl;
    }

    const dataset = deepCopy ? structuredClone(manifest) : manifest;

    return new Promise((resolve, reject) => {
      processDataSet(
        publicAPI,
        model,
        dataset,
        fetchArray,
        resolve,
        reject,
        loadData
      );
    });
  };

  // Fetch the actual data arrays
  publicAPI.loadData = () => {
    const datasetObj = model.dataset;
    const arrayToFecth = model.arrays
      .filter((array) => array.enable)
      .filter((array) => array.array.ref)
      .map((array) => array.array);

    return new Promise((resolve, reject) => {
      const error = (e) => {
        reject(e);
      };

      const processNext = () => {
        if (arrayToFecth.length) {
          const { progressCallback } = model;
          const compression = model.fetchGzip ? 'gz' : null;
          fetchArray(arrayToFecth.pop(), {
            compression,
            progressCallback,
          }).then(processNext, error);
        } else if (datasetObj) {
          // Perform array registration on new arrays
          model.arrays
            .filter(
              (metaArray) => metaArray.registration && !metaArray.array.ref
            )
            .forEach((metaArray) => {
              const newArray = ARRAY_BUILDERS[
                metaArray.array.vtkClass
              ].newInstance(metaArray.array);
              datasetObj[`get${macro.capitalize(metaArray.location)}`]()[
                metaArray.registration
              ](newArray);
              delete metaArray.registration;
            });
          datasetObj.modified();
          resolve(publicAPI, datasetObj);
        }
      };

      // Start processing queue
      processNext();
    });
  };

  publicAPI.requestData = (inData, outData) => {
    // do nothing loadData will eventually load up the data
  };

  // Toggle arrays to load
  publicAPI.enableArray = (location, name, enable = true) => {
    const activeArray = model.arrays.filter(
      (array) => array.name === name && array.location === location
    );
    if (activeArray.length === 1) {
      activeArray[0].enable = enable;
    }
  };

  // return Busy state
  publicAPI.isBusy = () => !!model.requestCount;
}

// ----------------------------------------------------------------------------
// Object factory
// ----------------------------------------------------------------------------

const DEFAULT_VALUES = {
  enableArray: true,
  fetchGzip: false,
  arrays: [],
  url: null,
  baseURL: null,
  requestCount: 0,
  // dataAccessHelper: null,
};

// ----------------------------------------------------------------------------

export function extend(publicAPI, model, initialValues = {}) {
  Object.assign(model, DEFAULT_VALUES, initialValues);

  // Build VTK API
  macro.obj(publicAPI, model);
  macro.get(publicAPI, model, [
    'enableArray',
    'fetchGzip',
    'url',
    'baseURL',
    'dataAccessHelper',
  ]);
  macro.set(publicAPI, model, ['dataAccessHelper', 'progressCallback']);
  macro.getArray(publicAPI, model, ['arrays']);
  macro.algo(publicAPI, model, 0, 1);
  macro.event(publicAPI, model, 'busy');

  // Object methods
  vtkHttpDataSetReader(publicAPI, model);

  // Make sure we can destructuring progressCallback from model
  if (model.progressCallback === undefined) {
    model.progressCallback = null;
  }
}

// ----------------------------------------------------------------------------

export const newInstance = macro.newInstance(extend, 'vtkHttpDataSetReader');

// ----------------------------------------------------------------------------

export default { newInstance, extend };