HttpDataSetReader

Introduction

The vtkHttpDataSetReader is using a custom format that only exist in vtk.js
which aims to simplify data fetching in an HTTP context. Basically the format
is composed of a JSON metadata file referencing all the required data array
as side binary files along with all the dataset configuration (i.e.: type,
extent…).

Usage

import vtkHttpDataSetReader from '@kitware/vtk.js/IO/Core/HttpDataSetReader';

const reader = vtkHttpDataSetReader.newInstance();
reader.setURL('/Data/can.ex2/index.json').then((reader, dataset) => {
console.log('Metadata loaded with the geometry', dataset);

reader.getArrays().forEach(array => {
console.log('-', array.name, array.location, ':', array.enable);
});

reader.update()
.then((reader, dataset) => {
console.log('dataset fully loaded', dataset);
});
});

Methods

enableArray

Enable or disable a given array.

reader.enableArray('pointData', 'Temperature');
reader.enableArray('pointData', 'Pressure', false);
reader.enableArray('cellData', 'CellId', true);
reader.enableArray('fieldData', 'labels', true);
Argument Type Required Description
location String Yes
name String Yes
enable Boolean No

extend

Method used to decorate a given object (publicAPI+model) with vtkHttpDataSetReader characteristics.

Argument Type Required Description
publicAPI Yes object on which methods will be bounds (public)
model Yes object on which data structure will be bounds (protected)
initialValues IHttpDataSetReaderInitialValues No (default: {})

getArrays

Get the list of available array with their location and if they are
enable or not for download using the update() method.

getArraysByReference

getBaseURL

Get the base url to use to download arrays or other data from the given
dataset.

reader.setURL('/Data/can.ex2/index.json');

if (reader.getBaseURL() === '/Data/can.ex2') {
console.log('Good guess...');
}

getDataAccessHelper

getEnableArray

getFetchGzip

getUrl

Get the url of the object to load.

invokeBusy

Argument Type Required Description
busy Boolean Yes

isBusy

Get the current status of the reader. True means busy and False means
idle.

loadData

newInstance

Method used to create a new instance of vtkHttpDataSetReader while enabling a
default behavior regarding the data array and the way they should be fetched
from the server.

The enableArray argument allow you to choose if you want to activate all
data array by default or if you will have to manually enable them before
downloading them.

Argument Type Required Description
initialValues IHttpDataSetReaderInitialValues No for pre-setting some of its content

onBusy

Attach listener to monitor when the reader is downloading data or not.

const subscription = reader.onBusy(busy => {
console.log('Reader is', busy ? 'downloading' : 'idle');
})

reader.update();
// much later
subscription.unsubscribe();
Argument Type Required Description
callback Yes

parseObject

Set the dataset object to use for data fetching.

Argument Type Required Description
manifest IDatasetManifest Yes The dataset manifest object
options IParseObjectOptions Yes

requestData

Argument Type Required Description
inData Yes
outData Yes

setDataAccessHelper

Argument Type Required Description
dataAccessHelper Yes

setProgressCallback

Argument Type Required Description
progressCallback Yes

setUrl

Set the url for the dataset to load.

const reader = HttpDataSetReader.newInstance();
isReady = reader.setURL('/Data/can.ex2/index.json');

// Same as
const reader = HttpDataSetReader.newInstance({ url: '/Data/can.ex2/index.json' });
isReady = reader.updateMetadata();
Argument Type Required Description
url String Yes the url of the object to load.
option IHttpDataSetReaderOptions No The Draco reader options.

updateMetadata

Source

index.d.ts
import { vtkAlgorithm, vtkObject, vtkSubscription } from "../../../interfaces";
import HtmlDataAccessHelper from "../DataAccessHelper/HtmlDataAccessHelper";
import HttpDataAccessHelper from "../DataAccessHelper/HttpDataAccessHelper";
import JSZipDataAccessHelper from "../DataAccessHelper/JSZipDataAccessHelper";
import LiteHttpDataAccessHelper from "../DataAccessHelper/LiteHttpDataAccessHelper";

/**
*
*/
export interface IHttpDataSetReaderInitialValues {
enableArray?: boolean;
fetchGzip?: boolean;
arrays?: any[];
url?: string;
baseURL?: string;
requestCount?: number;
}

export interface IHttpDataSetReaderOptions {
fullpath?: string,
compression?: string,
loadData?: boolean;
}

export interface IHttpDataSetReaderArray {
location: string;
name: string;
enable: boolean;
}

export interface IRange {
max: number,
component: unknown,
min: number
}

export interface IPointDataArray {
data: {
numberOfComponents: number,
name: string,
vtkClass: string,
dataType: string,
ranges: Array<IRange>,
ref: {
registration: string,
encode: string,
basepath: string,
id: string
},
size: number
}
}

export interface IDatasetManifest {
origin: [number, number, number],
cellData: {
arrays: Array<unknown>,
vtkClass: string
},
FieldData: {
arrays: Array<unknown>,
vtkClass: string,
},
vtkClass: string,
pointData: {
arrays: Array<IPointDataArray>,
vtkClass: string
},
spacing: [number, number, number],
extent: [number, number, number, number, number, number],
direction: [number, number, number, number, number, number, number, number, number],
metadata?: Record<string, unknown>
}

export interface IParseObjectOptions {
loadData: boolean,
baseUrl: string,
deepCopy: boolean
}

type vtkHttpDataSetReaderBase = vtkObject & Omit<vtkAlgorithm,
| 'getInputData'
| 'setInputData'
| 'setInputConnection'
| 'getInputConnection'
| 'addInputConnection'
| 'addInputData'>;

export interface vtkHttpDataSetReader extends vtkHttpDataSetReaderBase {

/**
* Enable or disable a given array.
*
* ```js
* reader.enableArray('pointData', 'Temperature');
* reader.enableArray('pointData', 'Pressure', false);
* reader.enableArray('cellData', 'CellId', true);
* reader.enableArray('fieldData', 'labels', true);
* ```
* @param {String} location
* @param {String} name
* @param {Boolean} [enable]
*/
enableArray(location: string, name: string, enable?: boolean): void;

/**
* Get the list of available array with their location and if they are
* enable or not for download using the __update()__ method.
*/
getArrays(): IHttpDataSetReaderArray[];

/**
*
*/
getArraysByReference(): IHttpDataSetReaderArray[];

/**
* Get the base url to use to download arrays or other data from the given
* dataset.
*
* ```js
* reader.setURL('/Data/can.ex2/index.json');
*
* if (reader.getBaseURL() === '/Data/can.ex2') {
* console.log('Good guess...');
* }
* ```
*/
getBaseURL(): string;

/**
*
*/
getDataAccessHelper(): HtmlDataAccessHelper | HttpDataAccessHelper | JSZipDataAccessHelper | LiteHttpDataAccessHelper;

/**
*
*/
getEnableArray(): boolean;

/**
*
*/
getFetchGzip(): boolean;

/**
* Get the url of the object to load.
*/
getUrl(): string;

/**
*
* @param {Boolean} busy
*/
invokeBusy(busy: boolean): void;

/**
* Get the current status of the reader. True means busy and False means
* idle.
*/
isBusy(): boolean;

/**
*
*/
loadData(): string;

/**
* Attach listener to monitor when the reader is downloading data or not.
*
* ```js
* const subscription = reader.onBusy(busy => {
* console.log('Reader is', busy ? 'downloading' : 'idle');
* })
*
* reader.update();
* // much later
* subscription.unsubscribe();
* ```
* @param callback
*/
onBusy(callback: (busy: boolean) => any): vtkSubscription;

/**
*
* @param inData
* @param outData
*/
requestData(inData: any, outData: any): void;

/**
*
* @param dataAccessHelper
*/
setDataAccessHelper(dataAccessHelper: HtmlDataAccessHelper | HttpDataAccessHelper | JSZipDataAccessHelper | LiteHttpDataAccessHelper): boolean;

/**
*
* @param progressCallback
*/
setProgressCallback(progressCallback: any): boolean;

/**
* Set the url for the dataset to load.
*
* ```js
* const reader = HttpDataSetReader.newInstance();
* isReady = reader.setURL('/Data/can.ex2/index.json');
*
* // Same as
* const reader = HttpDataSetReader.newInstance({ url: '/Data/can.ex2/index.json' });
* isReady = reader.updateMetadata();
* ```
* @param {String} url the url of the object to load.
* @param {IHttpDataSetReaderOptions} [option] The Draco reader options.
*/
setUrl(url: string, option?: IHttpDataSetReaderOptions): Promise<any>;

/**
* Set the dataset object to use for data fetching.
*
* @param {IDatasetManifest} manifest The dataset manifest object
* @param {IParseObjectOptions} options
*/
parseObject(manifest: IDatasetManifest, options: IParseObjectOptions): Promise<void>;

/**
*
*/
updateMetadata(): Promise<any>;
}

/**
* Method used to decorate a given object (publicAPI+model) with vtkHttpDataSetReader characteristics.
*
* @param publicAPI object on which methods will be bounds (public)
* @param model object on which data structure will be bounds (protected)
* @param {IHttpDataSetReaderInitialValues} [initialValues] (default: {})
*/
export function extend(publicAPI: object, model: object, initialValues?: IHttpDataSetReaderInitialValues): void;

/**
* Method used to create a new instance of vtkHttpDataSetReader while enabling a
* default behavior regarding the data array and the way they should be fetched
* from the server.
*
* The __enableArray__ argument allow you to choose if you want to activate all
* data array by default or if you will have to manually enable them before
* downloading them.
* @param {IHttpDataSetReaderInitialValues} [initialValues] for pre-setting some of its content
*/
export function newInstance(initialValues?: IHttpDataSetReaderInitialValues): vtkHttpDataSetReader;

/**
* The vtkHttpDataSetReader is using a custom format that only exist in vtk.js
* which aims to simplify data fetching in an HTTP context. Basically the format
* is composed of a JSON metadata file referencing all the required data array
* as side binary files along with all the dataset configuration (i.e.: type,
* extent...).
*
* @example
* ```js
* import vtkHttpDataSetReader from '@kitware/vtk.js/IO/Core/HttpDataSetReader';
*
* const reader = vtkHttpDataSetReader.newInstance();
* reader.setURL('/Data/can.ex2/index.json').then((reader, dataset) => {
* console.log('Metadata loaded with the geometry', dataset);
*
* reader.getArrays().forEach(array => {
* console.log('-', array.name, array.location, ':', array.enable);
* });
*
* reader.update()
* .then((reader, dataset) => {
* console.log('dataset fully loaded', dataset);
* });
* });
* ```
*/
export declare const vtkHttpDataSetReader: {
newInstance: typeof newInstance,
extend: typeof extend,
};
export default vtkHttpDataSetReader;
index.js
// For vtk factory
import 'vtk.js/Sources/Common/DataModel/ImageData';
import 'vtk.js/Sources/Common/DataModel/PolyData';

import vtk from 'vtk.js/Sources/vtk';
import macro from 'vtk.js/Sources/macros';
import DataAccessHelper from 'vtk.js/Sources/IO/Core/DataAccessHelper';
import vtkDataArray from 'vtk.js/Sources/Common/Core/DataArray';
import vtkStringArray from 'vtk.js/Sources/Common/Core/StringArray';

// Enable data soure for DataAccessHelper
import 'vtk.js/Sources/IO/Core/DataAccessHelper/LiteHttpDataAccessHelper'; // Just need HTTP
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/HttpDataAccessHelper'; // HTTP + zip
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/HtmlDataAccessHelper'; // html + base64 + zip
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/JSZipDataAccessHelper'; // zip

const fieldDataLocations = ['pointData', 'cellData', 'fieldData'];
const ARRAY_BUILDERS = {
vtkDataArray,
vtkStringArray,
};

// ----------------------------------------------------------------------------
// Global methods
// ----------------------------------------------------------------------------

const cachedArrays = {};

const GEOMETRY_ARRAYS = {
vtkPolyData(dataset) {
const arrayToDownload = [];
arrayToDownload.push(dataset.points);
['verts', 'lines', 'polys', 'strips'].forEach((cellName) => {
if (dataset[cellName]) {
arrayToDownload.push(dataset[cellName]);
}
});

return arrayToDownload;
},

vtkImageData(dataset) {
return [];
},

vtkUnstructuredGrid(dataset) {
const arrayToDownload = [];
arrayToDownload.push(dataset.points);
arrayToDownload.push(dataset.cells);
arrayToDownload.push(dataset.cellTypes);

return arrayToDownload;
},

vtkRectilinearGrid(dataset) {
const arrayToDownload = [];
arrayToDownload.push(dataset.xCoordinates);
arrayToDownload.push(dataset.yCoordinates);
arrayToDownload.push(dataset.zCoordinates);

return arrayToDownload;
},
};

function processDataSet(
publicAPI,
model,
dataset,
fetchArray,
resolve,
reject,
loadData
) {
const enable = model.enableArray;

// Generate array list
model.arrays = [];

fieldDataLocations.forEach((location) => {
if (dataset[location]) {
dataset[location].arrays
.map((i) => i.data)
.forEach((array) => {
model.arrays.push({
name: array.name,
enable,
location,
array,
registration: array.ref.registration || 'addArray',
});
});

// Reset data arrays
dataset[location].arrays = [];
}
});

// Fetch geometry arrays
const pendingPromises = [];
const { progressCallback } = model;
const compression = model.fetchGzip ? 'gz' : null;
GEOMETRY_ARRAYS[dataset.vtkClass](dataset).forEach((array) => {
pendingPromises.push(fetchArray(array, { compression, progressCallback }));
});

function success() {
model.dataset = vtk(dataset);
if (!loadData) {
model.output[0] = model.dataset;
resolve(publicAPI, model.output[0]);
} else {
publicAPI.loadData().then(() => {
model.output[0] = model.dataset;
resolve(publicAPI, model.output[0]);
});
}
}

// Wait for all geometry array to be fetched
if (pendingPromises.length) {
Promise.all(pendingPromises).then(success, (err) => {
reject(err);
});
} else {
success();
}
}

// ----------------------------------------------------------------------------
// vtkHttpDataSetReader methods
// ----------------------------------------------------------------------------

function vtkHttpDataSetReader(publicAPI, model) {
// Set our className
model.classHierarchy.push('vtkHttpDataSetReader');

// Empty output by default
model.output[0] = vtk({ vtkClass: 'vtkPolyData' });

// Create default dataAccessHelper if not available
if (!model.dataAccessHelper) {
model.dataAccessHelper = DataAccessHelper.get('http');
}

// Internal method to fetch Array
function fetchArray(array, options = {}) {
const arrayId = `${array.ref.id}|${array.vtkClass}`;
if (!cachedArrays[arrayId]) {
// Cache the promise while fetching
cachedArrays[arrayId] = model.dataAccessHelper
.fetchArray(publicAPI, model.baseURL, array, options)
.then((newArray) => {
// Replace the promise with the array in cache once downloaded
cachedArrays[arrayId] = newArray;
return newArray;
});
} else {
// cacheArrays[arrayId] can be a promise or value
Promise.resolve(cachedArrays[arrayId]).then((cachedArray) => {
if (array !== cachedArray) {
Object.assign(array, cachedArray);
delete array.ref;
}
});
}

return Promise.resolve(cachedArrays[arrayId]);
}

// Fetch dataset (metadata)
publicAPI.updateMetadata = (loadData = false) => {
if (model.compression === 'zip') {
return new Promise((resolve, reject) => {
DataAccessHelper.get('http')
.fetchBinary(model.url)
.then(
(zipContent) => {
model.dataAccessHelper = DataAccessHelper.get('zip', {
zipContent,
callback: (zip) => {
model.baseURL = '';
model.dataAccessHelper
.fetchJSON(publicAPI, 'index.json')
.then(
(dataset) => {
publicAPI
.parseObject(dataset, {
loadData,
deepCopy: false,
})
.then(resolve, reject);
},
(error) => {
reject(error);
}
);
},
});
},
(error) => {
reject(error);
}
);
});
}

return new Promise((resolve, reject) => {
model.dataAccessHelper.fetchJSON(publicAPI, model.url).then(
(dataset) => {
publicAPI
.parseObject(dataset, { loadData, deepCopy: false })
.then(resolve, reject);
},
(error) => {
reject(error);
}
);
});
};

// Set DataSet url
publicAPI.setUrl = (url, options = {}) => {
if (url.indexOf('index.json') === -1 && !options.fullpath) {
model.baseURL = url;
model.url = `${url}/index.json`;
} else {
model.url = url;

// Remove the file in the URL
const path = url.split('/');
path.pop();
model.baseURL = path.join('/');
}

model.compression = options.compression;

// Fetch metadata
return publicAPI.updateMetadata(!!options.loadData);
};

publicAPI.parseObject = (
manifest,
{ loadData, baseUrl, deepCopy = true }
) => {
if (baseUrl) {
model.baseURL = baseUrl;
}

const dataset = deepCopy ? structuredClone(manifest) : manifest;

return new Promise((resolve, reject) => {
processDataSet(
publicAPI,
model,
dataset,
fetchArray,
resolve,
reject,
loadData
);
});
};

// Fetch the actual data arrays
publicAPI.loadData = () => {
const datasetObj = model.dataset;
const arrayToFecth = model.arrays
.filter((array) => array.enable)
.filter((array) => array.array.ref)
.map((array) => array.array);

return new Promise((resolve, reject) => {
const error = (e) => {
reject(e);
};

const processNext = () => {
if (arrayToFecth.length) {
const { progressCallback } = model;
const compression = model.fetchGzip ? 'gz' : null;
fetchArray(arrayToFecth.pop(), {
compression,
progressCallback,
}).then(processNext, error);
} else if (datasetObj) {
// Perform array registration on new arrays
model.arrays
.filter(
(metaArray) => metaArray.registration && !metaArray.array.ref
)
.forEach((metaArray) => {
const newArray = ARRAY_BUILDERS[
metaArray.array.vtkClass
].newInstance(metaArray.array);
datasetObj[`get${macro.capitalize(metaArray.location)}`]()[
metaArray.registration
](newArray);
delete metaArray.registration;
});
datasetObj.modified();
resolve(publicAPI, datasetObj);
}
};

// Start processing queue
processNext();
});
};

publicAPI.requestData = (inData, outData) => {
// do nothing loadData will eventually load up the data
};

// Toggle arrays to load
publicAPI.enableArray = (location, name, enable = true) => {
const activeArray = model.arrays.filter(
(array) => array.name === name && array.location === location
);
if (activeArray.length === 1) {
activeArray[0].enable = enable;
}
};

// return Busy state
publicAPI.isBusy = () => !!model.requestCount;
}

// ----------------------------------------------------------------------------
// Object factory
// ----------------------------------------------------------------------------

const DEFAULT_VALUES = {
enableArray: true,
fetchGzip: false,
arrays: [],
url: null,
baseURL: null,
requestCount: 0,
// dataAccessHelper: null,
};

// ----------------------------------------------------------------------------

export function extend(publicAPI, model, initialValues = {}) {
Object.assign(model, DEFAULT_VALUES, initialValues);

// Build VTK API
macro.obj(publicAPI, model);
macro.get(publicAPI, model, [
'enableArray',
'fetchGzip',
'url',
'baseURL',
'dataAccessHelper',
]);
macro.set(publicAPI, model, ['dataAccessHelper', 'progressCallback']);
macro.getArray(publicAPI, model, ['arrays']);
macro.algo(publicAPI, model, 0, 1);
macro.event(publicAPI, model, 'busy');

// Object methods
vtkHttpDataSetReader(publicAPI, model);

// Make sure we can destructuring progressCallback from model
if (model.progressCallback === undefined) {
model.progressCallback = null;
}
}

// ----------------------------------------------------------------------------

export const newInstance = macro.newInstance(extend, 'vtkHttpDataSetReader');

// ----------------------------------------------------------------------------

export default { newInstance, extend };