XMLReader

Source

index.js

import pako from 'pako';

import DataAccessHelper from 'vtk.js/Sources/IO/Core/DataAccessHelper';
import Base64 from 'vtk.js/Sources/Common/Core/Base64';
import macro from 'vtk.js/Sources/macro';
import vtkDataArray from 'vtk.js/Sources/Common/Core/DataArray';
import BinaryHelper from 'vtk.js/Sources/IO/Core/BinaryHelper';

// ----------------------------------------------------------------------------
// Global methods
// ----------------------------------------------------------------------------

function stringToXML(xmlStr) {
  if (window.ActiveXObject) {
    const oXML = new window.ActiveXObject('Microsoft.XMLDOM');
    oXML.loadXML(xmlStr);
    return oXML;
  }
  return new DOMParser().parseFromString(xmlStr, 'application/xml');
}

function extractAppendedData(buffer) {
  // search for appended data tag
  const prefixRegex = /^\s*<AppendedData\s+encoding="raw">\s*_/m;
  const suffixRegex = /\n\s*<\/AppendedData>/m;
  return BinaryHelper.extractBinary(buffer, prefixRegex, suffixRegex);
}

// ----------------------------------------------------------------------------

const TYPED_ARRAY = {
  Int8: Int8Array,
  UInt8: Uint8Array,
  Int16: Int16Array,
  UInt16: Uint16Array,
  Int32: Int32Array,
  UInt32: Uint32Array,
  Int64: Int32Array, // Not supported with JavaScript will cause error in binary
  UInt64: Uint32Array, // Not supported with JavaScript will cause error in binary
  Float32: Float32Array,
  Float64: Float64Array,
};

// ----------------------------------------------------------------------------

const TYPED_ARRAY_BYTES = {
  Int8: 1,
  UInt8: 1,
  Int16: 2,
  UInt16: 2,
  Int32: 4,
  UInt32: 4,
  Int64: 8, // Not supported with JavaScript will cause error in binary
  UInt64: 8, // Not supported with JavaScript will cause error in binary
  Float32: 4,
  Float64: 8,
};

// ----------------------------------------------------------------------------

function integer64to32(array) {
  const maxIdx = array.length - 1; // Skip last
  return array.filter((v, i) => i < maxIdx && i % 2 === 0);
}

// ----------------------------------------------------------------------------

function readerHeader(uint8, headerType) {
  // We do not handle endianness or if more than 32 bits are needed to encode the data
  if (headerType === 'UInt64') {
    const offset = 8;
    let uint32 = new Uint32Array(uint8.buffer, 0, 6);
    const nbBlocks = uint32[0];
    const s1 = uint32[2];
    const s2 = uint32[4];
    const resultArray = [offset, nbBlocks, s1, s2];
    uint32 = new Uint32Array(uint8.buffer, 3 * 8, nbBlocks * 2);
    for (let i = 0; i < nbBlocks; i++) {
      resultArray.push(uint32[i * 2]);
    }
    return resultArray;
  }
  // UInt32
  let uint32 = new Uint32Array(uint8.buffer, 0, 3);
  const offset = 4;
  const nbBlocks = uint32[0];
  const s1 = uint32[1];
  const s2 = uint32[2];
  const resultArray = [offset, nbBlocks, s1, s2];
  uint32 = new Uint32Array(uint8.buffer, 3 * 4, nbBlocks);
  for (let i = 0; i < nbBlocks; i++) {
    resultArray.push(uint32[i]);
  }
  return resultArray;
}

// ----------------------------------------------------------------------------

function uncompressBlock(compressedUint8, output) {
  const uncompressedBlock = pako.inflate(compressedUint8);
  output.uint8.set(uncompressedBlock, output.offset);
  output.offset += uncompressedBlock.length;
}

// ----------------------------------------------------------------------------

function processDataArray(
  size,
  dataArrayElem,
  compressor,
  byteOrder,
  headerType,
  binaryBuffer
) {
  const dataType = dataArrayElem.getAttribute('type');
  const name = dataArrayElem.getAttribute('Name');
  const format = dataArrayElem.getAttribute('format'); // binary, ascii, appended
  const numberOfComponents = Number(
    dataArrayElem.getAttribute('NumberOfComponents') || '1'
  );
  let values = null;

  if (format === 'ascii') {
    values = new TYPED_ARRAY[dataType](size * numberOfComponents);
    let offset = 0;
    dataArrayElem.firstChild.nodeValue.split(/[\\t \\n]+/).forEach((token) => {
      if (token.trim().length) {
        values[offset++] = Number(token);
      }
    });
  } else if (format === 'binary') {
    const uint8 = new Uint8Array(
      Base64.toArrayBuffer(dataArrayElem.firstChild.nodeValue.trim())
    );
    if (compressor === 'vtkZLibDataCompressor') {
      const buffer = new ArrayBuffer(
        TYPED_ARRAY_BYTES[dataType] * size * numberOfComponents
      );
      values = new TYPED_ARRAY[dataType](buffer);
      const output = {
        offset: 0,
        uint8: new Uint8Array(buffer),
      };
      // ----------------------------------------------------------------------
      // Layout of the data
      // header[N, s1, s1, blockSize1, ..., blockSizeN], [padding???], block[compressedData], ..., block[compressedData]
      // [header] N, s1 and s2 are uint 32 or 64 (defined by header_type="UInt64" attribute on the root node)
      // [header] s1: uncompress size of each block except the last one
      // [header] s2: uncompress size of the last blocks
      // [header] blockSize: size of the block in compressed space that represent to bloc to inflate in zlib. (This also give the offset to the next block)
      // ----------------------------------------------------------------------
      // Header reading
      const header = readerHeader(uint8, headerType);
      const nbBlocks = header[1];
      let offset =
        uint8.length -
        (header.reduce((a, b) => a + b, 0) -
          (header[0] + header[1] + header[2] + header[3]));
      for (let i = 0; i < nbBlocks; i++) {
        const blockSize = header[4 + i];
        const compressedBlock = new Uint8Array(uint8.buffer, offset, blockSize);
        uncompressBlock(compressedBlock, output);
        offset += blockSize;
      }

      // Handle (u)int64 hoping for no overflow...
      if (dataType.indexOf('Int64') !== -1) {
        values = integer64to32(values);
      }
    } else {
      values = new TYPED_ARRAY[dataType](
        uint8.buffer,
        TYPED_ARRAY_BYTES[headerType]
      ); // Skip the count

      // Handle (u)int64 hoping no overflow...
      if (dataType.indexOf('Int64') !== -1) {
        values = integer64to32(values);
      }
    }
  } else if (format === 'appended') {
    let offset = Number(dataArrayElem.getAttribute('offset'));
    // read header
    // NOTE: this will incorrectly read the size if headerType is (U)Int64 and
    // the value requires (U)Int64.
    const header = new TYPED_ARRAY[headerType](binaryBuffer, offset, 1);
    let arraySize = header[0] / TYPED_ARRAY_BYTES[dataType];

    // if we are dealing with Uint64, we need to get double the values since
    // TYPED_ARRAY[Uint64] is Uint32.
    if (dataType.indexOf('Int64') !== -1) {
      arraySize *= 2;
    }

    offset += TYPED_ARRAY_BYTES[headerType];

    // read values
    // if offset is aligned to dataType, use view. Otherwise, slice due to misalignment.
    if (offset % TYPED_ARRAY_BYTES[dataType] === 0) {
      values = new TYPED_ARRAY[dataType](binaryBuffer, offset, arraySize);
    } else {
      values = new TYPED_ARRAY[dataType](
        binaryBuffer.slice(offset, offset + header[0])
      );
    }
    // remove higher order 32 bits assuming they're not used.
    if (dataType.indexOf('Int64') !== -1) {
      values = integer64to32(values);
    }
  } else {
    console.error('Format not supported', format);
  }

  return { name, values, numberOfComponents };
}

// ----------------------------------------------------------------------------

function processCells(
  size,
  containerElem,
  compressor,
  byteOrder,
  headerType,
  binaryBuffer
) {
  const arrayElems = {};
  const dataArrayElems = containerElem.getElementsByTagName('DataArray');
  for (let elIdx = 0; elIdx < dataArrayElems.length; elIdx++) {
    const el = dataArrayElems[elIdx];
    arrayElems[el.getAttribute('Name')] = el;
  }

  const offsets = processDataArray(
    size,
    arrayElems.offsets,
    compressor,
    byteOrder,
    headerType,
    binaryBuffer
  ).values;
  const connectivitySize = offsets[offsets.length - 1];
  const connectivity = processDataArray(
    connectivitySize,
    arrayElems.connectivity,
    compressor,
    byteOrder,
    headerType,
    binaryBuffer
  ).values;
  const values = new Uint32Array(size + connectivitySize);
  let writeOffset = 0;
  let previousOffset = 0;
  offsets.forEach((v) => {
    const cellSize = v - previousOffset;
    values[writeOffset++] = cellSize;

    for (let i = 0; i < cellSize; i++) {
      values[writeOffset++] = connectivity[previousOffset + i];
    }

    // save previous offset
    previousOffset = v;
  });

  return values;
}

// ----------------------------------------------------------------------------

function processFieldData(
  size,
  fieldElem,
  fieldContainer,
  compressor,
  byteOrder,
  headerType,
  binaryBuffer
) {
  if (fieldElem) {
    const attributes = ['Scalars', 'Vectors', 'Normals', 'Tensors', 'TCoords'];
    const nameBinding = {};
    attributes.forEach((attrName) => {
      const arrayName = fieldElem.getAttribute(attrName);
      if (arrayName) {
        nameBinding[arrayName] = fieldContainer[`set${attrName}`];
      }
    });

    const dataArrayElems = fieldElem.getElementsByTagName('DataArray');
    const nbArrays = dataArrayElems.length;
    for (let idx = 0; idx < nbArrays; idx++) {
      const array = dataArrayElems[idx];
      const dataArray = vtkDataArray.newInstance(
        processDataArray(
          size,
          array,
          compressor,
          byteOrder,
          headerType,
          binaryBuffer
        )
      );
      const name = dataArray.getName();
      (nameBinding[name] || fieldContainer.addArray)(dataArray);
    }
  }
}

// ----------------------------------------------------------------------------
// vtkXMLReader methods
// ----------------------------------------------------------------------------

function vtkXMLReader(publicAPI, model) {
  // Set our className
  model.classHierarchy.push('vtkXMLReader');

  // Create default dataAccessHelper if not available
  if (!model.dataAccessHelper) {
    model.dataAccessHelper = DataAccessHelper.get('http');
  }

  // Internal method to fetch Array
  function fetchData(url, option = {}) {
    return model.dataAccessHelper.fetchBinary(url, option);
  }

  // Set DataSet url
  publicAPI.setUrl = (url, option = {}) => {
    model.url = url;

    // Remove the file in the URL
    const path = url.split('/');
    path.pop();
    model.baseURL = path.join('/');

    // Fetch metadata
    return publicAPI.loadData(option);
  };

  // Fetch the actual data arrays
  publicAPI.loadData = (option = {}) =>
    fetchData(model.url, option).then(publicAPI.parseAsArrayBuffer);

  publicAPI.parseAsArrayBuffer = (arrayBuffer) => {
    if (!arrayBuffer) {
      return false;
    }
    if (arrayBuffer !== model.rawDataBuffer) {
      publicAPI.modified();
    } else {
      return true;
    }

    const { text: content, binaryBuffer } = extractAppendedData(arrayBuffer);
    model.rawDataBuffer = arrayBuffer;
    model.binaryBuffer = binaryBuffer;

    // Parse data here...
    const doc = stringToXML(content);
    const rootElem = doc.firstChild;
    const type = rootElem.getAttribute('type');
    const compressor = rootElem.getAttribute('compressor');
    const byteOrder = rootElem.getAttribute('byte_order');
    // default to UInt32. I think version 0.1 vtp/vti files default to UInt32.
    const headerType = rootElem.getAttribute('header_type') || 'UInt32';

    if (compressor && compressor !== 'vtkZLibDataCompressor') {
      console.error('Invalid compressor', compressor);
      return false;
    }

    if (byteOrder && byteOrder !== 'LittleEndian') {
      console.error('Only LittleEndian encoding is supported');
      return false;
    }

    if (type !== model.dataType) {
      console.error('Invalid data type', type, 'expecting', model.dataType);
      return false;
    }

    // appended format
    if (rootElem.querySelector('AppendedData')) {
      const appendedDataElem = rootElem.querySelector('AppendedData');
      const encoding = appendedDataElem.getAttribute('encoding');
      // Only get data arrays that are descendants of <Piece />
      // We don't parse DataArrays from FieldData right now.
      const arrayElems = rootElem.querySelectorAll('Piece DataArray');

      let appendedBuffer = model.binaryBuffer;

      if (encoding === 'base64') {
        // substr(1) is to remove the '_' prefix
        appendedBuffer = appendedDataElem.textContent.trim().substr(1);
      }

      // get data array chunks
      const dataArrays = [];
      for (let i = 0; i < arrayElems.length; ++i) {
        const offset = Number(arrayElems[i].getAttribute('offset'));
        let nextOffset = 0;
        if (i === arrayElems.length - 1) {
          nextOffset = appendedBuffer.length || appendedBuffer.byteLength;
        } else {
          nextOffset = Number(arrayElems[i + 1].getAttribute('offset'));
        }

        if (encoding === 'base64') {
          dataArrays.push(
            new Uint8Array(
              Base64.toArrayBuffer(appendedBuffer.substring(offset, nextOffset))
            )
          );
        } else {
          // encoding === 'raw'
          // Need to slice the ArrayBuffer so readerHeader() works properly
          dataArrays.push(
            new Uint8Array(appendedBuffer.slice(offset, nextOffset))
          );
        }
      }

      if (compressor === 'vtkZLibDataCompressor') {
        for (let arrayidx = 0; arrayidx < dataArrays.length; ++arrayidx) {
          const dataArray = dataArrays[arrayidx];

          // Header reading
          // Refer to processDataArray() above for info on header fields
          const header = readerHeader(dataArray, headerType);
          const nbBlocks = header[1];
          let compressedOffset =
            dataArray.length -
            (header.reduce((a, b) => a + b, 0) -
              (header[0] + header[1] + header[2] + header[3]));

          let buffer = null;
          if (nbBlocks > 0) {
            // If the last block's size is labeled as 0, that means the last block
            // really has size header[2].
            if (header[3] === 0) {
              buffer = new ArrayBuffer(header[2] * nbBlocks);
            } else {
              buffer = new ArrayBuffer(header[2] * (nbBlocks - 1) + header[3]);
            }
          } else {
            // if there is no blocks, then default to a zero array of size 0.
            buffer = new ArrayBuffer(0);
          }

          // uncompressed buffer
          const uncompressed = new Uint8Array(buffer);
          const output = {
            offset: 0,
            uint8: uncompressed,
          };

          for (let i = 0; i < nbBlocks; i++) {
            const blockSize = header[4 + i];
            const compressedBlock = new Uint8Array(
              dataArray.buffer,
              compressedOffset,
              blockSize
            );
            uncompressBlock(compressedBlock, output);
            compressedOffset += blockSize;
          }

          const data = new Uint8Array(
            uncompressed.length + TYPED_ARRAY_BYTES[headerType]
          );
          // set length header
          // TODO this does not work for lengths that are greater than the max Uint32 value.
          new TYPED_ARRAY[headerType](data.buffer, 0, 1)[0] =
            uncompressed.length;
          data.set(uncompressed, TYPED_ARRAY_BYTES[headerType]);

          dataArrays[arrayidx] = data;
        }
      }

      const bufferLength = dataArrays.reduce((acc, arr) => acc + arr.length, 0);
      const buffer = new ArrayBuffer(bufferLength);
      const view = new Uint8Array(buffer);

      for (let i = 0, offset = 0; i < dataArrays.length; ++i) {
        // set correct offsets
        arrayElems[i].setAttribute('offset', offset);
        // set final buffer data
        view.set(dataArrays[i], offset);
        offset += dataArrays[i].length;
      }

      model.binaryBuffer = buffer;

      if (!model.binaryBuffer) {
        console.error(
          'Processing appended data format: requires binaryBuffer to parse'
        );
        return false;
      }
    }

    publicAPI.parseXML(rootElem, type, compressor, byteOrder, headerType);
    return true;
  };

  publicAPI.requestData = (inData, outData) => {
    publicAPI.parseAsArrayBuffer(model.rawDataBuffer);
  };
}

// ----------------------------------------------------------------------------
// Object factory
// ----------------------------------------------------------------------------

const DEFAULT_VALUES = {
  // baseURL: null,
  // dataAccessHelper: null,
  // url: null,
};

// ----------------------------------------------------------------------------

export function extend(publicAPI, model, initialValues = {}) {
  Object.assign(model, DEFAULT_VALUES, initialValues);

  // Build VTK API
  macro.obj(publicAPI, model);
  macro.get(publicAPI, model, ['url', 'baseURL']);
  macro.setGet(publicAPI, model, ['dataAccessHelper']);
  macro.algo(publicAPI, model, 0, 1);

  // vtkXMLReader methods
  vtkXMLReader(publicAPI, model);
}

// ----------------------------------------------------------------------------

export default { extend, processDataArray, processFieldData, processCells };