All files / Sources/IO/Core/HttpDataSetReader index.js

76.07% Statements 124/163
62.06% Branches 36/58
64% Functions 32/50
76.39% Lines 123/161

Press n or j to go to the next uncovered block, b, p or k for the previous block.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447                                1x 1x                 1x 1x 1x   1x                           60x                                                             60x     60x   60x 180x 120x 60x   60x                   120x         60x 60x 60x 60x         60x 60x 28x 28x   32x 32x 32x           60x         60x                   24x     24x     24x 24x         60x 60x   35x       35x 19x 19x       16x 16x     16x 3x       13x 13x 54x         13x 94x       13x 13x 54x         13x 4x 4x 4x 4x       13x 1x     13x       25x 25x   25x 24x       25x 25x         60x       24x 60x                                                                     60x 60x   60x                       24x 60x 60x 60x                   60x     60x     24x       60x       60x   60x 60x                         24x 60x 60x 60x 60x 60x   60x 60x       60x 120x 60x 60x 60x       60x   60x   60x     60x     60x     60x   60x 60x         60x       24x         24x                   24x     24x 5x 3x 3x       24x             1x                             24x     24x 24x               24x         24x 24x 24x     24x     24x 24x           1x          
// For vtk factory
import 'vtk.js/Sources/Common/DataModel/ImageData';
import 'vtk.js/Sources/Common/DataModel/PolyData';
 
import vtk from 'vtk.js/Sources/vtk';
import macro from 'vtk.js/Sources/macros';
import DataAccessHelper from 'vtk.js/Sources/IO/Core/DataAccessHelper';
import vtkDataArray from 'vtk.js/Sources/Common/Core/DataArray';
import vtkStringArray from 'vtk.js/Sources/Common/Core/StringArray';
 
// Enable data soure for DataAccessHelper
import 'vtk.js/Sources/IO/Core/DataAccessHelper/LiteHttpDataAccessHelper'; // Just need HTTP
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/HttpDataAccessHelper'; // HTTP + zip
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/HtmlDataAccessHelper'; // html + base64 + zip
// import 'vtk.js/Sources/IO/Core/DataAccessHelper/JSZipDataAccessHelper'; // zip
 
const fieldDataLocations = ['pointData', 'cellData', 'fieldData'];
const ARRAY_BUILDERS = {
  vtkDataArray,
  vtkStringArray,
};
 
// ----------------------------------------------------------------------------
// Global methods
// ----------------------------------------------------------------------------
 
const cachedArraysAndPromises = {};
const cachedArraysMetaData = {};
const MiB = 1024 * 1024;
 
const GEOMETRY_ARRAYS = {
  vtkPolyData(dataset) {
    const arrayToDownload = [];
    arrayToDownload.push(dataset.points);
    ['verts', 'lines', 'polys', 'strips'].forEach((cellName) => {
      if (dataset[cellName]) {
        arrayToDownload.push(dataset[cellName]);
      }
    });
 
    return arrayToDownload;
  },
 
  vtkImageData(dataset) {
    return [];
  },
 
  vtkUnstructuredGrid(dataset) {
    const arrayToDownload = [];
    arrayToDownload.push(dataset.points);
    arrayToDownload.push(dataset.cells);
    arrayToDownload.push(dataset.cellTypes);
 
    return arrayToDownload;
  },
 
  vtkRectilinearGrid(dataset) {
    const arrayToDownload = [];
    arrayToDownload.push(dataset.xCoordinates);
    arrayToDownload.push(dataset.yCoordinates);
    arrayToDownload.push(dataset.zCoordinates);
 
    return arrayToDownload;
  },
};
 
function processDataSet(
  publicAPI,
  model,
  dataset,
  fetchArray,
  resolve,
  reject,
  loadData
) {
  const enable = model.enableArray;
 
  // Generate array list
  model.arrays = [];
 
  fieldDataLocations.forEach((location) => {
    if (dataset[location]) {
      dataset[location].arrays
        .map((i) => i.data)
        .forEach((array) => {
          model.arrays.push({
            name: array.name,
            enable,
            location,
            array,
            registration: array.ref.registration || 'addArray',
          });
        });
 
      // Reset data arrays
      dataset[location].arrays = [];
    }
  });
 
  // Fetch geometry arrays
  const pendingPromises = [];
  const { progressCallback } = model;
  const compression = model.fetchGzip ? 'gz' : null;
  GEOMETRY_ARRAYS[dataset.vtkClass](dataset).forEach((array) => {
    pendingPromises.push(fetchArray(array, { compression, progressCallback }));
  });
 
  function success() {
    model.dataset = vtk(dataset);
    if (!loadData) {
      model.output[0] = model.dataset;
      resolve(publicAPI, model.output[0]);
    } else {
      publicAPI.loadData().then(() => {
        model.output[0] = model.dataset;
        resolve(publicAPI, model.output[0]);
      });
    }
  }
 
  // Wait for all geometry array to be fetched
  Iif (pendingPromises.length) {
    Promise.all(pendingPromises).then(success, (err) => {
      reject(err);
    });
  } else {
    success();
  }
}
 
// ----------------------------------------------------------------------------
// vtkHttpDataSetReader methods
// ----------------------------------------------------------------------------
 
function vtkHttpDataSetReader(publicAPI, model) {
  // Set our className
  model.classHierarchy.push('vtkHttpDataSetReader');
 
  // Empty output by default
  model.output[0] = vtk({ vtkClass: 'vtkPolyData' });
 
  // Create default dataAccessHelper if not available
  if (!model.dataAccessHelper) {
    model.dataAccessHelper = DataAccessHelper.get('http');
  }
 
  // Internal method to fetch Array
  function fetchArray(array, options = {}) {
    const arrayId = `${array.ref.id}|${array.vtkClass}`;
    if (!cachedArraysAndPromises[arrayId]) {
      // Cache the promise while fetching
      cachedArraysAndPromises[arrayId] = model.dataAccessHelper
        .fetchArray(publicAPI, model.baseURL, array, options)
        .then((newArray) => {
          // Remove promise and return here if caching is disabled
          if (!model.maxCacheSize) {
            delete cachedArraysAndPromises[arrayId];
            return newArray;
          }
 
          // Replace the promise with the array in cache once downloaded
          cachedArraysAndPromises[arrayId] = newArray;
          cachedArraysMetaData[arrayId] = { lastAccess: new Date() };
 
          // If maxCacheSize is set to -1 the cache is unlimited
          if (model.maxCacheSize < 0) {
            return newArray;
          }
 
          // sort cache according to access times (potentially needed for creating space)
          const cachedArrays = {};
          Object.keys(cachedArraysMetaData).forEach((arrId) => {
            cachedArrays[arrId] = {
              array: cachedArraysAndPromises[arrId],
              lastAccess: cachedArraysMetaData[arrId].lastAccess,
            };
          });
          const sortedArrayCache = Object.entries(cachedArrays).sort((a, b) =>
            Math.sign(b[1].lastAccess - a[1].lastAccess)
          );
 
          // Check cache size
          const cacheSizeLimit = model.maxCacheSize * MiB;
          let cacheSize = Object.values(cachedArrays).reduce(
            (accSize, entry) => accSize + entry.array.values.byteLength,
            0
          );
 
          // Delete cache entries until size is below the limit
          while (cacheSize > cacheSizeLimit && sortedArrayCache.length > 0) {
            const [oldId, entry] = sortedArrayCache.pop();
            delete cachedArraysAndPromises[oldId];
            delete cachedArraysMetaData[oldId];
            cacheSize -= entry.array.values.byteLength;
          }
 
          // Edge case: If the new entry is bigger than the cache limit
          if (!cachedArraysMetaData[arrayId]) {
            macro.vtkWarningMacro('Cache size is too small for the dataset');
          }
 
          return newArray;
        });
    } else {
      // cacheArrays[arrayId] can be a promise or value
      Promise.resolve(cachedArraysAndPromises[arrayId]).then((cachedArray) => {
        if (array !== cachedArray) {
          //  Update last access for cache retention rules (if caching is enabled)
          if (model.maxCacheSize) {
            cachedArraysMetaData[arrayId].lastAccess = new Date();
          }
 
          // Assign cached array as result
          Object.assign(array, cachedArray);
          delete array.ref;
        }
      });
    }
 
    return Promise.resolve(cachedArraysAndPromises[arrayId]);
  }
 
  // Fetch dataset (metadata)
  publicAPI.updateMetadata = (loadData = false) => {
    Iif (model.compression === 'zip') {
      return new Promise((resolve, reject) => {
        DataAccessHelper.get('http')
          .fetchBinary(model.url)
          .then(
            (zipContent) => {
              model.dataAccessHelper = DataAccessHelper.get('zip', {
                zipContent,
                callback: (zip) => {
                  model.baseURL = '';
                  model.dataAccessHelper
                    .fetchJSON(publicAPI, 'index.json')
                    .then(
                      (dataset) => {
                        publicAPI
                          .parseObject(dataset, {
                            loadData,
                            deepCopy: false,
                          })
                          .then(resolve, reject);
                      },
                      (error) => {
                        reject(error);
                      }
                    );
                },
              });
            },
            (error) => {
              reject(error);
            }
          );
      });
    }
 
    return new Promise((resolve, reject) => {
      model.dataAccessHelper.fetchJSON(publicAPI, model.url).then(
        (dataset) => {
          publicAPI
            .parseObject(dataset, { loadData, deepCopy: false })
            .then(resolve, reject);
        },
        (error) => {
          reject(error);
        }
      );
    });
  };
 
  // Set DataSet url
  publicAPI.setUrl = (url, options = {}) => {
    if (url.indexOf('index.json') === -1 && !options.fullpath) {
      model.baseURL = url;
      model.url = `${url}/index.json`;
    } else E{
      model.url = url;
 
      // Remove the file in the URL
      const path = url.split('/');
      path.pop();
      model.baseURL = path.join('/');
    }
 
    model.compression = options.compression;
 
    // Fetch metadata
    return publicAPI.updateMetadata(!!options.loadData);
  };
 
  publicAPI.parseObject = (
    manifest,
    { loadData, baseUrl, deepCopy = true }
  ) => {
    Iif (baseUrl) {
      model.baseURL = baseUrl;
    }
 
    const dataset = deepCopy ? structuredClone(manifest) : manifest;
 
    return new Promise((resolve, reject) => {
      processDataSet(
        publicAPI,
        model,
        dataset,
        fetchArray,
        resolve,
        reject,
        loadData
      );
    });
  };
 
  // Fetch the actual data arrays
  publicAPI.loadData = () => {
    const datasetObj = model.dataset;
    const arrayToFecth = model.arrays
      .filter((array) => array.enable)
      .filter((array) => array.array.ref)
      .map((array) => array.array);
 
    return new Promise((resolve, reject) => {
      const error = (e) => {
        reject(e);
      };
 
      const processNext = () => {
        if (arrayToFecth.length) {
          const { progressCallback } = model;
          const compression = model.fetchGzip ? 'gz' : null;
          fetchArray(arrayToFecth.pop(), {
            compression,
            progressCallback,
          }).then(processNext, error);
        } else if (datasetObj) {
          // Perform array registration on new arrays
          model.arrays
            .filter(
              (metaArray) => metaArray.registration && !metaArray.array.ref
            )
            .forEach((metaArray) => {
              const newArray = ARRAY_BUILDERS[
                metaArray.array.vtkClass
              ].newInstance(metaArray.array);
              datasetObj[`get${macro.capitalize(metaArray.location)}`]()[
                metaArray.registration
              ](newArray);
              delete metaArray.registration;
            });
          datasetObj.modified();
          resolve(publicAPI, datasetObj);
        }
      };
 
      // Start processing queue
      processNext();
    });
  };
 
  publicAPI.requestData = (inData, outData) => {
    // do nothing loadData will eventually load up the data
  };
 
  // Toggle arrays to load
  publicAPI.enableArray = (location, name, enable = true) => {
    const activeArray = model.arrays.filter(
      (array) => array.name === name && array.location === location
    );
    if (activeArray.length === 1) {
      activeArray[0].enable = enable;
    }
  };
 
  // return id's of cached arrays
  publicAPI.getCachedArrayIds = () => Object.keys(cachedArraysMetaData);
 
  // clear global array cache
  publicAPI.clearCache = () =>
    Object.keys(cachedArraysMetaData).forEach((k) => {
      delete cachedArraysAndPromises[k];
      delete cachedArraysMetaData[k];
    });
 
  // return Busy state
  publicAPI.isBusy = () => !!model.requestCount;
}
 
// ----------------------------------------------------------------------------
// Object factory
// ----------------------------------------------------------------------------
 
const DEFAULT_VALUES = {
  enableArray: true,
  fetchGzip: false,
  arrays: [],
  url: null,
  baseURL: null,
  requestCount: 0,
  arrayCachingEnabled: true,
  maxCacheSize: 2048,
  // dataAccessHelper: null,
};
 
// ----------------------------------------------------------------------------
 
export function extend(publicAPI, model, initialValues = {}) {
  Object.assign(model, DEFAULT_VALUES, initialValues);
 
  // Build VTK API
  macro.obj(publicAPI, model);
  macro.get(publicAPI, model, [
    'enableArray',
    'fetchGzip',
    'url',
    'baseURL',
    'dataAccessHelper',
    'maxCacheSize',
  ]);
  macro.set(publicAPI, model, [
    'dataAccessHelper',
    'progressCallback',
    'maxCacheSize',
  ]);
  macro.getArray(publicAPI, model, ['arrays']);
  macro.algo(publicAPI, model, 0, 1);
  macro.event(publicAPI, model, 'busy');
 
  // Object methods
  vtkHttpDataSetReader(publicAPI, model);
 
  // Make sure we can destructuring progressCallback from model
  if (model.progressCallback === undefined) {
    model.progressCallback = null;
  }
}
 
// ----------------------------------------------------------------------------
 
export const newInstance = macro.newInstance(extend, 'vtkHttpDataSetReader');
 
// ----------------------------------------------------------------------------
 
export default { newInstance, extend };