SortedCompositeImageBuilder

Source

index.js
import AbstractImageBuilder from '../AbstractImageBuilder';
import CPUCompositor from './sorted-compositor-cpu';
import EqualizerModel from '../../../Common/State/EqualizerState';
import GPUCompositor from './sorted-compositor-gpu';
import ToggleModel from '../../../Common/State/ToggleState';

import '../../../React/CollapsibleControls/CollapsibleControlFactory/VolumeControlWidget';
import '../../../React/CollapsibleControls/CollapsibleControlFactory/QueryDataModelWidget';

const LUT_NAME = 'VolumeScalar';

export default class SortedCompositeImageBuilder extends AbstractImageBuilder {
// ------------------------------------------------------------------------

constructor(queryDataModel, lookupTableManager) {
super({
queryDataModel,
lookupTableManager,
dimensions: queryDataModel.originalData.SortedComposite.dimensions,
});

this.dataQuery = {
name: 'data_fetch',
categories: [],
};
this.metadata = queryDataModel.originalData.SortedComposite;

// Add Lut
this.originalRange = [
this.metadata.scalars[0],
this.metadata.scalars[this.metadata.scalars.length - 1],
];
this.lutTextureData = new Uint8Array(this.metadata.layers * 4);
lookupTableManager.addFields(
{ VolumeScalar: [0, 1] },
this.queryDataModel.originalData.LookupTables
);
this.lookupTable = lookupTableManager.getLookupTable(LUT_NAME);
this.registerSubscription(
this.lookupTable.onChange((data, envelope) => {
for (let idx = 0; idx < this.metadata.layers; idx++) {
const color = this.lookupTable.getColor(this.metadata.scalars[idx]);

this.lutTextureData[idx * 4] = color[0] * 255;
this.lutTextureData[idx * 4 + 1] = color[1] * 255;
this.lutTextureData[idx * 4 + 2] = color[2] * 255;
}
this.render();
})
);

this.compositors = [
new CPUCompositor(
queryDataModel,
this,
this.lutTextureData,
this.metadata.reverseCompositePass
),
new GPUCompositor(
queryDataModel,
this,
this.lutTextureData,
this.metadata.reverseCompositePass
),
];
this.compositor = this.compositors[1];

this.intensityModel = new ToggleModel(true);
this.computationModel = new ToggleModel(true); // true: GPU / false: CPU
this.equalizerModel = new EqualizerModel({
size: this.metadata.layers,
scalars: this.metadata.scalars,
lookupTable: this.lookupTable,
});

this.intensityModel.onChange((data, envelope) => {
this.update();
});
this.computationModel.onChange((data, envelope) => {
this.compositor = this.compositors[data ? 1 : 0];
this.update();
});
this.equalizerModel.onChange((data, envelope) => {
const opacities = data.getOpacities();
for (let idx = 0; idx < this.metadata.layers; idx++) {
this.lutTextureData[idx * 4 + 3] = opacities[idx] * 255;
}
this.render();
});

// Force the filling of the color texture
this.lookupTable.setScalarRange(
this.originalRange[0],
this.originalRange[1]
);

// Relay normal data fetch to query based on
this.registerSubscription(
this.queryDataModel.onDataChange(() => {
this.update();
})
);

this.registerSubscription(
queryDataModel.on('data_fetch', (data, envelope) => {
this.compositor.updateData(data);
this.render();
})
);

// Handle destroy
this.registerObjectToFree(this.compositors[0]);
this.registerObjectToFree(this.compositors[1]);
this.registerObjectToFree(this.intensityModel);
this.registerObjectToFree(this.computationModel);
this.registerObjectToFree(this.equalizerModel);
}

// ------------------------------------------------------------------------

update() {
if (this.intensityModel.getState()) {
this.dataQuery.categories = ['_', 'intensity'];
} else {
this.dataQuery.categories = ['_'];
}

this.queryDataModel.fetchData(this.dataQuery);
}

// ------------------------------------------------------------------------

render() {
this.compositor.render();
}

// ------------------------------------------------------------------------

destroy() {
super.destroy();

this.compositor = null;
this.compositors = null;
this.computationModel = null;
this.dataQuery = null;
this.equalizerModel = null;
this.intensityModel = null;
this.lookupTable = null;
this.lutTextureData = null;
this.metadata = null;
this.originalRange = null;
}

// ------------------------------------------------------------------------

getControlWidgets() {
const {
lookupTable,
equalizer,
intensity,
computation,
queryDataModel,
} = this.getControlModels();
return [
{
name: 'VolumeControlWidget',
lookupTable,
equalizer,
intensity,
computation,
},
{
name: 'QueryDataModelWidget',
queryDataModel,
},
];
}

// ------------------------------------------------------------------------

getControlModels() {
return {
lookupTable: {
lookupTable: this.lookupTable,
lookupTableManager: this.lookupTableManager,
originalRange: this.originalRange,
},
equalizer: this.equalizerModel,
intensity: this.intensityModel,
computation: this.computationModel,
queryDataModel: this.queryDataModel,
dimensions: this.metadata.dimensions,
};
}
}
sorted-compositor-cpu.js
import CanvasOffscreenBuffer from '../../../Common/Misc/CanvasOffscreenBuffer';
import { loop } from '../../../Common/Misc/Loop';

export default class SortedVolumeCompositor {
constructor(queryDataModel, imageBuilder, colorTable, reverseCompositePass) {
this.queryDataModel = queryDataModel;
this.imageBuilder = imageBuilder;
this.metadata = this.queryDataModel.originalData.SortedComposite;
this.orderData = null;
this.alphaData = null;
this.intensityData = null;
this.numLayers = this.metadata.layers;
this.colorTable = colorTable;
this.reverseCompositePass = reverseCompositePass;

this.width = this.metadata.dimensions[0];
this.height = this.metadata.dimensions[1];
this.bgCanvas = new CanvasOffscreenBuffer(this.width, this.height);
this.imageBuffer = this.bgCanvas
.get2DContext()
.createImageData(this.width, this.height);
}

// --------------------------------------------------------------------------

updateData(data) {
this.orderData = new Uint8Array(data.order.data);
this.alphaData = new Uint8Array(data.alpha.data);
if (data.intensity) {
this.intensityData = new Uint8Array(data.intensity.data);
} else {
this.intensityData = null;
}
}

// --------------------------------------------------------------------------

setLayerColors(colorTable) {
this.colorTable = colorTable;
}

// --------------------------------------------------------------------------

render() {
if (!this.alphaData || !this.orderData || !this.colorTable) {
return;
}

const imageSize = this.width * this.height;
const pixels = this.imageBuffer.data;
const height = this.height;
const width = this.width;
const ctx = this.bgCanvas.get2DContext();

// Reset pixels
if (pixels.fill) {
pixels.fill(0);
} else {
let count = width * height * 4;
while (count) {
count -= 1;
pixels[count] = 0;
}
}

// Just iterate through all the layers in the data for now
loop(!!this.reverseCompositePass, this.numLayers, (drawIdx) => {
for (let y = 0; y < this.height; y++) {
for (let x = 0; x < this.width; x++) {
const idx = this.width * y + x;
const flipIdx = (height - y - 1) * width + x;
const layerIdx = this.orderData[drawIdx * imageSize + idx];
const multiplier = this.colorTable[layerIdx * 4 + 3] / 255.0;
const alphB = this.alphaData[drawIdx * imageSize + idx] / 255.0;

let intensity = 1.0;

if (this.intensityData) {
intensity = this.intensityData[drawIdx * imageSize + idx] / 255.0;
}

// Blend
const alphA = pixels[flipIdx * 4 + 3] / 255.0;
const alphANeg = 1.0 - alphA;
const rgbA = [
pixels[flipIdx * 4],
pixels[flipIdx * 4 + 1],
pixels[flipIdx * 4 + 2],
];
const rgbB = [
this.colorTable[layerIdx * 4] *
intensity *
alphB *
multiplier *
alphANeg,
this.colorTable[layerIdx * 4 + 1] *
intensity *
alphB *
multiplier *
alphANeg,
this.colorTable[layerIdx * 4 + 2] *
intensity *
alphB *
multiplier *
alphANeg,
];
const alphOut = alphA + alphB * multiplier * (1.0 - alphA);

pixels[flipIdx * 4] = (rgbA[0] * alphA + rgbB[0]) / alphOut;
pixels[flipIdx * 4 + 1] = (rgbA[1] * alphA + rgbB[1]) / alphOut;
pixels[flipIdx * 4 + 2] = (rgbA[2] * alphA + rgbB[2]) / alphOut;
pixels[flipIdx * 4 + 3] = alphOut * 255.0;
}
}
});

// Draw the result to the canvas
ctx.putImageData(this.imageBuffer, 0, 0);

const readyImage = {
canvas: this.bgCanvas.el,
area: [0, 0, this.width, this.height],
outputSize: [this.width, this.height],
builder: this.imageBuilder,
};

this.imageBuilder.imageReady(readyImage);
}

// --------------------------------------------------------------------------

destroy() {
this.bgCanvas.destroy();
this.bgCanvas = null;

this.queryDataModel = null;
this.imageBuilder = null;
}
}
sorted-compositor-gpu.js
import CanvasOffscreenBuffer from '../../../Common/Misc/CanvasOffscreenBuffer';
import WebGlUtil from '../../../Common/Misc/WebGl';
import { loop } from '../../../Common/Misc/Loop';
import PingPong from '../../../Common/Misc/PingPong';

import vertexShader from '../../../Common/Misc/WebGl/shaders/vertex/basic.c';
import fragmentShaderDisplay from './shaders/fragment/display.c';
import fragmentShaderColor from './shaders/fragment/rgbaColor.c';
import fragmentShaderAlpha from './shaders/fragment/alphaBlend.c';

const texParameter = [
['TEXTURE_MAG_FILTER', 'NEAREST'],
['TEXTURE_MIN_FILTER', 'NEAREST'],
['TEXTURE_WRAP_S', 'CLAMP_TO_EDGE'],
['TEXTURE_WRAP_T', 'CLAMP_TO_EDGE'],
];
const pixelStore = [['UNPACK_FLIP_Y_WEBGL', true]];
const align1PixelStore = [
['UNPACK_FLIP_Y_WEBGL', true],
['UNPACK_ALIGNMENT', 1],
];

export default class WebGLSortedVolumeCompositor {
constructor(queryDataModel, imageBuilder, colorTable, reverseCompositePass) {
this.queryDataModel = queryDataModel;
this.imageBuilder = imageBuilder;
this.infoJson = this.queryDataModel.originalData;
this.orderData = null;
this.alphaData = null;
this.intensityData = null;
this.intensitySize = 1;
this.defaultIntensityData = new Uint8Array([255]);
this.numLayers = this.infoJson.SortedComposite.layers;
this.reverseCompositePass = reverseCompositePass;

this.lutView = colorTable;

this.width = this.infoJson.SortedComposite.dimensions[0];
this.height = this.infoJson.SortedComposite.dimensions[1];
this.glCanvas = new CanvasOffscreenBuffer(this.width, this.height);

// Inialize GL context
this.gl = this.glCanvas.get3DContext();
if (!this.gl) {
console.error('Unable to get WebGl context');
return;
}

// Set clear color to white, fully transparent
this.gl.clearColor(1.0, 1.0, 1.0, 0.0);

// Set up GL resources
this.glConfig = {
programs: {
displayProgram: {
vertexShader,
fragmentShader: fragmentShaderDisplay,
mapping: 'default',
},
colorProgram: {
vertexShader,
fragmentShader: fragmentShaderColor,
mapping: 'default',
},
blendProgram: {
vertexShader,
fragmentShader: fragmentShaderAlpha,
mapping: 'default',
},
},
resources: {
buffers: [
{
id: 'texCoord',
data: new Float32Array([
0.0,
0.0,
1.0,
0.0,
0.0,
1.0,
0.0,
1.0,
1.0,
0.0,
1.0,
1.0,
]),
},
{
id: 'posCoord',
data: new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]),
},
],
textures: [
{
id: 'orderTexture',
pixelStore: align1PixelStore,
texParameter,
},
{
id: 'alphaTexture',
pixelStore: align1PixelStore,
texParameter,
},
{
id: 'intensityTexture',
pixelStore: align1PixelStore,
texParameter,
},
{
id: 'lutTexture',
pixelStore,
texParameter,
},
{
id: 'ping',
pixelStore,
texParameter,
},
{
id: 'pong',
pixelStore,
texParameter,
},
{
id: 'colorRenderTexture',
pixelStore,
texParameter,
},
],
framebuffers: [
{
id: 'ping',
width: this.width,
height: this.height,
},
{
id: 'pong',
width: this.width,
height: this.height,
},
{
id: 'colorFbo',
width: this.width,
height: this.height,
},
],
},
mappings: {
default: [
{
id: 'posCoord',
name: 'positionLocation',
attribute: 'a_position',
format: [2, this.gl.FLOAT, false, 0, 0],
},
{
id: 'texCoord',
name: 'texCoordLocation',
attribute: 'a_texCoord',
format: [2, this.gl.FLOAT, false, 0, 0],
},
],
},
};

this.glResources = WebGlUtil.createGLResources(this.gl, this.glConfig);

WebGlUtil.bindTextureToFramebuffer(
this.gl,
this.glResources.framebuffers.colorFbo,
this.glResources.textures.colorRenderTexture
);

this.pingPong = new PingPong(
this.gl,
[this.glResources.framebuffers.ping, this.glResources.framebuffers.pong],
[this.glResources.textures.ping, this.glResources.textures.pong]
);
}

// --------------------------------------------------------------------------

updateData(data) {
this.orderData = data.order.data;
this.alphaData = data.alpha.data;
if (data.intensity) {
this.intensityData = data.intensity.data;
this.intensitySize = [this.width, this.height];
} else {
this.intensityData = this.defaultIntensityData;
this.intensitySize = [1, 1];
}
}

// --------------------------------------------------------------------------

extractLayerData(buffer, layerIndex) {
const offset = layerIndex * this.width * this.height;
const length = this.width * this.height;

return new Uint8Array(buffer, offset, length);
}

// --------------------------------------------------------------------------

render() {
if (!this.alphaData || !this.orderData || !this.lutView) {
return;
}

// Clear the ping pong fbo
this.pingPong.clearFbo();

// Just iterate through all the layers in the data for now
loop(!this.reverseCompositePass, this.numLayers, (layerIdx) => {
this.drawColorPass(
this.extractLayerData(this.orderData, layerIdx),
this.extractLayerData(this.alphaData, layerIdx),
this.extractLayerData(this.intensityData, layerIdx)
);
this.drawBlendPass();
});

// Draw the result to the gl canvas
this.drawDisplayPass();

const readyImage = {
canvas: this.glCanvas.el,
area: [0, 0, this.width, this.height],
outputSize: [this.width, this.height],
builder: this.imageBuilder,
};

this.imageBuilder.imageReady(readyImage);
}

// --------------------------------------------------------------------------

destroy() {
this.glResources.destroy();
this.glResources = null;

this.pingPong = null;
}

// --------------------------------------------------------------------------

drawDisplayPass() {
// Draw to the screen framebuffer
this.gl.bindFramebuffer(this.gl.FRAMEBUFFER, null);

// Using the display shader program
this.gl.useProgram(this.glResources.programs.displayProgram);

this.gl.clear(this.gl.COLOR_BUFFER_BIT);
this.gl.viewport(0, 0, this.width, this.height);

// Set up the sampler uniform and bind the rendered texture
const uImage = this.gl.getUniformLocation(
this.glResources.programs.displayProgram,
'u_image'
);
this.gl.uniform1i(uImage, 0);
this.gl.activeTexture(this.gl.TEXTURE0 + 0);
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.pingPong.getRenderingTexture()
);

// Draw the rectangle.
this.gl.drawArrays(this.gl.TRIANGLES, 0, 6);

this.gl.finish();

// Unbind the single texture we used
this.gl.activeTexture(this.gl.TEXTURE0);
this.gl.bindTexture(this.gl.TEXTURE_2D, null);
}

// --------------------------------------------------------------------------

drawBlendPass() {
// Draw to the ping pong fbo on this pass
this.gl.bindFramebuffer(
this.gl.FRAMEBUFFER,
this.pingPong.getFramebuffer()
);

// Use the alpha blending program for this pass
this.gl.useProgram(this.glResources.programs.blendProgram);

this.gl.viewport(0, 0, this.width, this.height);

// Set up the ping pong render texture as the 'under' layer
const under = this.gl.getUniformLocation(
this.glResources.programs.blendProgram,
'underLayerSampler'
);
this.gl.uniform1i(under, 0);
this.gl.activeTexture(this.gl.TEXTURE0 + 0);
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.pingPong.getRenderingTexture()
);

// Set up the color fbo render texture as the 'over' layer
const over = this.gl.getUniformLocation(
this.glResources.programs.blendProgram,
'overLayerSampler'
);
this.gl.uniform1i(over, 1);
this.gl.activeTexture(this.gl.TEXTURE0 + 1);
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.glResources.textures.colorRenderTexture
);

// Draw the rectangle.
this.gl.drawArrays(this.gl.TRIANGLES, 0, 6);

this.gl.finish();

// Ping-pong
this.pingPong.swap();

// Now unbind the textures we used
for (let i = 0; i < 2; i += 1) {
this.gl.activeTexture(this.gl.TEXTURE0 + i);
this.gl.bindTexture(this.gl.TEXTURE_2D, null);
}
}

// --------------------------------------------------------------------------

drawColorPass(layerOrderData, layerAlphaData, layerIntensityData) {
// Draw to the color fbo on this pass
this.gl.bindFramebuffer(
this.gl.FRAMEBUFFER,
this.glResources.framebuffers.colorFbo
);

// Using the coloring shader program
this.gl.useProgram(this.glResources.programs.colorProgram);

// this.gl.clear(this.gl.COLOR_BUFFER_BIT);
this.gl.viewport(0, 0, this.width, this.height);

// Send uniform specifying the number of total layers
const numLayersLoc = this.gl.getUniformLocation(
this.glResources.programs.colorProgram,
'numberOfLayers'
);
this.gl.uniform1f(numLayersLoc, this.numLayers);

let texCount = 0;

// Set up the order layer texture
const orderLayer = this.gl.getUniformLocation(
this.glResources.programs.colorProgram,
'orderSampler'
);
this.gl.uniform1i(orderLayer, texCount);
this.gl.activeTexture(this.gl.TEXTURE0 + texCount);
texCount += 1;
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.glResources.textures.orderTexture
);
this.gl.texImage2D(
this.gl.TEXTURE_2D,
0,
this.gl.LUMINANCE,
this.width,
this.height,
0,
this.gl.LUMINANCE,
this.gl.UNSIGNED_BYTE,
layerOrderData
);

// Set up the alpha sprite texture
const alphaSpriteLoc = this.gl.getUniformLocation(
this.glResources.programs.colorProgram,
'alphaSampler'
);
this.gl.uniform1i(alphaSpriteLoc, texCount);
this.gl.activeTexture(this.gl.TEXTURE0 + texCount);
texCount += 1;
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.glResources.textures.alphaTexture
);
this.gl.texImage2D(
this.gl.TEXTURE_2D,
0,
this.gl.LUMINANCE,
this.width,
this.height,
0,
this.gl.LUMINANCE,
this.gl.UNSIGNED_BYTE,
layerAlphaData
);

// Set up the intensity sprite texture
const intensitySpriteLoc = this.gl.getUniformLocation(
this.glResources.programs.colorProgram,
'intensitySampler'
);
this.gl.uniform1i(intensitySpriteLoc, texCount);
this.gl.activeTexture(this.gl.TEXTURE0 + texCount);
texCount += 1;
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.glResources.textures.intensityTexture
);
this.gl.texImage2D(
this.gl.TEXTURE_2D,
0,
this.gl.LUMINANCE,
this.intensitySize[0],
this.intensitySize[1],
0,
this.gl.LUMINANCE,
this.gl.UNSIGNED_BYTE,
layerIntensityData
);

// Set up the lookup texture (contains alphas to multiply each layer color)
const lutLoc = this.gl.getUniformLocation(
this.glResources.programs.colorProgram,
'lutSampler'
);
this.gl.uniform1i(lutLoc, texCount);
this.gl.activeTexture(this.gl.TEXTURE0 + texCount);
texCount += 1;
this.gl.bindTexture(
this.gl.TEXTURE_2D,
this.glResources.textures.lutTexture
);
this.gl.texImage2D(
this.gl.TEXTURE_2D,
0,
this.gl.RGBA,
this.numLayers,
1,
0,
this.gl.RGBA,
this.gl.UNSIGNED_BYTE,
this.lutView
);

// Draw the rectangle.
this.gl.drawArrays(this.gl.TRIANGLES, 0, 6);

this.gl.finish();

// Now unbind the textures we used
for (let i = 0; i < texCount; i += 1) {
this.gl.activeTexture(this.gl.TEXTURE0 + i);
this.gl.bindTexture(this.gl.TEXTURE_2D, null);
}
}
}