From aa9efda5648bdba0445f458b0a754f06c95fc9fd Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 24 Jul 2024 17:25:15 +0200 Subject: [PATCH] feat(transfer-function): show data CDF in transfer function (#607) * feat: mark transfer function as needing histogram * feat: mark bounds for transfer function histogram * refactor: pull histogram calculation into function * feat: link up a transfer function CDF line shader * fix: linting * fix(log): correect console.log of histogram from VR * fix: correct histogram index when invlerp and tfs are mixed * feat: change transfer function line colours * refactor: define CDF shader as a function --- src/volume_rendering/volume_render_layer.ts | 2 +- src/webgl/shader_ui_controls.ts | 11 +- src/widget/invlerp.ts | 105 ++++++++++---------- src/widget/shader_controls.ts | 55 ++++++---- src/widget/transfer_function.ts | 77 +++++++++++++- 5 files changed, 171 insertions(+), 79 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 2eb4325d6..f7b68bfed 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -1253,7 +1253,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); for (let j = 0; j < 256; ++j) { tempBuffer2[j] = tempBuffer[j * 4]; } - console.log("histogram%d", i, tempBuffer2.join(" ")); + console.log(`histogram${i}`, tempBuffer2.join(" ")); } } endHistogramShader(); diff --git a/src/webgl/shader_ui_controls.ts b/src/webgl/shader_ui_controls.ts index b046fb3b1..98ebde8a8 100644 --- a/src/webgl/shader_ui_controls.ts +++ b/src/webgl/shader_ui_controls.ts @@ -1418,7 +1418,11 @@ export class ShaderControlState (state) => { const channels: HistogramChannelSpecification[] = []; for (const { control, trackable } of state.values()) { - if (control.type !== "imageInvlerp") continue; + if ( + control.type !== "imageInvlerp" && + control.type !== "transferFunction" + ) + continue; channels.push({ channel: trackable.value.channel }); } return channels; @@ -1444,7 +1448,10 @@ export class ShaderControlState const histogramBounds = makeCachedLazyDerivedWatchableValue((state) => { const bounds: DataTypeInterval[] = []; for (const { control, trackable } of state.values()) { - if (control.type === "imageInvlerp") { + if ( + control.type === "imageInvlerp" || + control.type === "transferFunction" + ) { bounds.push(trackable.value.window); } else if (control.type === "propertyInvlerp") { const { dataType, range, window } = diff --git a/src/widget/invlerp.ts b/src/widget/invlerp.ts index 263f620b7..336ab3c4a 100644 --- a/src/widget/invlerp.ts +++ b/src/widget/invlerp.ts @@ -49,6 +49,7 @@ import { Uint64 } from "#src/util/uint64.js"; import { getWheelZoomAmount } from "#src/util/wheel_zoom.js"; import type { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js"; import { getMemoizedBuffer } from "#src/webgl/buffer.js"; +import type { GL } from "#src/webgl/context.js"; import type { ParameterizedEmitterDependentShaderGetter } from "#src/webgl/dynamic_shader.js"; import { parameterizedEmitterDependentShaderGetter } from "#src/webgl/dynamic_shader.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; @@ -78,6 +79,55 @@ const inputEventMap = EventActionMap.fromObject({ "shift?+wheel": { action: "zoom-via-wheel" }, }); +export function createCDFLineShader(gl: GL, textureUnit: symbol) { + const builder = new ShaderBuilder(gl); + defineLineShader(builder); + builder.addTextureSampler("sampler2D", "uHistogramSampler", textureUnit); + builder.addOutputBuffer("vec4", "out_color", 0); + builder.addAttribute("uint", "aDataValue"); + builder.addUniform("float", "uBoundsFraction"); + builder.addVertexCode(` +float getCount(int i) { + return texelFetch(uHistogramSampler, ivec2(i, 0), 0).x; +} +vec4 getVertex(float cdf, int i) { + float x; + if (i == 0) { + x = -1.0; + } else if (i == 255) { + x = 1.0; + } else { + x = float(i) / 254.0 * uBoundsFraction * 2.0 - 1.0; + } + return vec4(x, cdf * (2.0 - uLineParams.y) - 1.0 + uLineParams.y * 0.5, 0.0, 1.0); +} +`); + builder.setVertexMain(` +int lineNumber = int(aDataValue); +int dataValue = lineNumber; +float cumSum = 0.0; +for (int i = 0; i <= dataValue; ++i) { + cumSum += getCount(i); +} +float total = cumSum + getCount(dataValue + 1); +float cumSumEnd = dataValue == ${NUM_CDF_LINES - 1} ? cumSum : total; +if (dataValue == ${NUM_CDF_LINES - 1}) { + cumSum + getCount(dataValue + 1); +} +for (int i = dataValue + 2; i < 256; ++i) { + total += getCount(i); +} +total = max(total, 1.0); +float cdf1 = cumSum / total; +float cdf2 = cumSumEnd / total; +emitLine(getVertex(cdf1, lineNumber), getVertex(cdf2, lineNumber + 1), 1.0); +`); + builder.setFragmentMain(` +out_color = vec4(0.0, 1.0, 1.0, getLineAlpha()); +`); + return builder.build(); +} + export class CdfController< T extends RangeAndWindowIntervals, > extends RefCounted { @@ -287,7 +337,7 @@ export function getUpdatedRangeAndWindowParameters< // 256 bins in total. The first and last bin are for values below the lower bound/above the upper // bound. const NUM_HISTOGRAM_BINS_IN_RANGE = 254; -const NUM_CDF_LINES = NUM_HISTOGRAM_BINS_IN_RANGE + 1; +export const NUM_CDF_LINES = NUM_HISTOGRAM_BINS_IN_RANGE + 1; /** * Panel that shows Cumulative Distribution Function (CDF) of visible data. @@ -325,58 +375,7 @@ class CdfPanel extends IndirectRenderedPanel { ).value; private lineShader = this.registerDisposer( - (() => { - const builder = new ShaderBuilder(this.gl); - defineLineShader(builder); - builder.addTextureSampler( - "sampler2D", - "uHistogramSampler", - histogramSamplerTextureUnit, - ); - builder.addOutputBuffer("vec4", "out_color", 0); - builder.addAttribute("uint", "aDataValue"); - builder.addUniform("float", "uBoundsFraction"); - builder.addVertexCode(` -float getCount(int i) { - return texelFetch(uHistogramSampler, ivec2(i, 0), 0).x; -} -vec4 getVertex(float cdf, int i) { - float x; - if (i == 0) { - x = -1.0; - } else if (i == 255) { - x = 1.0; - } else { - x = float(i) / 254.0 * uBoundsFraction * 2.0 - 1.0; - } - return vec4(x, cdf * (2.0 - uLineParams.y) - 1.0 + uLineParams.y * 0.5, 0.0, 1.0); -} -`); - builder.setVertexMain(` -int lineNumber = int(aDataValue); -int dataValue = lineNumber; -float cumSum = 0.0; -for (int i = 0; i <= dataValue; ++i) { - cumSum += getCount(i); -} -float total = cumSum + getCount(dataValue + 1); -float cumSumEnd = dataValue == ${NUM_CDF_LINES - 1} ? cumSum : total; -if (dataValue == ${NUM_CDF_LINES - 1}) { - cumSum + getCount(dataValue + 1); -} -for (int i = dataValue + 2; i < 256; ++i) { - total += getCount(i); -} -total = max(total, 1.0); -float cdf1 = cumSum / total; -float cdf2 = cumSumEnd / total; -emitLine(getVertex(cdf1, lineNumber), getVertex(cdf2, lineNumber + 1), 1.0); -`); - builder.setFragmentMain(` -out_color = vec4(0.0, 1.0, 1.0, getLineAlpha()); -`); - return builder.build(); - })(), + (() => createCDFLineShader(this.gl, histogramSamplerTextureUnit))(), ); private regionCornersBuffer = getSquareCornersBuffer(this.gl, 0, -1, 1, 1); diff --git a/src/widget/shader_controls.ts b/src/widget/shader_controls.ts index aa19a7719..f9a392143 100644 --- a/src/widget/shader_controls.ts +++ b/src/widget/shader_controls.ts @@ -75,16 +75,6 @@ function getShaderLayerControlFactory( case "checkbox": return checkboxLayerControl(() => controlState.trackable); case "imageInvlerp": { - let histogramIndex = 0; - for (const [ - otherName, - { - control: { type: otherType }, - }, - ] of shaderControlState.state) { - if (otherName === controlId) break; - if (otherType === "imageInvlerp") ++histogramIndex; - } return channelInvlerpLayerControl(() => ({ dataType: control.dataType, defaultChannel: control.default.channel, @@ -92,26 +82,16 @@ function getShaderLayerControlFactory( channelCoordinateSpaceCombiner: shaderControlState.channelCoordinateSpaceCombiner, histogramSpecifications: shaderControlState.histogramSpecifications, - histogramIndex, + histogramIndex: calculateHistogramIndex(), legendShaderOptions: layerShaderControls.legendShaderOptions, })); } case "propertyInvlerp": { - let histogramIndex = 0; - for (const [ - otherName, - { - control: { type: otherType }, - }, - ] of shaderControlState.state) { - if (otherName === controlId) break; - if (otherType === "propertyInvlerp") ++histogramIndex; - } return propertyInvlerpLayerControl(() => ({ properties: control.properties, watchableValue: controlState.trackable, histogramSpecifications: shaderControlState.histogramSpecifications, - histogramIndex, + histogramIndex: calculateHistogramIndex(), legendShaderOptions: layerShaderControls.legendShaderOptions, })); } @@ -122,9 +102,40 @@ function getShaderLayerControlFactory( channelCoordinateSpaceCombiner: shaderControlState.channelCoordinateSpaceCombiner, defaultChannel: control.default.channel, + histogramSpecifications: shaderControlState.histogramSpecifications, + histogramIndex: calculateHistogramIndex(), })); } } + + function calculateHistogramIndex(controlType: string = control.type) { + const isMatchingControlType = (otherControlType: string) => { + if ( + controlType === "imageInvlerp" || + controlType === "transferFunction" + ) { + return ( + otherControlType === "imageInvlerp" || + otherControlType === "transferFunction" + ); + } else if (controlType === "propertyInvlerp") { + return otherControlType === "propertyInvlerp"; + } else { + throw new Error(`${controlType} does not support histogram index.`); + } + }; + let histogramIndex = 0; + for (const [ + otherName, + { + control: { type: otherType }, + }, + ] of shaderControlState.state) { + if (otherName === controlId) break; + if (isMatchingControlType(otherType)) histogramIndex++; + } + return histogramIndex; + } } function getShaderLayerControlDefinition( diff --git a/src/widget/transfer_function.ts b/src/widget/transfer_function.ts index 1d47f3581..3ac1f21e1 100644 --- a/src/widget/transfer_function.ts +++ b/src/widget/transfer_function.ts @@ -41,6 +41,7 @@ import { computeInvlerp, computeLerp, defaultDataTypeRange, + getIntervalBoundsEffectiveFraction, parseDataTypeValue, } from "#src/util/lerp.js"; import { MouseEventBinder } from "#src/util/mouse_bindings.js"; @@ -50,6 +51,7 @@ import type { WatchableVisibilityPriority } from "#src/visibility_priority/front import type { Buffer } from "#src/webgl/buffer.js"; import { getMemoizedBuffer } from "#src/webgl/buffer.js"; import type { GL } from "#src/webgl/context.js"; +import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; import { defineInvlerpShaderFunction, enableLerpShaderFunction, @@ -71,6 +73,8 @@ import { getUpdatedRangeAndWindowParameters, updateInputBoundValue, updateInputBoundWidth, + createCDFLineShader, + NUM_CDF_LINES, } from "#src/widget/invlerp.js"; import type { LayerControlFactory, @@ -88,6 +92,7 @@ const TRANSFER_FUNCTION_BORDER_WIDTH = 0.05; const transferFunctionSamplerTextureUnit = Symbol( "transferFunctionSamplerTexture", ); +const histogramSamplerTextureUnit = Symbol("histogramSamplerTexture"); const defaultTransferFunctionSizes: Record = { [DataType.UINT8]: 256, @@ -631,6 +636,18 @@ class TransferFunctionPanel extends IndirectRenderedPanel { }, ), ); + private dataValuesBuffer = this.registerDisposer( + getMemoizedBuffer(this.gl, WebGL2RenderingContext.ARRAY_BUFFER, () => { + const array = new Uint8Array(NUM_CDF_LINES * VERTICES_PER_LINE); + for (let i = 0; i < NUM_CDF_LINES; ++i) { + for (let j = 0; j < VERTICES_PER_LINE; ++j) { + array[i * VERTICES_PER_LINE + j] = i; + } + } + return array; + }), + ).value; + constructor(public parent: TransferFunctionWidget) { super(parent.display, document.createElement("div"), parent.visibility); const { element, gl } = this; @@ -963,6 +980,10 @@ class TransferFunctionPanel extends IndirectRenderedPanel { } } + private histogramLineShader = this.registerDisposer( + (() => createCDFLineShader(this.gl, histogramSamplerTextureUnit))(), + ); + private transferFunctionLineShader = this.registerDisposer( (() => { const builder = new ShaderBuilder(this.gl); @@ -976,7 +997,7 @@ vec4 end = vec4(aLineStartEnd[2], aLineStartEnd[3], 0.0, 1.0); emitLine(start, end, 1.0); `); builder.setFragmentMain(` -out_color = vec4(0.0, 1.0, 1.0, getLineAlpha()); +out_color = vec4(0.35, 0.35, 0.35, getLineAlpha()); `); return builder.build(); })(), @@ -1042,6 +1063,7 @@ out_color = tempColor * alpha; gl, transferFunctionShader, controlPointsShader, + histogramLineShader: lineShader, } = this; this.setGLLogicalViewport(); gl.clearColor(0.0, 0.0, 0.0, 0.0); @@ -1078,6 +1100,42 @@ out_color = tempColor * alpha; gl.disableVertexAttribArray(aVertexPosition); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); } + // Draw CDF lines + if (this.parent.histogramSpecifications.producerVisibility.visible) { + const { renderViewport } = this; + lineShader.bind(); + initializeLineShader( + lineShader, + { + width: renderViewport.logicalWidth, + height: renderViewport.logicalHeight, + }, + /*featherWidthInPixels=*/ 1.0, + ); + const histogramTextureUnit = lineShader.textureUnit( + histogramSamplerTextureUnit, + ); + gl.uniform1f( + lineShader.uniform("uBoundsFraction"), + getIntervalBoundsEffectiveFraction( + this.parent.dataType, + this.parent.trackable.value.window, + ), + ); + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + histogramTextureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, this.parent.texture); + setRawTextureParameters(gl); + const aDataValue = lineShader.attribute("aDataValue"); + this.dataValuesBuffer.bindToVertexAttribI( + aDataValue, + /*componentsPerVertexAttribute=*/ 1, + /*attributeType=*/ WebGL2RenderingContext.UNSIGNED_BYTE, + ); + drawLines(gl, /*linesPerInstance=*/ NUM_CDF_LINES, /*numInstances=*/ 1); + gl.disableVertexAttribArray(aDataValue); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); + } + // Draw lines and control points on top of transfer function - if there are any if (this.controlPointsPositionArray.length > 0) { const { renderViewport } = this; @@ -1481,13 +1539,24 @@ class TransferFunctionWidget extends Tab { ); window = createWindowBoundInputs(this.dataType, this.trackable); + + get texture() { + return this.histogramSpecifications.getFramebuffers(this.display.gl)[ + this.histogramIndex + ].colorBuffers[0].texture; + } constructor( visibility: WatchableVisibilityPriority, public display: DisplayContext, public dataType: DataType, public trackable: WatchableValueInterface, + public histogramSpecifications: HistogramSpecifications, + public histogramIndex: number, ) { super(visibility); + this.registerDisposer( + histogramSpecifications.visibility.add(this.visibility), + ); const { element } = this; element.classList.add("neuroglancer-transfer-function-widget"); element.appendChild(this.transferFunctionPanel.element); @@ -1660,6 +1729,8 @@ export function transferFunctionLayerControl( watchableValue: WatchableValueInterface; defaultChannel: number[]; channelCoordinateSpaceCombiner: CoordinateSpaceCombiner | undefined; + histogramSpecifications: HistogramSpecifications; + histogramIndex: number; dataType: DataType; }, ): LayerControlFactory { @@ -1669,6 +1740,8 @@ export function transferFunctionLayerControl( watchableValue, channelCoordinateSpaceCombiner, defaultChannel, + histogramSpecifications, + histogramIndex, dataType, } = getter(layer); @@ -1717,6 +1790,8 @@ export function transferFunctionLayerControl( options.display, dataType, watchableValue, + histogramSpecifications, + histogramIndex, ), ); return { control, controlElement: control.element };