diff --git a/common/reviews/api/core.api.md b/common/reviews/api/core.api.md index a20979089..e2b611698 100644 --- a/common/reviews/api/core.api.md +++ b/common/reviews/api/core.api.md @@ -882,6 +882,17 @@ interface IContourSet { readonly sizeInBytes: number; } +// @public (undocumented) +interface IDynamicImageVolume extends IImageVolume { + // (undocumented) + getScalarDataArrays(): VolumeScalarData[]; + // (undocumented) + get numTimePoints(): number; + // (undocumented) + get timePointIndex(): number; + set timePointIndex(newTimePointIndex: number); +} + // @public (undocumented) interface IEnabledElement { // (undocumented) @@ -1066,15 +1077,25 @@ interface IImageVolume { // (undocumented) convertToCornerstoneImage?: (imageId: string, imageIdIndex: number) => IImageLoadObject; // (undocumented) + destroy(): void; + // (undocumented) dimensions: Point3; // (undocumented) direction: Mat3; // (undocumented) + getImageIdIndex(imageId: string): number; + // (undocumented) + getImageURIIndex(imageURI: string): number; + // (undocumented) + getScalarData(): VolumeScalarData; + // (undocumented) hasPixelSpacing: boolean; // (undocumented) imageData?: vtkImageData; // (undocumented) - imageIds?: Array; + imageIds: Array; + // (undocumented) + isDynamicVolume(): boolean; // (undocumented) isPrescaled: boolean; // (undocumented) @@ -1088,8 +1109,6 @@ interface IImageVolume { // (undocumented) referencedVolumeId?: string; // (undocumented) - scalarData: any; - // (undocumented) scaling?: { PET?: { SUVlbmFactor?: number; @@ -1238,15 +1257,26 @@ export class ImageVolume implements IImageVolume { // (undocumented) cancelLoading: () => void; // (undocumented) + destroy(): void; + // (undocumented) dimensions: Point3; // (undocumented) direction: Mat3; // (undocumented) + getImageIdIndex(imageId: string): number; + // (undocumented) + getImageURIIndex(imageURI: string): number; + // (undocumented) + getScalarData(): VolumeScalarData; + // (undocumented) hasPixelSpacing: boolean; // (undocumented) imageData?: any; // (undocumented) - imageIds?: Array; + get imageIds(): Array; + set imageIds(newImageIds: Array); + // (undocumented) + isDynamicVolume(): boolean; // (undocumented) isPrescaled: boolean; // (undocumented) @@ -1260,7 +1290,7 @@ export class ImageVolume implements IImageVolume { // (undocumented) referencedVolumeId?: string; // (undocumented) - scalarData: Float32Array | Uint8Array; + protected scalarData: VolumeScalarData | Array; // (undocumented) scaling?: { PET?: { @@ -1561,7 +1591,7 @@ interface IVolume { // (undocumented) referencedVolumeId?: string; // (undocumented) - scalarData: Float32Array | Uint8Array; + scalarData: VolumeScalarData | Array; // (undocumented) scaling?: { PET?: { @@ -2123,8 +2153,10 @@ declare namespace Types { IEnabledElement, ICache, IVolume, + VolumeScalarData, IViewportId, IImageVolume, + IDynamicImageVolume, IRenderingEngine, ScalingParameters, PTScaling, @@ -2522,6 +2554,9 @@ type VolumeNewImageEventDetail = { renderingEngineId: string; }; +// @public (undocumented) +type VolumeScalarData = Float32Array | Uint8Array; + // @public (undocumented) export class VolumeViewport extends BaseVolumeViewport { constructor(props: ViewportInput); diff --git a/common/reviews/api/streaming-image-volume-loader.api.md b/common/reviews/api/streaming-image-volume-loader.api.md index a3ef43ab0..125d5262e 100644 --- a/common/reviews/api/streaming-image-volume-loader.api.md +++ b/common/reviews/api/streaming-image-volume-loader.api.md @@ -4,6 +4,7 @@ ```ts +import { default as default_2 } from 'packages/core/dist/esm/enums/RequestType'; import type { mat4 } from 'gl-matrix'; import type vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; import type { vtkImageData } from '@kitware/vtk.js/Common/DataModel/ImageData'; @@ -75,6 +76,11 @@ enum ContourType { OPEN_PLANAR = 'OPEN_PLANAR', } +// @public (undocumented) +export function cornerstoneStreamingDynamicImageVolumeLoader(volumeId: string, options: { + imageIds: string[]; +}): IVolumeLoader_2; + // @public (undocumented) export function cornerstoneStreamingImageVolumeLoader(volumeId: string, options: { imageIds: string[]; @@ -612,6 +618,14 @@ interface IContourSet { readonly sizeInBytes: number; } +// @public +interface IDynamicImageVolume extends IImageVolume { + getScalarDataArrays(): VolumeScalarData[]; + get numTimePoints(): number; + get timePointIndex(): number; + set timePointIndex(newTimePointIndex: number); +} + // @public interface IEnabledElement { FrameOfReferenceUID: string; @@ -750,18 +764,22 @@ interface IImageVolume { imageId: string, imageIdIndex: number ) => IImageLoadObject; + destroy(): void; dimensions: Point3; direction: Mat3; + getImageIdIndex(imageId: string): number; + getImageURIIndex(imageURI: string): number; + getScalarData(): VolumeScalarData; hasPixelSpacing: boolean; imageData?: vtkImageData; - imageIds?: Array; + imageIds: Array; + isDynamicVolume(): boolean; isPrescaled: boolean; loadStatus?: Record; metadata: Metadata; numVoxels: number; origin: Point3; referencedVolumeId?: string; - scalarData: any; scaling?: { PET?: { SUVlbmFactor?: number; @@ -985,6 +1003,7 @@ interface IStreamingImageVolume extends ImageVolume { // @public (undocumented) interface IStreamingVolumeProperties { imageIds: Array; + loadStatus: { loaded: boolean; loading: boolean; @@ -1056,7 +1075,7 @@ interface IVolume { metadata: Metadata; origin: Point3; referencedVolumeId?: string; - scalarData: Float32Array | Uint8Array; + scalarData: VolumeScalarData | Array; scaling?: { PET?: { // @TODO: Do these values exist? @@ -1318,16 +1337,24 @@ type StackViewportScrollEventDetail = { }; // @public (undocumented) -export class StreamingImageVolume extends ImageVolume { +export class StreamingDynamicImageVolume extends BaseStreamingImageVolume implements Types.IDynamicImageVolume { constructor(imageVolumeProperties: Types.IVolume, streamingProperties: Types.IStreamingVolumeProperties); // (undocumented) - cancelLoading: () => void; + getImageLoadRequests: (priority: number) => any[]; // (undocumented) - clearLoadCallbacks(): void; + getScalarData(): Types.VolumeScalarData; // (undocumented) - convertToCornerstoneImage(imageId: string, imageIdIndex: number): Types.IImageLoadObject; + isDynamicVolume(): boolean; // (undocumented) - decache(completelyRemove?: boolean): void; + get numTimePoints(): number; + // (undocumented) + get timePointIndex(): number; + set timePointIndex(newTimePointIndex: number); +} + +// @public (undocumented) +export class StreamingImageVolume extends BaseStreamingImageVolume { + constructor(imageVolumeProperties: Types.IVolume, streamingProperties: Types.IStreamingVolumeProperties); // (undocumented) getImageLoadRequests: (priority: number) => { callLoadImage: (imageId: any, imageIdIndex: any, options: any) => Promise; @@ -1347,20 +1374,13 @@ export class StreamingImageVolume extends ImageVolume { }; }; priority: number; - requestType: Enums.RequestType; + requestType: default_2; additionalDetails: { volumeId: string; }; }[]; // (undocumented) - load: (callback: (...args: unknown[]) => void, priority?: number) => void; - // (undocumented) - loadStatus: { - loaded: boolean; - loading: boolean; - cachedFrames: Array; - callbacks: Array<(...args: unknown[]) => void>; - }; + getScalarData(): Types.VolumeScalarData; } // @public @@ -1504,6 +1524,9 @@ type VolumeNewImageEventDetail = { renderingEngineId: string; }; +// @public (undocumented) +type VolumeScalarData = Float32Array | Uint8Array; + // @public type VolumeViewportProperties = { voiRange?: VOIRange; diff --git a/common/reviews/api/tools.api.md b/common/reviews/api/tools.api.md index 6313ffba6..705138c06 100644 --- a/common/reviews/api/tools.api.md +++ b/common/reviews/api/tools.api.md @@ -2185,6 +2185,14 @@ type IDistance = { world: number; }; +// @public +interface IDynamicImageVolume extends IImageVolume { + getScalarDataArrays(): VolumeScalarData[]; + get numTimePoints(): number; + get timePointIndex(): number; + set timePointIndex(newTimePointIndex: number); +} + // @public interface IEnabledElement { FrameOfReferenceUID: string; @@ -2323,18 +2331,22 @@ interface IImageVolume { imageId: string, imageIdIndex: number ) => IImageLoadObject; + destroy(): void; dimensions: Point3; direction: Mat3; + getImageIdIndex(imageId: string): number; + getImageURIIndex(imageURI: string): number; + getScalarData(): VolumeScalarData; hasPixelSpacing: boolean; imageData?: vtkImageData; - imageIds?: Array; + imageIds: Array; + isDynamicVolume(): boolean; isPrescaled: boolean; loadStatus?: Record; metadata: Metadata; numVoxels: number; origin: Point3; referencedVolumeId?: string; - scalarData: any; scaling?: { PET?: { SUVlbmFactor?: number; @@ -2609,6 +2621,7 @@ interface IStreamingImageVolume extends ImageVolume { // @public (undocumented) interface IStreamingVolumeProperties { imageIds: Array; + loadStatus: { loaded: boolean; loading: boolean; @@ -2785,7 +2798,7 @@ interface IVolume { metadata: Metadata; origin: Point3; referencedVolumeId?: string; - scalarData: Float32Array | Uint8Array; + scalarData: VolumeScalarData | Array; scaling?: { PET?: { // @TODO: Do these values exist? @@ -5212,6 +5225,9 @@ export class VolumeRotateMouseWheelTool extends BaseTool { static toolName: any; } +// @public (undocumented) +type VolumeScalarData = Float32Array | Uint8Array; + // @public type VolumeViewportProperties = { voiRange?: VOIRange; diff --git a/packages/core/examples/dynamicVolume/index.ts b/packages/core/examples/dynamicVolume/index.ts new file mode 100644 index 000000000..ad5f8143f --- /dev/null +++ b/packages/core/examples/dynamicVolume/index.ts @@ -0,0 +1,141 @@ +import { + RenderingEngine, + Types, + Enums, + volumeLoader, + getRenderingEngine, +} from '@cornerstonejs/core'; +import { + initDemo, + createImageIdsAndCacheMetaData, + setTitleAndDescription, + setPetTransferFunctionForVolumeActor, + addSliderToToolbar, + addDropdownToToolbar, +} from '../../../../utils/demo/helpers'; + +// This is for debugging purposes +console.warn( + 'Click on index.ts to open source code for this example --------->' +); + +const { ViewportType } = Enums; +const renderingEngineId = 'myRenderingEngine'; +const viewportId = 'CT_SAGITTAL_STACK'; +const orientations = [ + Enums.OrientationAxis.AXIAL, + Enums.OrientationAxis.SAGITTAL, + Enums.OrientationAxis.CORONAL, +]; + +const description = [ + 'Displays a 4D DICOM series in a Volume viewport.', + 'DataSet: PET 255 x 255 images / 40 time points / 235 images per time point / 9,400 images total', +].join('\n'); + +// ======== Set up page ======== // +setTitleAndDescription('Volume 4D', description); + +const content = document.getElementById('content'); +const element = document.createElement('div'); +element.id = 'cornerstone-element'; +element.style.width = '500px'; +element.style.height = '500px'; + +content.appendChild(element); +// ============================= // + +addDropdownToToolbar({ + options: { + values: orientations, + defaultValue: orientations[0], + }, + onSelectedValueChange: (selectedValue) => { + // Get the rendering engine + const renderingEngine = getRenderingEngine(renderingEngineId); + + // Get the volume viewport + const viewport = ( + renderingEngine.getViewport(viewportId) + ); + + viewport.setOrientation(selectedValue); + viewport.render(); + }, +}); + +function addTimePointSlider(volume) { + addSliderToToolbar({ + title: 'Time Point', + range: [0, volume.numTimePoints - 1], + defaultValue: 0, + onSelectedValueChange: (value) => { + const timePointIndex = Number(value); + volume.timePointIndex = timePointIndex; + }, + }); +} + +/** + * Runs the demo + */ +async function run() { + // Init Cornerstone and related libraries + await initDemo(); + + // Get Cornerstone imageIds and fetch metadata into RAM + const imageIds = await createImageIdsAndCacheMetaData({ + StudyInstanceUID: + '1.3.6.1.4.1.12842.1.1.14.3.20220915.105557.468.2963630849', + SeriesInstanceUID: + '1.3.6.1.4.1.12842.1.1.22.4.20220915.124758.560.4125514885', + wadoRsRoot: 'https://d28o5kq0jsoob5.cloudfront.net/dicomweb', + }); + + // Instantiate a rendering engine + const renderingEngine = new RenderingEngine(renderingEngineId); + + // Create a stack viewport + const viewportInput = { + viewportId, + type: ViewportType.ORTHOGRAPHIC, + element, + defaultOptions: { + orientation: Enums.OrientationAxis.ACQUISITION, + background: [0.2, 0, 0.2], + }, + }; + + renderingEngine.enableElement(viewportInput); + + // Get the stack viewport that was created + const viewport = ( + renderingEngine.getViewport(viewportId) + ); + + // Define a unique id for the volume + const volumeName = 'CT_VOLUME_ID'; // Id of the volume less loader prefix + // const volumeLoaderScheme = 'cornerstoneStreamingImageVolume'; // Loader id which defines which volume loader to use + const volumeLoaderScheme = 'cornerstoneStreamingDynamicImageVolume'; // Loader id which defines which volume loader to use + const volumeId = `${volumeLoaderScheme}:${volumeName}`; // VolumeId with loader id + volume id + + // Define a volume in memory + const volume = await volumeLoader.createAndCacheVolume(volumeId, { + imageIds, + }); + + // Set the volume to load + volume.load(); + + addTimePointSlider(volume); + + // Set the volume on the viewport + viewport.setVolumes([ + { volumeId, callback: setPetTransferFunctionForVolumeActor }, + ]); + + // Render the image + viewport.render(); +} + +run(); diff --git a/packages/core/src/RenderingEngine/VolumeViewport.ts b/packages/core/src/RenderingEngine/VolumeViewport.ts index c94dbb67e..601fd49fc 100644 --- a/packages/core/src/RenderingEngine/VolumeViewport.ts +++ b/packages/core/src/RenderingEngine/VolumeViewport.ts @@ -242,7 +242,7 @@ class VolumeViewport extends BaseVolumeViewport { index[1] * dimensions[0] + index[0]; - return volume.scalarData[voxelIndex]; + return volume.getScalarData()[voxelIndex]; } public setBlendMode( diff --git a/packages/core/src/RenderingEngine/helpers/setDefaultVolumeVOI.ts b/packages/core/src/RenderingEngine/helpers/setDefaultVolumeVOI.ts index 9dfb98c6b..23a6681cc 100644 --- a/packages/core/src/RenderingEngine/helpers/setDefaultVolumeVOI.ts +++ b/packages/core/src/RenderingEngine/helpers/setDefaultVolumeVOI.ts @@ -115,7 +115,8 @@ function getVOIFromMetadata(imageVolume: IImageVolume): VOIRange { * @returns The VOIRange with lower and upper values */ async function getVOIFromMinMax(imageVolume: IImageVolume): Promise { - const { scalarData, imageIds } = imageVolume; + const { imageIds } = imageVolume; + const scalarData = imageVolume.getScalarData(); // Get the middle image from the list of imageIds const imageIdIndex = Math.floor(imageIds.length / 2); diff --git a/packages/core/src/cache/cache.ts b/packages/core/src/cache/cache.ts index 4afbe79cd..36615b649 100644 --- a/packages/core/src/cache/cache.ts +++ b/packages/core/src/cache/cache.ts @@ -455,22 +455,16 @@ class Cache implements ICache { for (const volumeId of volumeIds) { const cachedVolume = this._volumeCache.get(volumeId); + const { volume } = cachedVolume; - if (!cachedVolume.volume) { + if (!volume?.imageIds?.length) { return; } - let { imageIds } = cachedVolume.volume; + const imageIdIndex = volume.getImageURIIndex(imageIdToUse); - if (!imageIds || imageIds.length === 0) { - continue; - } - - imageIds = imageIds.map((id) => imageIdToURI(id)); - - const imageIdIndex = imageIds.indexOf(imageIdToUse); if (imageIdIndex > -1) { - return { volume: cachedVolume.volume, imageIdIndex }; + return { volume, imageIdIndex }; } } } diff --git a/packages/core/src/cache/classes/ImageVolume.ts b/packages/core/src/cache/classes/ImageVolume.ts index 8bfe82cf0..26bbb1797 100644 --- a/packages/core/src/cache/classes/ImageVolume.ts +++ b/packages/core/src/cache/classes/ImageVolume.ts @@ -1,10 +1,25 @@ +import isTypedArray from '../../utilities/isTypedArray'; +import { imageIdToURI } from '../../utilities'; import { vtkStreamingOpenGLTexture } from '../../RenderingEngine/vtkClasses'; -import { IVolume, Metadata, Point3, IImageVolume, Mat3 } from '../../types'; +import { + IVolume, + VolumeScalarData, + Metadata, + Point3, + IImageVolume, + Mat3, +} from '../../types'; /** The base class for volume data. It includes the volume metadata * and the volume data along with the loading status. */ export class ImageVolume implements IImageVolume { + private _imageIds: Array; + private _imageIdsIndexMap = new Map(); + private _imageURIsIndexMap = new Map(); + /** volume scalar data 3D or 4D */ + protected scalarData: VolumeScalarData | Array; + /** Read-only unique identifier for the volume */ readonly volumeId: string; /** Dimensions of the volume */ @@ -15,8 +30,6 @@ export class ImageVolume implements IImageVolume { metadata: Metadata; /** volume origin, Note this is an opinionated origin for the volume */ origin: Point3; - /** volume scalar data */ - scalarData: Float32Array | Uint8Array; /** Whether preScaling has been performed on the volume */ isPrescaled = false; /** volume scaling parameters if it contains scaled data */ @@ -42,8 +55,6 @@ export class ImageVolume implements IImageVolume { vtkOpenGLTexture: any; // No good way of referencing vtk classes as they aren't classes. /** load status object for the volume */ loadStatus?: Record; - /** optional image ids for the volume if it is made of separated images */ - imageIds?: Array; /** optional reference volume id if the volume is derived from another volume */ referencedVolumeId?: string; /** whether the metadata for the pixel spacing is not undefined */ @@ -72,7 +83,73 @@ export class ImageVolume implements IImageVolume { } } + /** return the image ids for the volume if it is made of separated images */ + public get imageIds(): Array { + return this._imageIds; + } + + /** updates the image ids */ + public set imageIds(newImageIds: Array) { + this._imageIds = newImageIds; + this._reprocessImageIds(); + } + + private _reprocessImageIds() { + this._imageIdsIndexMap.clear(); + this._imageURIsIndexMap.clear(); + + this._imageIds.forEach((imageId, i) => { + const imageURI = imageIdToURI(imageId); + + this._imageIdsIndexMap.set(imageId, i); + this._imageURIsIndexMap.set(imageURI, i); + }); + } + cancelLoading: () => void; + + /** return true if it is a 4D volume or false if it is 3D volume */ + public isDynamicVolume(): boolean { + return false; + } + + /** + * Return the scalar data for 3D volumes or the active scalar data + * (current time point) for 4D volumes + */ + public getScalarData(): VolumeScalarData { + if (isTypedArray(this.scalarData)) { + return this.scalarData; + } + + throw new Error('Unknow scalar data type'); + } + + /** + * return the index of a given imageId + * @param imageId - imageId + * @returns imageId index + */ + public getImageIdIndex(imageId: string): number { + return this._imageIdsIndexMap.get(imageId); + } + + /** + * return the index of a given imageURI + * @param imageId - imageURI + * @returns imageURI index + */ + public getImageURIIndex(imageURI: string): number { + return this._imageURIsIndexMap.get(imageURI); + } + + /** + * destroy the volume and make it unusable + */ + destroy(): void { + this.vtkOpenGLTexture.delete(); + this.scalarData = null; + } } export default ImageVolume; diff --git a/packages/core/src/loaders/volumeLoader.ts b/packages/core/src/loaders/volumeLoader.ts index b079c1a06..9eea03fd8 100644 --- a/packages/core/src/loaders/volumeLoader.ts +++ b/packages/core/src/loaders/volumeLoader.ts @@ -34,14 +34,49 @@ interface LocalVolumeOptions { direction: Mat3; } -function createInternalVTKRepresentation({ - dimensions, - metadata, - spacing, - direction, - origin, - scalarData, -}): vtkImageDataType { +/** + * Adds a single scalar data to a 3D volume + */ +function addScalarDataToImageData( + imageData: vtkImageDataType, + scalarData: Types.VolumeScalarData, + dataArrayAttrs +) { + const scalarArray = vtkDataArray.newInstance({ + name: `Pixels`, + values: scalarData, + ...dataArrayAttrs, + }); + + imageData.getPointData().setScalars(scalarArray); +} + +/** + * Adds multiple scalar data (time points) to a 4D volume + */ +function addScalarDataArraysToImageData( + imageData: vtkImageDataType, + scalarDataArrays: Types.VolumeScalarData[], + dataArrayAttrs +) { + scalarDataArrays.forEach((scalarData, i) => { + const vtkScalarArray = vtkDataArray.newInstance({ + name: `timePoint-${i}`, + values: scalarData, + ...dataArrayAttrs, + }); + + imageData.getPointData().addArray(vtkScalarArray); + }); + + // Set the first as active otherwise nothing is displayed on the screen + imageData.getPointData().setActiveScalars('timePoint-0'); +} + +function createInternalVTKRepresentation( + volume: Types.IImageVolume +): vtkImageDataType { + const { dimensions, metadata, spacing, direction, origin } = volume; const { PhotometricInterpretation } = metadata; let numComponents = 1; @@ -49,19 +84,26 @@ function createInternalVTKRepresentation({ numComponents = 3; } - const scalarArray = vtkDataArray.newInstance({ - name: 'Pixels', - numberOfComponents: numComponents, - values: scalarData, - }); - const imageData = vtkImageData.newInstance(); + const dataArrayAttrs = { numberOfComponents: numComponents }; imageData.setDimensions(dimensions); imageData.setSpacing(spacing); imageData.setDirection(direction); imageData.setOrigin(origin); - imageData.getPointData().setScalars(scalarArray); + + // Add scalar datas to 3D or 4D volume + if (volume.isDynamicVolume()) { + const scalarDataArrays = (( + volume + )).getScalarDataArrays(); + + addScalarDataArraysToImageData(imageData, scalarDataArrays, dataArrayAttrs); + } else { + const scalarData = volume.getScalarData(); + + addScalarDataToImageData(imageData, scalarData, dataArrayAttrs); + } return imageData; } @@ -224,8 +266,8 @@ export async function createAndCacheDerivedVolume( volumeId = uuidv4(); } - const { metadata, dimensions, spacing, origin, direction, scalarData } = - referencedVolume; + const { metadata, dimensions, spacing, origin, direction } = referencedVolume; + const scalarData = referencedVolume.getScalarData(); const scalarLength = scalarData.length; let numBytes, TypedArray; diff --git a/packages/core/src/requestPool/requestPoolManager.ts b/packages/core/src/requestPool/requestPoolManager.ts index e0cc12f39..8d8e3835f 100644 --- a/packages/core/src/requestPool/requestPoolManager.ts +++ b/packages/core/src/requestPool/requestPoolManager.ts @@ -289,9 +289,15 @@ class RequestPoolManager { } if (this.grabDelay !== undefined) { - this.timeoutHandle = window.setTimeout(() => { - this.startGrabbing(); - }, this.grabDelay); + // Prevents calling setTimeout hundreds of times when hundreds of requests + // are added which make it slower and works in an unexpected way when + // destroy/clearTimeout is called because only the last handle is stored. + if (!this.timeoutHandle) { + this.timeoutHandle = window.setTimeout(() => { + this.timeoutHandle = null; + this.startGrabbing(); + }, this.grabDelay); + } } else { this.startGrabbing(); } diff --git a/packages/core/src/types/IDynamicImageVolume.ts b/packages/core/src/types/IDynamicImageVolume.ts new file mode 100644 index 000000000..1630c6402 --- /dev/null +++ b/packages/core/src/types/IDynamicImageVolume.ts @@ -0,0 +1,18 @@ +import { IImageVolume, VolumeScalarData } from '../types'; + +/** + * Cornerstone ImageVolume interface. Todo: we should define new IVolume class + * with appropriate typings for the other types of volume that don't have images (nrrd, nifti) + */ +interface IDynamicImageVolume extends IImageVolume { + /** Returns the active time point index */ + get timePointIndex(): number; + /** Set the active time point index which also updates the active scalar data */ + set timePointIndex(newTimePointIndex: number); + /** Returns the number of time points */ + get numTimePoints(): number; + /** return scalar data arrays (one per timepoint) */ + getScalarDataArrays(): VolumeScalarData[]; +} + +export default IDynamicImageVolume; diff --git a/packages/core/src/types/IImageVolume.ts b/packages/core/src/types/IImageVolume.ts index bbac21925..c30a2771f 100644 --- a/packages/core/src/types/IImageVolume.ts +++ b/packages/core/src/types/IImageVolume.ts @@ -1,5 +1,11 @@ import type { vtkImageData } from '@kitware/vtk.js/Common/DataModel/ImageData'; -import { Metadata, Point3, IImageLoadObject, Mat3 } from '../types'; +import { + Metadata, + VolumeScalarData, + Point3, + IImageLoadObject, + Mat3, +} from '../types'; /** * Cornerstone ImageVolume interface. Todo: we should define new IVolume class @@ -16,8 +22,6 @@ interface IImageVolume { metadata: Metadata; /** volume origin - set to the imagePositionPatient of the last image in the volume */ origin: Point3; - /** volume scalar data */ - scalarData: any; /** Whether preScaling has been performed on the volume */ isPrescaled: boolean; /** volume scaling metadata */ @@ -42,11 +46,13 @@ interface IImageVolume { /** loading status object for the volume containing loaded/loading statuses */ loadStatus?: Record; /** imageIds of the volume (if it is built of separate imageIds) */ - imageIds?: Array; + imageIds: Array; /** volume referencedVolumeId (if it is derived from another volume) */ referencedVolumeId?: string; // if volume is derived from another volume /** whether the metadata for the pixel spacing is not undefined */ hasPixelSpacing: boolean; + /** return true if it is a 4D volume or false if it is 3D volume */ + isDynamicVolume(): boolean; /** method to convert the volume data in the volume cache, to separate images in the image cache */ convertToCornerstoneImage?: ( imageId: string, @@ -55,6 +61,18 @@ interface IImageVolume { //cancel load cancelLoading?: () => void; + + /** return the volume scalar data */ + getScalarData(): VolumeScalarData; + + /** return the index of a given imageId */ + getImageIdIndex(imageId: string): number; + + /** return the index of a given imageURI */ + getImageURIIndex(imageURI: string): number; + + /** destroy the volume and make it unusable */ + destroy(): void; } export default IImageVolume; diff --git a/packages/core/src/types/IStreamingVolumeProperties.ts b/packages/core/src/types/IStreamingVolumeProperties.ts index 06aa78fc2..f305d02d3 100644 --- a/packages/core/src/types/IStreamingVolumeProperties.ts +++ b/packages/core/src/types/IStreamingVolumeProperties.ts @@ -1,6 +1,7 @@ interface IStreamingVolumeProperties { /** imageIds of the volume */ imageIds: Array; + /** loading status object for the volume containing loaded/loading statuses */ loadStatus: { loaded: boolean; diff --git a/packages/core/src/types/IVolume.ts b/packages/core/src/types/IVolume.ts index c46105bcd..017009754 100644 --- a/packages/core/src/types/IVolume.ts +++ b/packages/core/src/types/IVolume.ts @@ -3,6 +3,8 @@ import type Point3 from './Point3'; import type Metadata from './Metadata'; import Mat3 from './Mat3'; +type VolumeScalarData = Float32Array | Uint8Array; + /** * Cornerstone ImageVolume interface. */ @@ -20,7 +22,7 @@ interface IVolume { /** volume direction */ direction: Mat3; /** volume scalarData */ - scalarData: Float32Array | Uint8Array; + scalarData: VolumeScalarData | Array; /** volume size in bytes */ sizeInBytes?: number; /** volume image data as vtkImageData */ @@ -40,4 +42,4 @@ interface IVolume { }; } -export default IVolume; +export { IVolume as default, IVolume, VolumeScalarData }; diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index 5729b82fa..a0dc4ecd5 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -3,10 +3,11 @@ import type ICamera from './ICamera'; import type IEnabledElement from './IEnabledElement'; import type ICache from './ICache'; -import type IVolume from './IVolume'; +import type { IVolume, VolumeScalarData } from './IVolume'; import type { VOI, VOIRange } from './voi'; import type ImageLoaderFn from './ImageLoaderFn'; import type IImageVolume from './IImageVolume'; +import type IDynamicImageVolume from './IDynamicImageVolume'; import type VolumeLoaderFn from './VolumeLoaderFn'; import type IRegisterImageLoader from './IRegisterImageLoader'; import type IStreamingVolumeProperties from './IStreamingVolumeProperties'; @@ -80,8 +81,10 @@ export type { IEnabledElement, ICache, IVolume, + VolumeScalarData, IViewportId, IImageVolume, + IDynamicImageVolume, IRenderingEngine, ScalingParameters, PTScaling, diff --git a/packages/core/src/utilities/isTypedArray.ts b/packages/core/src/utilities/isTypedArray.ts new file mode 100644 index 000000000..490db55fc --- /dev/null +++ b/packages/core/src/utilities/isTypedArray.ts @@ -0,0 +1,20 @@ +/** + * checks if an object is an instance of a TypedArray + * + * @param obj - Object to check + * + * @returns True if the object is a TypedArray. + */ +export default function isTypedArray(obj: any): boolean { + return ( + obj instanceof Int8Array || + obj instanceof Uint8Array || + obj instanceof Uint8ClampedArray || + obj instanceof Int16Array || + obj instanceof Uint16Array || + obj instanceof Int32Array || + obj instanceof Uint32Array || + obj instanceof Float32Array || + obj instanceof Float64Array + ); +} diff --git a/packages/streaming-image-volume-loader/package.json b/packages/streaming-image-volume-loader/package.json index c605f769b..28656efe8 100644 --- a/packages/streaming-image-volume-loader/package.json +++ b/packages/streaming-image-volume-loader/package.json @@ -26,14 +26,14 @@ }, "dependencies": { "@cornerstonejs/core": "^0.31.2", - "cornerstone-wado-image-loader": "^4.8.0" + "cornerstone-wado-image-loader": "^4.10.0" }, "peerDependencies": { "@cornerstonejs/calculate-suv": "1.0.2" }, "devDependencies": { "@cornerstonejs/calculate-suv": "1.0.2", - "cornerstone-wado-image-loader": "^4.8.0" + "cornerstone-wado-image-loader": "^4.10.0" }, "contributors": [ { diff --git a/packages/streaming-image-volume-loader/src/BaseStreamingImageVolume.ts b/packages/streaming-image-volume-loader/src/BaseStreamingImageVolume.ts new file mode 100644 index 000000000..eba7a4677 --- /dev/null +++ b/packages/streaming-image-volume-loader/src/BaseStreamingImageVolume.ts @@ -0,0 +1,892 @@ +import { + Enums, + eventTarget, + metaData, + imageLoadPoolManager, + triggerEvent, + ImageVolume, + cache, + imageLoader, + utilities as csUtils, +} from '@cornerstonejs/core'; + +import type { Types } from '@cornerstonejs/core'; +import { scaleArray, autoLoad } from './helpers'; + +const requestType = Enums.RequestType.Prefetch; +const { getMinMax } = csUtils; + +/** + * Streaming Image Volume Class that extends ImageVolume base class. + * It implements load method to load the imageIds and insert them into the volume. + */ +export default class BaseStreamingImageVolume extends ImageVolume { + private framesLoaded = 0; + private framesProcessed = 0; + protected numFrames: number; + protected cornerstoneImageMetaData = null; + + loadStatus: { + loaded: boolean; + loading: boolean; + cachedFrames: Array; + callbacks: Array<(...args: unknown[]) => void>; + }; + + constructor( + imageVolumeProperties: Types.IVolume, + streamingProperties: Types.IStreamingVolumeProperties + ) { + super(imageVolumeProperties); + this.imageIds = streamingProperties.imageIds; + this.loadStatus = streamingProperties.loadStatus; + this.numFrames = this._getNumFrames(); + + this._createCornerstoneImageMetaData(); + } + + /** + * Returns the number of frames stored in a scalarData object. The number of + * frames is equal to the number of images for 3D volumes or the number of + * frames per time poins for 4D volumes. + * @returns number of frames per volume + */ + private _getNumFrames(): number { + const { imageIds, scalarData } = this; + const scalarDataCount = this.isDynamicVolume() ? scalarData.length : 1; + + return imageIds.length / scalarDataCount; + } + + private _getScalarDataLength(): number { + const { scalarData } = this; + return this.isDynamicVolume() + ? (scalarData)[0].length + : (scalarData).length; + } + + /** + * Creates the metadata required for converting the volume to an cornerstoneImage + */ + private _createCornerstoneImageMetaData() { + const { numFrames } = this; + + if (numFrames === 0) { + return; + } + + const bytesPerImage = this.sizeInBytes / numFrames; + const scalarDataLength = this._getScalarDataLength(); + const numComponents = scalarDataLength / this.numVoxels; + const pixelsPerImage = + this.dimensions[0] * this.dimensions[1] * numComponents; + + const { PhotometricInterpretation, voiLut, VOILUTFunction } = this.metadata; + + let windowCenter = []; + let windowWidth = []; + + if (voiLut && voiLut.length) { + windowCenter = voiLut.map((voi) => { + return voi.windowCenter; + }); + + windowWidth = voiLut.map((voi) => { + return voi.windowWidth; + }); + } + + const color = numComponents > 1 ? true : false; //todo: fix this + + this.cornerstoneImageMetaData = { + bytesPerImage, + numComponents, + pixelsPerImage, + windowCenter, + windowWidth, + color, + spacing: this.spacing, + dimensions: this.dimensions, + PhotometricInterpretation, + voiLUTFunction: VOILUTFunction, + invert: PhotometricInterpretation === 'MONOCHROME1', + }; + } + + /** + * Converts imageIdIndex into frameIndex which will be the same + * for 3D volumes but different for 4D volumes + */ + private _imageIdIndexToFrameIndex(imageIdIndex: number): number { + return imageIdIndex % this.numFrames; + } + + /** + * Return all scalar data objects (buffers) which will be only one for + * 3D volumes and one per time point for 4D volumes + * images of each 3D volume is stored + * @returns scalar data array + */ + public getScalarDataArrays(): Types.VolumeScalarData[] { + return this.isDynamicVolume() + ? this.scalarData + : [this.scalarData]; + } + + private _getScalarDataByImageIdIndex( + imageIdIndex: number + ): Types.VolumeScalarData { + if (imageIdIndex < 0 || imageIdIndex >= this.imageIds.length) { + throw new Error('imageIdIndex out of range'); + } + + const scalarDataArrays = this.getScalarDataArrays(); + const scalarDataIndex = Math.floor(imageIdIndex / this.numFrames); + + return scalarDataArrays[scalarDataIndex]; + } + + protected invalidateVolume(immediate: boolean): void { + const { imageData, vtkOpenGLTexture } = this; + const { numFrames } = this; + + for (let i = 0; i < numFrames; i++) { + vtkOpenGLTexture.setUpdatedFrame(i); + } + + imageData.modified(); + + if (immediate) { + autoLoad(this.volumeId); + } + } + + /** + * It cancels loading the images of the volume. It sets the loading status to false + * and filters any imageLoad request in the requestPoolManager that has the same + * volumeId + */ + public cancelLoading = (): void => { + const { loadStatus } = this; + + if (!loadStatus || !loadStatus.loading) { + return; + } + + // Set to not loading. + loadStatus.loading = false; + + // Remove all the callback listeners + this.clearLoadCallbacks(); + + // Create a filter function which only keeps requests + // which do not match this volume's Id + const filterFunction = ({ additionalDetails }) => { + return additionalDetails.volumeId !== this.volumeId; + }; + + // Instruct the request pool manager to filter queued + // requests to ensure requests we no longer need are + // no longer sent. + imageLoadPoolManager.filterRequests(filterFunction); + }; + + /** + * Clear the load callbacks + */ + public clearLoadCallbacks(): void { + this.loadStatus.callbacks = []; + } + + /** + * It triggers a prefetch for images in the volume. + * @param callback - A callback function to be called when the volume is fully loaded + * @param priority - The priority for loading the volume images, lower number is higher priority + * @returns + */ + public load = ( + callback: (...args: unknown[]) => void, + priority = 5 + ): void => { + const { imageIds, loadStatus, numFrames } = this; + + if (loadStatus.loading === true) { + console.log( + `loadVolume: Loading is already in progress for ${this.volumeId}` + ); + return; // Already loading, will get callbacks from main load. + } + + const { loaded } = this.loadStatus; + const totalNumFrames = imageIds.length; + + if (loaded) { + if (callback) { + callback({ + success: true, + framesLoaded: totalNumFrames, + framesProcessed: totalNumFrames, + numFrames, + totalNumFrames, + }); + } + return; + } + + if (callback) { + this.loadStatus.callbacks.push(callback); + } + + this._prefetchImageIds(priority); + }; + + protected getImageIdsRequests = ( + imageIds: string[], + scalarData: Types.VolumeScalarData, + priority: number + ) => { + const { loadStatus } = this; + const { cachedFrames } = loadStatus; + + const { vtkOpenGLTexture, imageData, metadata, volumeId } = this; + const { FrameOfReferenceUID } = metadata; + + loadStatus.loading = true; + + // SharedArrayBuffer + const arrayBuffer = scalarData.buffer; + const numFrames = imageIds.length; + + // Length of one frame in voxels + const length = scalarData.length / numFrames; + // Length of one frame in bytes + const lengthInBytes = arrayBuffer.byteLength / numFrames; + + let type; + + if (scalarData instanceof Uint8Array) { + type = 'Uint8Array'; + } else if (scalarData instanceof Float32Array) { + type = 'Float32Array'; + } else { + throw new Error('Unsupported array type'); + } + + const totalNumFrames = this.imageIds.length; + const autoRenderOnLoad = true; + const autoRenderPercentage = 2; + + let reRenderFraction; + let reRenderTarget; + + if (autoRenderOnLoad) { + reRenderFraction = totalNumFrames * (autoRenderPercentage / 100); + reRenderTarget = reRenderFraction; + } + + function callLoadStatusCallback(evt) { + // TODO: probably don't want this here + + if (autoRenderOnLoad) { + if ( + evt.framesProcessed > reRenderTarget || + evt.framesProcessed === evt.totalNumFrames + ) { + reRenderTarget += reRenderFraction; + autoLoad(volumeId); + } + } + + if (evt.framesProcessed === evt.totalNumFrames) { + loadStatus.callbacks.forEach((callback) => callback(evt)); + } + } + + const successCallback = ( + imageIdIndex: number, + imageId: string, + scalingParameters + ) => { + const frameIndex = this._imageIdIndexToFrameIndex(imageIdIndex); + + // Check if there is a cached image for the same imageURI (different + // data loader scheme) + const cachedImage = cache.getCachedImageBasedOnImageURI(imageId); + + // check if we are still loading the volume and we have not canceled loading + if (!loadStatus.loading) { + return; + } + + if (!cachedImage || !cachedImage.image) { + return updateTextureAndTriggerEvents(this, imageIdIndex, imageId); + } + const imageScalarData = this._scaleIfNecessary( + cachedImage.image, + scalingParameters + ); + // todo add scaling and slope + const { pixelsPerImage, bytesPerImage } = this.cornerstoneImageMetaData; + const TypedArray = scalarData.constructor; + let byteOffset = bytesPerImage * frameIndex; + + // create a view on the volume arraybuffer + const bytePerPixel = bytesPerImage / pixelsPerImage; + + if (scalarData.BYTES_PER_ELEMENT !== bytePerPixel) { + byteOffset *= scalarData.BYTES_PER_ELEMENT / bytePerPixel; + } + + // @ts-ignore + const volumeBufferView = new TypedArray( + arrayBuffer, + byteOffset, + pixelsPerImage + ); + cachedImage.imageLoadObject.promise + .then((image) => { + volumeBufferView.set(imageScalarData); + updateTextureAndTriggerEvents(this, imageIdIndex, imageId); + }) + .catch((err) => { + errorCallback(err, imageIdIndex, imageId); + }); + return; + }; + + const updateTextureAndTriggerEvents = ( + volume: BaseStreamingImageVolume, + imageIdIndex, + imageId + ) => { + const frameIndex = this._imageIdIndexToFrameIndex(imageIdIndex); + + cachedFrames[imageIdIndex] = true; + this.framesLoaded++; + this.framesProcessed++; + + vtkOpenGLTexture.setUpdatedFrame(frameIndex); + imageData.modified(); + + const eventDetail: Types.EventTypes.ImageVolumeModifiedEventDetail = { + FrameOfReferenceUID, + imageVolume: volume, + }; + + triggerEvent( + eventTarget, + Enums.Events.IMAGE_VOLUME_MODIFIED, + eventDetail + ); + + if (this.framesProcessed === totalNumFrames) { + loadStatus.loaded = true; + loadStatus.loading = false; + + // TODO: Should we remove the callbacks in favour of just using events? + callLoadStatusCallback({ + success: true, + imageIdIndex, + imageId, + framesLoaded: this.framesLoaded, + framesProcessed: this.framesProcessed, + numFrames, + totalNumFrames, + }); + loadStatus.callbacks = []; + } else { + callLoadStatusCallback({ + success: true, + imageIdIndex, + imageId, + framesLoaded: this.framesLoaded, + framesProcessed: this.framesProcessed, + numFrames, + totalNumFrames, + }); + } + }; + + function errorCallback(error, imageIdIndex, imageId) { + this.framesProcessed++; + + if (this.framesProcessed === totalNumFrames) { + loadStatus.loaded = true; + loadStatus.loading = false; + + callLoadStatusCallback({ + success: false, + imageId, + imageIdIndex, + error, + framesLoaded: this.framesLoaded, + framesProcessed: this.framesProcessed, + numFrames, + totalNumFrames, + }); + + loadStatus.callbacks = []; + } else { + callLoadStatusCallback({ + success: false, + imageId, + imageIdIndex, + error, + framesLoaded: this.framesLoaded, + framesProcessed: this.framesProcessed, + numFrames, + totalNumFrames, + }); + } + + const eventDetail = { + error, + imageIdIndex, + imageId, + }; + + triggerEvent(eventTarget, Enums.Events.IMAGE_LOAD_ERROR, eventDetail); + } + + function handleArrayBufferLoad(scalarData, image, options) { + if (!(scalarData.buffer instanceof ArrayBuffer)) { + return; + } + + const offset = options.targetBuffer.offset; // in bytes + const length = options.targetBuffer.length; // in frames + try { + if (scalarData instanceof Float32Array) { + const bytesInFloat = 4; + const floatView = new Float32Array(image.pixelData); + if (floatView.length !== length) { + throw 'Error pixelData length does not match frame length'; + } + scalarData.set(floatView, offset / bytesInFloat); + } + if (scalarData instanceof Uint8Array) { + const bytesInUint8 = 1; + const intView = new Uint8Array(image.pixelData); + if (intView.length !== length) { + throw 'Error pixelData length does not match frame length'; + } + scalarData.set(intView, offset / bytesInUint8); + } + } catch (e) { + console.error(e); + } + } + + // 4D datasets load one time point at a time and the frameIndex is + // the position of the imageId in the current time point while the + // imageIdIndex is its absolute position in the array that contains + // all other imageIds. In a 4D dataset the frameIndex can also be + // calculated as `imageIdIndex % numFrames` where numFrames is the + // number of frames per time point. The frameIndex and imageIdIndex + // will be the same when working with 3D datasets. + const requests = imageIds.map((imageId, frameIndex) => { + const imageIdIndex = this.getImageIdIndex(imageId); + + if (cachedFrames[imageIdIndex]) { + this.framesLoaded++; + this.framesProcessed++; + return; + } + + const modalityLutModule = + metaData.get('modalityLutModule', imageId) || {}; + + const generalSeriesModule = + metaData.get('generalSeriesModule', imageId) || {}; + + const scalingParameters: Types.ScalingParameters = { + rescaleSlope: modalityLutModule.rescaleSlope, + rescaleIntercept: modalityLutModule.rescaleIntercept, + modality: generalSeriesModule.modality, + }; + + if (scalingParameters.modality === 'PT') { + const suvFactor = metaData.get('scalingModule', imageId); + + if (suvFactor) { + this._addScalingToVolume(suvFactor); + scalingParameters.suvbw = suvFactor.suvbw; + } + } + + const options = { + // WADO Image Loader + targetBuffer: { + // keeping this in the options means a large empty volume array buffer + // will be transferred to the worker. This is undesirable for streaming + // volume without shared array buffer because the target is now an empty + // 300-500MB volume array buffer. Instead the volume should be progressively + // set in the main thread. + arrayBuffer: + arrayBuffer instanceof ArrayBuffer ? undefined : arrayBuffer, + offset: frameIndex * lengthInBytes, + length, + type, + }, + skipCreateImage: true, + preScale: { + enabled: true, + // we need to pass in the scalingParameters here, since the streaming + // volume loader doesn't go through the createImage phase in the loader, + // and therefore doesn't have the scalingParameters + scalingParameters, + }, + }; + + // Use loadImage because we are skipping the Cornerstone Image cache + // when we load directly into the Volume cache + const callLoadImage = (imageId, imageIdIndex, options) => { + return imageLoader.loadImage(imageId, options).then( + (image) => { + // scalarData is the volume container we are progressively loading into + // image is the pixelData decoded from workers in cornerstoneWADOImageLoader + handleArrayBufferLoad(scalarData, image, options); + successCallback(imageIdIndex, imageId, scalingParameters); + }, + (error) => { + errorCallback(error, imageIdIndex, imageId); + } + ); + }; + + return { + callLoadImage, + imageId, + imageIdIndex, + options, + priority, + requestType, + additionalDetails: { + volumeId: this.volumeId, + }, + }; + }); + + return requests; + }; + + /** + * It returns the imageLoad requests for the streaming image volume instance. + * It involves getting all the imageIds of the volume and creating a success callback + * which would update the texture (when the image has loaded) and the failure callback. + * Note that this method does not executes the requests but only returns the requests. + * It can be used for sorting requests outside of the volume loader itself + * e.g. loading a single slice of CT, followed by a single slice of PET (interleaved), before + * moving to the next slice. + * + * @returns Array of requests including imageId of the request, its imageIdIndex, + * options (targetBuffer and scaling parameters), and additionalDetails (volumeId) + */ + public getImageLoadRequests(_priority: number): any[] { + throw new Error('Abstract method'); + } + + private _prefetchImageIds(priority: number): void { + const requests = this.getImageLoadRequests(priority); + + // requests.reverse().forEach((request) => { + requests.forEach((request) => { + if (!request) { + // there is a cached image for the imageId and no requests will fire + return; + } + + const { + callLoadImage, + imageId, + imageIdIndex, + options, + priority, + requestType, + additionalDetails, + } = request; + + imageLoadPoolManager.addRequest( + callLoadImage.bind(this, imageId, imageIdIndex, options), + requestType, + additionalDetails, + priority + ); + }); + } + + /** + * This function decides whether or not to scale the image based on the + * scalingParameters. If the image is already scaled, we should take that + * into account when scaling the image again, so if the rescaleSlope and/or + * rescaleIntercept are different from the ones that were used to scale the + * image, we should scale the image again according to the new parameters. + */ + private _scaleIfNecessary( + image, + scalingParametersToUse: Types.ScalingParameters + ) { + const imageIsAlreadyScaled = image.preScale?.scaled; + const noScalingParametersToUse = + !scalingParametersToUse || + !scalingParametersToUse.rescaleIntercept || + !scalingParametersToUse.rescaleSlope; + + if (!imageIsAlreadyScaled && noScalingParametersToUse) { + // no need to scale the image + return image.getPixelData().slice(0); + } + + if ( + !imageIsAlreadyScaled && + scalingParametersToUse && + scalingParametersToUse.rescaleIntercept !== undefined && + scalingParametersToUse.rescaleSlope !== undefined + ) { + // if not already scaled, just scale the image. + // copy so that it doesn't get modified + const pixelDataCopy = image.getPixelData().slice(0); + const scaledArray = scaleArray(pixelDataCopy, scalingParametersToUse); + return scaledArray; + } + + // if the image is already scaled, + const { + rescaleSlope: rescaleSlopeToUse, + rescaleIntercept: rescaleInterceptToUse, + suvbw: suvbwToUse, + } = scalingParametersToUse; + + const { + rescaleSlope: rescaleSlopeUsed, + rescaleIntercept: rescaleInterceptUsed, + suvbw: suvbwUsed, + } = image.preScale.scalingParameters; + + const rescaleSlopeIsSame = rescaleSlopeToUse === rescaleSlopeUsed; + const rescaleInterceptIsSame = + rescaleInterceptToUse === rescaleInterceptUsed; + const suvbwIsSame = suvbwToUse === suvbwUsed; + + if (rescaleSlopeIsSame && rescaleInterceptIsSame && suvbwIsSame) { + // if the scaling parameters are the same, we don't need to scale the image again + return image.getPixelData(); + } + + const pixelDataCopy = image.getPixelData().slice(0); + // the general formula for scaling is scaledPixelValue = suvbw * (pixelValue * rescaleSlope) + rescaleIntercept + const newSuvbw = suvbwToUse / suvbwUsed; + const newRescaleSlope = rescaleSlopeToUse / rescaleSlopeUsed; + const newRescaleIntercept = + rescaleInterceptToUse - rescaleInterceptUsed * newRescaleSlope; + + const newScalingParameters = { + ...scalingParametersToUse, + rescaleSlope: newRescaleSlope, + rescaleIntercept: newRescaleIntercept, + suvbw: newSuvbw, + }; + + const scaledArray = scaleArray(pixelDataCopy, newScalingParameters); + return scaledArray; + } + + private _addScalingToVolume(suvFactor) { + // Todo: handle case where suvFactors are not the same for all frames + if (this.scaling) { + return; + } + + const { suvbw, suvlbm, suvbsa } = suvFactor; + + const petScaling = {}; + + if (suvlbm) { + petScaling.suvbwToSuvlbm = suvlbm / suvbw; + } + + if (suvbsa) { + petScaling.suvbwToSuvbsa = suvbsa / suvbw; + } + + this.scaling = { PET: petScaling }; + this.isPrescaled = true; + } + + private _removeFromCache() { + // TODO: not 100% sure this is the same Id as the volume loader's volumeId? + // so I have no idea if this will work + cache.removeVolumeLoadObject(this.volumeId); + } + + /** + * Converts the requested imageId inside the volume to a cornerstoneImage + * object. It uses the typedArray set method to copy the pixelData from the + * correct offset in the scalarData to a new array for the image + * + * @param imageId - the imageId of the image to be converted + * @param imageIdIndex - the index of the imageId in the imageIds array + * @returns imageLoadObject containing the promise that resolves + * to the cornerstone image + */ + public convertToCornerstoneImage( + imageId: string, + imageIdIndex: number + ): Types.IImageLoadObject { + const { imageIds } = this; + const frameIndex = this._imageIdIndexToFrameIndex(imageIdIndex); + + const { + bytesPerImage, + pixelsPerImage, + windowCenter, + windowWidth, + numComponents, + color, + dimensions, + spacing, + invert, + voiLUTFunction, + } = this.cornerstoneImageMetaData; + + // 1. Grab the buffer and it's type + const scalarData = this._getScalarDataByImageIdIndex(imageIdIndex); + const volumeBuffer = scalarData.buffer; + // (not sure if this actually works, TypeScript keeps complaining) + const TypedArray = scalarData.constructor; + + // 2. Given the index of the image and frame length in bytes, + // create a view on the volume arraybuffer + const bytePerPixel = bytesPerImage / pixelsPerImage; + + let byteOffset = bytesPerImage * frameIndex; + + // If there is a discrepancy between the volume typed array + // and the bitsAllocated for the image. The reason is that VTK uses Float32 + // on the GPU and if the type is not Float32, it will convert it. So for not + // having a performance issue, we convert all types initially to Float32 even + // if they are not Float32. + if (scalarData.BYTES_PER_ELEMENT !== bytePerPixel) { + byteOffset *= scalarData.BYTES_PER_ELEMENT / bytePerPixel; + } + + // 3. Create a new TypedArray of the same type for the new + // Image that will be created + // @ts-ignore + const imageScalarData = new TypedArray(pixelsPerImage); + // @ts-ignore + const volumeBufferView = new TypedArray( + volumeBuffer, + byteOffset, + pixelsPerImage + ); + + // 4. Use e.g. TypedArray.set() to copy the data from the larger + // buffer's view into the smaller one + imageScalarData.set(volumeBufferView); + + // 5. Create an Image Object from imageScalarData and put it into the Image cache + const volumeImageId = imageIds[imageIdIndex]; + const modalityLutModule = + metaData.get('modalityLutModule', volumeImageId) || {}; + const minMax = getMinMax(imageScalarData); + const intercept = modalityLutModule.rescaleIntercept + ? modalityLutModule.rescaleIntercept + : 0; + + const image: Types.IImage = { + imageId, + intercept, + windowCenter, + windowWidth, + voiLUTFunction, + color, + numComps: numComponents, + rows: dimensions[0], + columns: dimensions[1], + sizeInBytes: imageScalarData.byteLength, + getPixelData: () => imageScalarData, + minPixelValue: minMax.min, + maxPixelValue: minMax.max, + slope: modalityLutModule.rescaleSlope + ? modalityLutModule.rescaleSlope + : 1, + getCanvas: undefined, // todo: which canvas? + height: dimensions[0], + width: dimensions[1], + rgba: undefined, // todo: how + columnPixelSpacing: spacing[0], + rowPixelSpacing: spacing[1], + invert, + }; + + // 5. Create the imageLoadObject + const imageLoadObject = { + promise: Promise.resolve(image), + }; + + return imageLoadObject; + } + + /** + * Converts all the volume images (imageIds) to cornerstoneImages and caches them. + * It iterates over all the imageIds and convert them until there is no + * enough space left inside the imageCache. Finally it will decache the Volume. + * + */ + private _convertToImages() { + // 1. Try to decache images in the volatile Image Cache to provide + // enough space to store another entire copy of the volume (as Images). + // If we do not have enough, we will store as many images in the cache + // as possible, and the rest of the volume will be decached. + const byteLength = this.sizeInBytes; + const numImages = this.imageIds.length; + const { bytesPerImage } = this.cornerstoneImageMetaData; + + let bytesRemaining = cache.decacheIfNecessaryUntilBytesAvailable( + byteLength, + this.imageIds + ); + + for (let imageIdIndex = 0; imageIdIndex < numImages; imageIdIndex++) { + const imageId = this.imageIds[imageIdIndex]; + + bytesRemaining = bytesRemaining - bytesPerImage; + + // 2. Convert each imageId to a cornerstone Image object which is + // resolved inside the promise of imageLoadObject + const imageLoadObject = this.convertToCornerstoneImage( + imageId, + imageIdIndex + ); + + // 3. Caching the image + cache.putImageLoadObject(imageId, imageLoadObject).catch((err) => { + console.error(err); + }); + + // 4. If we know we won't be able to add another Image to the cache + // without breaching the limit, stop here. + if (bytesRemaining <= bytesPerImage) { + break; + } + } + // 5. When as much of the Volume is processed into Images as possible + // without breaching the cache limit, remove the Volume + this._removeFromCache(); + } + + /** + * If completelyRemove is true, remove the volume completely from the cache. Otherwise, + * convert the volume to cornerstone images (stack images) and store it in the cache + * @param completelyRemove - If true, the image will be removed from the + * cache completely. + */ + public decache(completelyRemove = false): void { + if (completelyRemove) { + this._removeFromCache(); + } else { + this._convertToImages(); + } + } +} diff --git a/packages/streaming-image-volume-loader/src/StreamingDynamicImageVolume.ts b/packages/streaming-image-volume-loader/src/StreamingDynamicImageVolume.ts new file mode 100644 index 000000000..ca07a29ee --- /dev/null +++ b/packages/streaming-image-volume-loader/src/StreamingDynamicImageVolume.ts @@ -0,0 +1,182 @@ +import type { Types } from '@cornerstonejs/core'; +import BaseStreamingImageVolume from './BaseStreamingImageVolume'; + +type TimePoint = { + /** imageIds of each timepoint */ + imageIds: Array; + /** volume scalar data */ + scalarData: Float32Array | Uint8Array; +}; + +/** + * Streaming Image Volume Class that extends StreamingImageVolume base class. + * It implements load method to load the imageIds and insert them into the volume. + */ +export default class StreamingDynamicImageVolume + extends BaseStreamingImageVolume + implements Types.IDynamicImageVolume +{ + private _numTimePoints: number; + private _timePoints: TimePoint[]; + private _timePointIndex = 0; + + constructor( + imageVolumeProperties: Types.IVolume, + streamingProperties: Types.IStreamingVolumeProperties + ) { + StreamingDynamicImageVolume._ensureValidData( + imageVolumeProperties, + streamingProperties + ); + + super(imageVolumeProperties, streamingProperties); + this._numTimePoints = (this.scalarData).length; + this._timePoints = this._getTimePointsData(); + } + + private static _ensureValidData( + imageVolumeProperties: Types.IVolume, + streamingProperties: Types.IStreamingVolumeProperties + ): void { + const imageIds = streamingProperties.imageIds; + const scalarDataArrays = ( + imageVolumeProperties.scalarData + ); + + if (imageIds.length % scalarDataArrays.length !== 0) { + throw new Error( + `Number of imageIds is not a multiple of ${scalarDataArrays.length}` + ); + } + } + + /** + * Use the image ids and scalar data array to create TimePoint objects + * and make it a bit easier to work with when loading requests + */ + private _getTimePointsData(): TimePoint[] { + const { imageIds } = this; + const scalarData = this.scalarData; + + const { numFrames } = this; + const numTimePoints = scalarData.length; + const timePoints: TimePoint[] = []; + + for (let i = 0; i < numTimePoints; i++) { + const start = i * numFrames; + const end = start + numFrames; + + timePoints.push({ + imageIds: imageIds.slice(start, end), + scalarData: scalarData[i], + }); + } + + return timePoints; + } + + private _getTimePointsToLoad() { + const timePoints = this._timePoints; + const initialTimePointIndex = this._timePointIndex; + const timePointsToLoad = [timePoints[initialTimePointIndex]]; + + let leftIndex = initialTimePointIndex - 1; + let rightIndex = initialTimePointIndex + 1; + + while (leftIndex >= 0 || rightIndex < timePoints.length) { + if (leftIndex >= 0) { + timePointsToLoad.push(timePoints[leftIndex--]); + } + + if (rightIndex < timePoints.length) { + timePointsToLoad.push(timePoints[rightIndex++]); + } + } + + return timePointsToLoad; + } + + private _getTimePointRequests = (timePoint, priority: number) => { + const { imageIds, scalarData } = timePoint; + + return this.getImageIdsRequests(imageIds, scalarData, priority); + }; + + private _getTimePointsRequests = (priority: number) => { + const timePoints = this._getTimePointsToLoad(); + let timePointsRequests = []; + + timePoints.forEach((timePoint) => { + const timePointRequests = this._getTimePointRequests(timePoint, priority); + timePointsRequests = timePointsRequests.concat(timePointRequests); + }); + + return timePointsRequests; + }; + + /** return true if it is a 4D volume or false if it is 3D volume */ + public isDynamicVolume(): boolean { + return true; + } + + /** + * Returns the active time point index + * @returns active time point index + */ + public get timePointIndex(): number { + return this._timePointIndex; + } + + /** + * Set the active time point index which also updates the active scalar data + * @returns current time point index + */ + public set timePointIndex(newTimePointIndex: number) { + if (newTimePointIndex < 0 || newTimePointIndex >= this.numTimePoints) { + throw new Error(`Invalid timePointIndex (${newTimePointIndex})`); + } + + // Nothing to do when time point index does not change + if (this._timePointIndex === newTimePointIndex) { + return; + } + + const { imageData } = this; + + this._timePointIndex = newTimePointIndex; + imageData.getPointData().setActiveScalars(`timePoint-${newTimePointIndex}`); + this.invalidateVolume(true); + } + + /** + * Returns the number of time points + * @returns number of time points + */ + public get numTimePoints(): number { + return this._numTimePoints; + } + + /** + * Return the active scalar data (buffer) + * @returns volume scalar data + */ + public getScalarData(): Types.VolumeScalarData { + return (this.scalarData)[this._timePointIndex]; + } + + /** + * It returns the imageLoad requests for the streaming image volume instance. + * It involves getting all the imageIds of the volume and creating a success callback + * which would update the texture (when the image has loaded) and the failure callback. + * Note that this method does not executes the requests but only returns the requests. + * It can be used for sorting requests outside of the volume loader itself + * e.g. loading a single slice of CT, followed by a single slice of PET (interleaved), before + * moving to the next slice. + * + * @returns Array of requests including imageId of the request, its imageIdIndex, + * options (targetBuffer and scaling parameters), and additionalDetails (volumeId) + */ + public getImageLoadRequests = (priority: number) => { + return this._getTimePointsRequests(priority); + }; +} diff --git a/packages/streaming-image-volume-loader/src/StreamingImageVolume.ts b/packages/streaming-image-volume-loader/src/StreamingImageVolume.ts index ae88cb3e8..9da664754 100644 --- a/packages/streaming-image-volume-loader/src/StreamingImageVolume.ts +++ b/packages/streaming-image-volume-loader/src/StreamingImageVolume.ts @@ -1,184 +1,31 @@ -import { - Enums, - eventTarget, - metaData, - imageLoadPoolManager, - triggerEvent, - ImageVolume, - cache, - imageLoader, - utilities as csUtils, -} from '@cornerstonejs/core'; - -import type { Types } from '@cornerstonejs/core'; -import { scaleArray, autoLoad } from './helpers'; - -const requestType = Enums.RequestType.Prefetch; -const { getMinMax } = csUtils; +import { Types } from '@cornerstonejs/core'; +import BaseStreamingImageVolume from './BaseStreamingImageVolume'; /** * Streaming Image Volume Class that extends ImageVolume base class. * It implements load method to load the imageIds and insert them into the volume. */ -export default class StreamingImageVolume extends ImageVolume { - private _cornerstoneImageMetaData; - - loadStatus: { - loaded: boolean; - loading: boolean; - cachedFrames: Array; - callbacks: Array<(...args: unknown[]) => void>; - }; - +export default class StreamingImageVolume extends BaseStreamingImageVolume { constructor( imageVolumeProperties: Types.IVolume, streamingProperties: Types.IStreamingVolumeProperties ) { - super(imageVolumeProperties); - this.imageIds = streamingProperties.imageIds; - this.loadStatus = streamingProperties.loadStatus; - - this._createCornerstoneImageMetaData(); + super(imageVolumeProperties, streamingProperties); } /** - * Creates the metadata required for converting the volume to an cornerstoneImage + * Return the scalar data (buffer) + * @returns volume scalar data */ - private _createCornerstoneImageMetaData() { - const numImages = this.imageIds.length; - const bytesPerImage = this.sizeInBytes / numImages; - const numComponents = this.scalarData.length / this.numVoxels; - const pixelsPerImage = - this.dimensions[0] * this.dimensions[1] * numComponents; - - const { PhotometricInterpretation, voiLut, VOILUTFunction } = this.metadata; - - let windowCenter = []; - let windowWidth = []; - - if (voiLut && voiLut.length) { - windowCenter = voiLut.map((voi) => { - return voi.windowCenter; - }); - - windowWidth = voiLut.map((voi) => { - return voi.windowWidth; - }); - } - - const color = numComponents > 1 ? true : false; //todo: fix this - - this._cornerstoneImageMetaData = { - bytesPerImage, - numComponents, - pixelsPerImage, - windowCenter, - windowWidth, - color, - spacing: this.spacing, - dimensions: this.dimensions, - PhotometricInterpretation, - voiLUTFunction: VOILUTFunction, - invert: PhotometricInterpretation === 'MONOCHROME1', - }; + public getScalarData(): Types.VolumeScalarData { + return this.scalarData; } - private _hasLoaded = (): boolean => { - const { loadStatus, imageIds } = this; - const numFrames = imageIds.length; - - for (let i = 0; i < numFrames; i++) { - if (!loadStatus.cachedFrames[i]) { - return false; - } - } - - return true; - }; - - /** - * It cancels loading the images of the volume. It sets the loading status to false - * and filters any imageLoad request in the requestPoolManager that has the same - * volumeId - */ - public cancelLoading = () => { - const { loadStatus } = this; - - if (!loadStatus || !loadStatus.loading) { - return; - } - - // Set to not loading. - loadStatus.loading = false; - - // Remove all the callback listeners - this.clearLoadCallbacks(); - - // Create a filter function which only keeps requests - // which do not match this volume's Id - const filterFunction = ({ additionalDetails }) => { - return additionalDetails.volumeId !== this.volumeId; - }; - - // Instruct the request pool manager to filter queued - // requests to ensure requests we no longer need are - // no longer sent. - imageLoadPoolManager.filterRequests(filterFunction); - }; - - /** - * Clear the load callbacks - */ - public clearLoadCallbacks(): void { - this.loadStatus.callbacks = []; - } - - /** - * It triggers a prefetch for images in the volume. - * @param callback - A callback function to be called when the volume is fully loaded - * @param priority - The priority for loading the volume images, lower number is higher priority - * @returns - */ - public load = ( - callback: (...args: unknown[]) => void, - priority = 5 - ): void => { - const { imageIds, loadStatus } = this; - - if (loadStatus.loading === true) { - console.log( - `loadVolume: Loading is already in progress for ${this.volumeId}` - ); - return; // Already loading, will get callbacks from main load. - } - - const { loaded } = this.loadStatus; - const numFrames = imageIds.length; - - if (loaded) { - if (callback) { - callback({ - success: true, - framesLoaded: numFrames, - numFrames, - framesProcessed: numFrames, - }); - } - return; - } - - if (callback) { - this.loadStatus.callbacks.push(callback); - } - - this._prefetchImageIds(priority); - }; - /** * It returns the imageLoad requests for the streaming image volume instance. * It involves getting all the imageIds of the volume and creating a success callback * which would update the texture (when the image has loaded) and the failure callback. - * Note that this method does not run executes the requests but only returns the requests. + * Note that this method does not executes the requests but only returns the requests. * It can be used for sorting requests outside of the volume loader itself * e.g. loading a single slice of CT, followed by a single slice of PET (interleaved), before * moving to the next slice. @@ -187,614 +34,9 @@ export default class StreamingImageVolume extends ImageVolume { * options (targetBuffer and scaling parameters), and additionalDetails (volumeId) */ public getImageLoadRequests = (priority: number) => { - const { scalarData, loadStatus } = this; - const { cachedFrames } = loadStatus; - - const { imageIds, vtkOpenGLTexture, imageData, metadata, volumeId } = this; - const { FrameOfReferenceUID } = metadata; - loadStatus.loading = true; - - // SharedArrayBuffer - const arrayBuffer = scalarData.buffer; - const numFrames = imageIds.length; - - // Length of one frame in voxels - const length = scalarData.length / numFrames; - // Length of one frame in bytes - const lengthInBytes = arrayBuffer.byteLength / numFrames; - - let type; - - if (scalarData instanceof Uint8Array) { - type = 'Uint8Array'; - } else if (scalarData instanceof Float32Array) { - type = 'Float32Array'; - } else { - throw new Error('Unsupported array type'); - } - - let framesLoaded = 0; - let framesProcessed = 0; - - const autoRenderOnLoad = true; - const autoRenderPercentage = 2; - - let reRenderFraction; - let reRenderTarget; - - if (autoRenderOnLoad) { - reRenderFraction = numFrames * (autoRenderPercentage / 100); - reRenderTarget = reRenderFraction; - } - - function callLoadStatusCallback(evt) { - // TODO: probably don't want this here - if (autoRenderOnLoad) { - if ( - evt.framesProcessed > reRenderTarget || - evt.framesProcessed === evt.numFrames - ) { - reRenderTarget += reRenderFraction; - - autoLoad(volumeId); - } - } - - if (evt.framesProcessed === evt.numFrames) { - loadStatus.callbacks.forEach((callback) => callback(evt)); - } - } - - const successCallback = ( - imageIdIndex: number, - imageId: string, - scalingParameters - ) => { - // Check if there is a cached image for the same imageURI (different - // data loader scheme) - const cachedImage = cache.getCachedImageBasedOnImageURI(imageId); - - // check if we are still loading the volume and we have not canceled loading - if (!loadStatus.loading) { - return; - } - - if (!cachedImage || !cachedImage.image) { - return updateTextureAndTriggerEvents(this, imageIdIndex, imageId); - } - const imageScalarData = this._scaleIfNecessary( - cachedImage.image, - scalingParameters - ); - // todo add scaling and slope - const { pixelsPerImage, bytesPerImage } = this._cornerstoneImageMetaData; - const TypedArray = this.scalarData.constructor; - let byteOffset = bytesPerImage * imageIdIndex; - - // create a view on the volume arraybuffer - const bytePerPixel = bytesPerImage / pixelsPerImage; - - if (this.scalarData.BYTES_PER_ELEMENT !== bytePerPixel) { - byteOffset *= this.scalarData.BYTES_PER_ELEMENT / bytePerPixel; - } - - // @ts-ignore - const volumeBufferView = new TypedArray( - arrayBuffer, - byteOffset, - pixelsPerImage - ); - cachedImage.imageLoadObject.promise - .then((image) => { - volumeBufferView.set(imageScalarData); - updateTextureAndTriggerEvents(this, imageIdIndex, imageId); - }) - .catch((err) => { - errorCallback(err, imageIdIndex, imageId); - }); - return; - }; - - function updateTextureAndTriggerEvents( - volume: StreamingImageVolume, - imageIdIndex, - imageId - ) { - cachedFrames[imageIdIndex] = true; - framesLoaded++; - framesProcessed++; - - vtkOpenGLTexture.setUpdatedFrame(imageIdIndex); - imageData.modified(); - - const eventDetail: Types.EventTypes.ImageVolumeModifiedEventDetail = { - FrameOfReferenceUID, - imageVolume: volume, - }; - - triggerEvent( - eventTarget, - Enums.Events.IMAGE_VOLUME_MODIFIED, - eventDetail - ); - - if (framesProcessed === numFrames) { - loadStatus.loaded = true; - loadStatus.loading = false; - - // TODO: Should we remove the callbacks in favour of just using events? - callLoadStatusCallback({ - success: true, - imageIdIndex, - imageId, - framesLoaded, - framesProcessed, - numFrames, - }); - loadStatus.callbacks = []; - } else { - callLoadStatusCallback({ - success: true, - imageIdIndex, - imageId, - framesLoaded, - framesProcessed, - numFrames, - }); - } - } - - function errorCallback(error, imageIdIndex, imageId) { - framesProcessed++; - - if (framesProcessed === numFrames) { - loadStatus.loaded = true; - loadStatus.loading = false; - - callLoadStatusCallback({ - success: false, - imageId, - imageIdIndex, - error, - framesLoaded, - framesProcessed, - numFrames, - }); - - loadStatus.callbacks = []; - } else { - callLoadStatusCallback({ - success: false, - imageId, - imageIdIndex, - error, - framesLoaded, - framesProcessed, - numFrames, - }); - } - - const eventDetail = { - error, - imageIdIndex, - imageId, - }; - - triggerEvent(eventTarget, Enums.Events.IMAGE_LOAD_ERROR, eventDetail); - } - - function handleArrayBufferLoad(scalarData, image, options) { - if (!(scalarData.buffer instanceof ArrayBuffer)) { - return; - } - - const offset = options.targetBuffer.offset; // in bytes - const length = options.targetBuffer.length; // in frames - try { - if (scalarData instanceof Float32Array) { - const bytesInFloat = 4; - const floatView = new Float32Array(image.pixelData); - if (floatView.length !== length) { - throw 'Error pixelData length does not match frame length'; - } - scalarData.set(floatView, offset / bytesInFloat); - } - if (scalarData instanceof Uint8Array) { - const bytesInUint8 = 1; - const intView = new Uint8Array(image.pixelData); - if (intView.length !== length) { - throw 'Error pixelData length does not match frame length'; - } - scalarData.set(intView, offset / bytesInUint8); - } - } catch (e) { - console.error(e); - } - } - - const requests = imageIds.map((imageId, imageIdIndex) => { - if (cachedFrames[imageIdIndex]) { - framesLoaded++; - framesProcessed++; - return; - } - - const modalityLutModule = - metaData.get('modalityLutModule', imageId) || {}; - - const generalSeriesModule = - metaData.get('generalSeriesModule', imageId) || {}; - - const scalingParameters: Types.ScalingParameters = { - rescaleSlope: modalityLutModule.rescaleSlope, - rescaleIntercept: modalityLutModule.rescaleIntercept, - modality: generalSeriesModule.modality, - }; - - if (scalingParameters.modality === 'PT') { - const suvFactor = metaData.get('scalingModule', imageId); - - if (suvFactor) { - this._addScalingToVolume(suvFactor); - scalingParameters.suvbw = suvFactor.suvbw; - } - } - - const options = { - // WADO Image Loader - targetBuffer: { - // keeping this in the options means a large empty volume array buffer - // will be transferred to the worker. This is undesirable for streaming - // volume without shared array buffer because the target is now an empty - // 300-500MB volume array buffer. Instead the volume should be progressively - // set in the main thread. - arrayBuffer: - arrayBuffer instanceof ArrayBuffer ? undefined : arrayBuffer, - offset: imageIdIndex * lengthInBytes, - length, - type, - }, - skipCreateImage: true, - preScale: { - enabled: true, - // we need to pass in the scalingParameters here, since the streaming - // volume loader doesn't go through the createImage phase in the loader, - // and therefore doesn't have the scalingParameters - scalingParameters, - }, - }; - - // Use loadImage because we are skipping the Cornerstone Image cache - // when we load directly into the Volume cache - const callLoadImage = (imageId, imageIdIndex, options) => { - return imageLoader.loadImage(imageId, options).then( - (image) => { - // scalarData is the volume container we are progressively loading into - // image is the pixelData decoded from workers in cornerstoneWADOImageLoader - const scalarData = this.scalarData; - handleArrayBufferLoad(scalarData, image, options); - successCallback(imageIdIndex, imageId, scalingParameters); - }, - (error) => { - errorCallback(error, imageIdIndex, imageId); - } - ); - }; - - return { - callLoadImage, - imageId, - imageIdIndex, - options, - priority, - requestType, - additionalDetails: { - volumeId: this.volumeId, - }, - }; - }); - - return requests; - }; - - private _prefetchImageIds(priority: number) { - const requests = this.getImageLoadRequests(priority); - - requests.reverse().forEach((request) => { - if (!request) { - // there is a cached image for the imageId and no requests will fire - return; - } - - const { - callLoadImage, - imageId, - imageIdIndex, - options, - priority, - requestType, - additionalDetails, - } = request; - - imageLoadPoolManager.addRequest( - callLoadImage.bind(this, imageId, imageIdIndex, options), - requestType, - additionalDetails, - priority - ); - }); - } - - /** - * This function decides whether or not to scale the image based on the - * scalingParameters. If the image is already scaled, we should take that - * into account when scaling the image again, so if the rescaleSlope and/or - * rescaleIntercept are different from the ones that were used to scale the - * image, we should scale the image again according to the new parameters. - */ - private _scaleIfNecessary( - image, - scalingParametersToUse: Types.ScalingParameters - ) { - const imageIsAlreadyScaled = image.preScale?.scaled; - const noScalingParametersToUse = - !scalingParametersToUse || - !scalingParametersToUse.rescaleIntercept || - !scalingParametersToUse.rescaleSlope; - - if (!imageIsAlreadyScaled && noScalingParametersToUse) { - // no need to scale the image - return image.getPixelData().slice(0); - } - - if ( - !imageIsAlreadyScaled && - scalingParametersToUse && - scalingParametersToUse.rescaleIntercept !== undefined && - scalingParametersToUse.rescaleSlope !== undefined - ) { - // if not already scaled, just scale the image. - // copy so that it doesn't get modified - const pixelDataCopy = image.getPixelData().slice(0); - const scaledArray = scaleArray(pixelDataCopy, scalingParametersToUse); - return scaledArray; - } - - // if the image is already scaled, - const { - rescaleSlope: rescaleSlopeToUse, - rescaleIntercept: rescaleInterceptToUse, - suvbw: suvbwToUse, - } = scalingParametersToUse; - - const { - rescaleSlope: rescaleSlopeUsed, - rescaleIntercept: rescaleInterceptUsed, - suvbw: suvbwUsed, - } = image.preScale.scalingParameters; - - const rescaleSlopeIsSame = rescaleSlopeToUse === rescaleSlopeUsed; - const rescaleInterceptIsSame = - rescaleInterceptToUse === rescaleInterceptUsed; - const suvbwIsSame = suvbwToUse === suvbwUsed; - - if (rescaleSlopeIsSame && rescaleInterceptIsSame && suvbwIsSame) { - // if the scaling parameters are the same, we don't need to scale the image again - return image.getPixelData(); - } - - const pixelDataCopy = image.getPixelData().slice(0); - // the general formula for scaling is scaledPixelValue = suvbw * (pixelValue * rescaleSlope) + rescaleIntercept - const newSuvbw = suvbwToUse / suvbwUsed; - const newRescaleSlope = rescaleSlopeToUse / rescaleSlopeUsed; - const newRescaleIntercept = - rescaleInterceptToUse - rescaleInterceptUsed * newRescaleSlope; - - const newScalingParameters = { - ...scalingParametersToUse, - rescaleSlope: newRescaleSlope, - rescaleIntercept: newRescaleIntercept, - suvbw: newSuvbw, - }; - - const scaledArray = scaleArray(pixelDataCopy, newScalingParameters); - return scaledArray; - } - - private _addScalingToVolume(suvFactor) { - // Todo: handle case where suvFactors are not the same for all frames - if (this.scaling) { - return; - } - - const { suvbw, suvlbm, suvbsa } = suvFactor; - - const petScaling = {}; - - if (suvlbm) { - petScaling.suvbwToSuvlbm = suvlbm / suvbw; - } - - if (suvbsa) { - petScaling.suvbwToSuvbsa = suvbsa / suvbw; - } - - this.scaling = { PET: petScaling }; - this.isPrescaled = true; - } - - private _removeFromCache() { - // TODO: not 100% sure this is the same Id as the volume loader's volumeId? - // so I have no idea if this will work - cache.removeVolumeLoadObject(this.volumeId); - } - - /** - * Converts the requested imageId inside the volume to a cornerstoneImage - * object. It uses the typedArray set method to copy the pixelData from the - * correct offset in the scalarData to a new array for the image - * - * @param imageId - the imageId of the image to be converted - * @param imageIdIndex - the index of the imageId in the imageIds array - * @returns imageLoadObject containing the promise that resolves - * to the cornerstone image - */ - public convertToCornerstoneImage( - imageId: string, - imageIdIndex: number - ): Types.IImageLoadObject { const { imageIds } = this; + const scalarData = this.scalarData; - const { - bytesPerImage, - pixelsPerImage, - windowCenter, - windowWidth, - numComponents, - color, - dimensions, - spacing, - invert, - voiLUTFunction, - } = this._cornerstoneImageMetaData; - - // 1. Grab the buffer and it's type - const volumeBuffer = this.scalarData.buffer; - // (not sure if this actually works, TypeScript keeps complaining) - const TypedArray = this.scalarData.constructor; - - // 2. Given the index of the image and frame length in bytes, - // create a view on the volume arraybuffer - const bytePerPixel = bytesPerImage / pixelsPerImage; - - let byteOffset = bytesPerImage * imageIdIndex; - - // If there is a discrepancy between the volume typed array - // and the bitsAllocated for the image. The reason is that VTK uses Float32 - // on the GPU and if the type is not Float32, it will convert it. So for not - // having a performance issue, we convert all types initially to Float32 even - // if they are not Float32. - if (this.scalarData.BYTES_PER_ELEMENT !== bytePerPixel) { - byteOffset *= this.scalarData.BYTES_PER_ELEMENT / bytePerPixel; - } - - // 3. Create a new TypedArray of the same type for the new - // Image that will be created - // @ts-ignore - const imageScalarData = new TypedArray(pixelsPerImage); - // @ts-ignore - const volumeBufferView = new TypedArray( - volumeBuffer, - byteOffset, - pixelsPerImage - ); - - // 4. Use e.g. TypedArray.set() to copy the data from the larger - // buffer's view into the smaller one - imageScalarData.set(volumeBufferView); - - // 5. Create an Image Object from imageScalarData and put it into the Image cache - const volumeImageId = imageIds[imageIdIndex]; - const modalityLutModule = - metaData.get('modalityLutModule', volumeImageId) || {}; - const minMax = getMinMax(imageScalarData); - const intercept = modalityLutModule.rescaleIntercept - ? modalityLutModule.rescaleIntercept - : 0; - - const image: Types.IImage = { - imageId, - intercept, - windowCenter, - windowWidth, - voiLUTFunction, - color, - numComps: numComponents, - rows: dimensions[0], - columns: dimensions[1], - sizeInBytes: imageScalarData.byteLength, - getPixelData: () => imageScalarData, - minPixelValue: minMax.min, - maxPixelValue: minMax.max, - slope: modalityLutModule.rescaleSlope - ? modalityLutModule.rescaleSlope - : 1, - getCanvas: undefined, // todo: which canvas? - height: dimensions[0], - width: dimensions[1], - rgba: undefined, // todo: how - columnPixelSpacing: spacing[0], - rowPixelSpacing: spacing[1], - invert, - }; - - // 5. Create the imageLoadObject - const imageLoadObject = { - promise: Promise.resolve(image), - }; - - return imageLoadObject; - } - - /** - * Converts all the volume images (imageIds) to cornerstoneImages and caches them. - * It iterates over all the imageIds and convert them until there is no - * enough space left inside the imageCache. Finally it will decache the Volume. - * - */ - private _convertToImages() { - // 1. Try to decache images in the volatile Image Cache to provide - // enough space to store another entire copy of the volume (as Images). - // If we do not have enough, we will store as many images in the cache - // as possible, and the rest of the volume will be decached. - const byteLength = this.sizeInBytes; - const numImages = this.imageIds.length; - const { bytesPerImage } = this._cornerstoneImageMetaData; - - let bytesRemaining = cache.decacheIfNecessaryUntilBytesAvailable( - byteLength, - this.imageIds - ); - - for (let imageIdIndex = 0; imageIdIndex < numImages; imageIdIndex++) { - const imageId = this.imageIds[imageIdIndex]; - - bytesRemaining = bytesRemaining - bytesPerImage; - - // 2. Convert each imageId to a cornerstone Image object which is - // resolved inside the promise of imageLoadObject - const imageLoadObject = this.convertToCornerstoneImage( - imageId, - imageIdIndex - ); - - // 3. Caching the image - cache.putImageLoadObject(imageId, imageLoadObject).catch((err) => { - console.error(err); - }); - - // 4. If we know we won't be able to add another Image to the cache - // without breaching the limit, stop here. - if (bytesRemaining <= bytesPerImage) { - break; - } - } - // 5. When as much of the Volume is processed into Images as possible - // without breaching the cache limit, remove the Volume - this._removeFromCache(); - } - - /** - * If completelyRemove is true, remove the volume completely from the cache. Otherwise, - * convert the volume to cornerstone images (stack images) and store it in the cache - * @param completelyRemove - If true, the image will be removed from the - * cache completely. - */ - public decache(completelyRemove = false): void { - if (completelyRemove) { - this._removeFromCache(); - } else { - this._convertToImages(); - } - } + return this.getImageIdsRequests(imageIds, scalarData, priority); + }; } diff --git a/packages/streaming-image-volume-loader/src/cornerstoneStreamingDynamicImageVolumeLoader.ts b/packages/streaming-image-volume-loader/src/cornerstoneStreamingDynamicImageVolumeLoader.ts new file mode 100644 index 000000000..4be9b4e21 --- /dev/null +++ b/packages/streaming-image-volume-loader/src/cornerstoneStreamingDynamicImageVolumeLoader.ts @@ -0,0 +1,96 @@ +import { getVolumeInfo, splitImageIdsBy4DTags } from './helpers'; +import StreamingDynamicImageVolume from './StreamingDynamicImageVolume'; + +interface IVolumeLoader { + promise: Promise; + cancel: () => void; + decache: () => void; +} + +function get4DVolumeInfo(imageIds: string[]) { + const imageIdsGroups = splitImageIdsBy4DTags(imageIds); + return imageIdsGroups.map((imageIds) => getVolumeInfo(imageIds)); +} + +/** + * It handles loading of a image by streaming in its imageIds. It will be the + * volume loader if the schema for the volumeID is `cornerstoneStreamingImageVolume`. + * This function returns a promise that resolves to the StreamingDynamicImageVolume instance. + * + * In order to use the cornerstoneStreamingDynamicImageVolumeLoader you should use + * createAndCacheVolume helper from the cornerstone-core volumeLoader module. + * + * @param volumeId - The ID of the volume + * @param options - options for loading, imageIds + * @returns a promise that resolves to a StreamingDynamicImageVolume + */ +function cornerstoneStreamingDynamicImageVolumeLoader( + volumeId: string, + options: { + imageIds: string[]; + } +): IVolumeLoader { + if (!options || !options.imageIds || !options.imageIds.length) { + throw new Error( + 'ImageIds must be provided to create a 4D streaming image volume' + ); + } + + const { imageIds } = options; + const volumesInfo = get4DVolumeInfo(imageIds); + + const { + metadata: volumeMetadata, + dimensions, + spacing, + origin, + direction, + sizeInBytes, + } = volumesInfo[0]; + + const sortedImageIdsArrays = []; + const scalarDataArrays = []; + + volumesInfo.forEach((volumeInfo) => { + sortedImageIdsArrays.push(volumeInfo.sortedImageIds); + scalarDataArrays.push(volumeInfo.scalarData); + }); + + let streamingImageVolume = new StreamingDynamicImageVolume( + // ImageVolume properties + { + volumeId, + metadata: volumeMetadata, + dimensions, + spacing, + origin, + direction, + scalarData: scalarDataArrays, + sizeInBytes, + }, + // Streaming properties + { + imageIds: sortedImageIdsArrays.flat(), + loadStatus: { + // todo: loading and loaded should be on ImageVolume + loaded: false, + loading: false, + cachedFrames: [], + callbacks: [], + }, + } + ); + + return { + promise: Promise.resolve(streamingImageVolume), + decache: () => { + streamingImageVolume.destroy(); + streamingImageVolume = null; + }, + cancel: () => { + streamingImageVolume.cancelLoading(); + }, + }; +} + +export default cornerstoneStreamingDynamicImageVolumeLoader; diff --git a/packages/streaming-image-volume-loader/src/cornerstoneStreamingImageVolumeLoader.ts b/packages/streaming-image-volume-loader/src/cornerstoneStreamingImageVolumeLoader.ts index 17421becf..03d5ad5db 100644 --- a/packages/streaming-image-volume-loader/src/cornerstoneStreamingImageVolumeLoader.ts +++ b/packages/streaming-image-volume-loader/src/cornerstoneStreamingImageVolumeLoader.ts @@ -218,8 +218,7 @@ function cornerstoneStreamingImageVolumeLoader( promise: streamingImageVolumePromise, decache: () => { streamingImageVolumePromise.then((streamingImageVolume) => { - streamingImageVolume.vtkOpenGLTexture.delete(); - streamingImageVolume.scalarData = null; + streamingImageVolume.destroy(); streamingImageVolume = null; }); }, diff --git a/packages/streaming-image-volume-loader/src/helpers/getVolumeInfo.ts b/packages/streaming-image-volume-loader/src/helpers/getVolumeInfo.ts new file mode 100644 index 000000000..4ef98b51d --- /dev/null +++ b/packages/streaming-image-volume-loader/src/helpers/getVolumeInfo.ts @@ -0,0 +1,121 @@ +import { cache, utilities, Enums } from '@cornerstonejs/core'; +import type { Types } from '@cornerstonejs/core'; +import { vec3 } from 'gl-matrix'; +import makeVolumeMetadata from './makeVolumeMetadata'; +import sortImageIdsAndGetSpacing from './sortImageIdsAndGetSpacing'; + +const { createUint8SharedArray, createFloat32SharedArray } = utilities; + +function getVolumeInfo(imageIds: string[]) { + const volumeMetadata = makeVolumeMetadata(imageIds); + + const { + BitsAllocated, + PixelRepresentation, + PhotometricInterpretation, + ImageOrientationPatient, + PixelSpacing, + Columns, + Rows, + } = volumeMetadata; + + const rowCosineVec = vec3.fromValues( + ImageOrientationPatient[0], + ImageOrientationPatient[1], + ImageOrientationPatient[2] + ); + + const colCosineVec = vec3.fromValues( + ImageOrientationPatient[3], + ImageOrientationPatient[4], + ImageOrientationPatient[5] + ); + + const scanAxisNormal = vec3.create(); + + vec3.cross(scanAxisNormal, rowCosineVec, colCosineVec); + + const { zSpacing, origin, sortedImageIds } = sortImageIdsAndGetSpacing( + imageIds, + scanAxisNormal + ); + + const numFrames = imageIds.length; + + // Spacing goes [1] then [0], as [1] is column spacing (x) and [0] is row spacing (y) + const spacing = [PixelSpacing[1], PixelSpacing[0], zSpacing]; + const dimensions = [Columns, Rows, numFrames]; + const direction = [ + ...rowCosineVec, + ...colCosineVec, + ...scanAxisNormal, + ] as Types.Mat3; + const signed = PixelRepresentation === 1; + + // Check if it fits in the cache before we allocate data + // TODO Improve this when we have support for more types + // NOTE: We use 4 bytes per voxel as we are using Float32. + const bytesPerVoxel = BitsAllocated === 16 ? 4 : 1; + const sizeInBytesPerComponent = + bytesPerVoxel * dimensions[0] * dimensions[1] * dimensions[2]; + + let numComponents = 1; + if (PhotometricInterpretation === 'RGB') { + numComponents = 3; + } + + const sizeInBytes = sizeInBytesPerComponent * numComponents; + + // check if there is enough space in unallocated + image Cache + const isCacheable = cache.isCacheable(sizeInBytes); + if (!isCacheable) { + throw new Error(Enums.Events.CACHE_SIZE_EXCEEDED); + } + + cache.decacheIfNecessaryUntilBytesAvailable(sizeInBytes); + + let scalarData; + + switch (BitsAllocated) { + case 8: + if (signed) { + throw new Error( + '8 Bit signed images are not yet supported by this plugin.' + ); + } else { + scalarData = createUint8SharedArray( + dimensions[0] * dimensions[1] * dimensions[2] + ); + } + + break; + + case 16: + scalarData = createFloat32SharedArray( + dimensions[0] * dimensions[1] * dimensions[2] + ); + + break; + + case 24: + // hacky because we don't support alpha channel in dicom + scalarData = createUint8SharedArray( + dimensions[0] * dimensions[1] * dimensions[2] * numComponents + ); + + break; + } + + return { + metadata: volumeMetadata, + sortedImageIds, + dimensions, + spacing, + origin, + direction, + scalarData, + sizeInBytes, + }; +} + +export { getVolumeInfo, getVolumeInfo as default }; diff --git a/packages/streaming-image-volume-loader/src/helpers/index.ts b/packages/streaming-image-volume-loader/src/helpers/index.ts index 75a796e90..3ebf7c9ac 100644 --- a/packages/streaming-image-volume-loader/src/helpers/index.ts +++ b/packages/streaming-image-volume-loader/src/helpers/index.ts @@ -1,6 +1,15 @@ +import getVolumeInfo from './getVolumeInfo'; import sortImageIdsAndGetSpacing from './sortImageIdsAndGetSpacing'; import makeVolumeMetadata from './makeVolumeMetadata'; import autoLoad from './autoLoad'; import scaleArray from './scaleArray'; +import splitImageIdsBy4DTags from './splitImageIdsBy4DTags'; -export { sortImageIdsAndGetSpacing, makeVolumeMetadata, autoLoad, scaleArray }; +export { + getVolumeInfo, + sortImageIdsAndGetSpacing, + makeVolumeMetadata, + autoLoad, + scaleArray, + splitImageIdsBy4DTags, +}; diff --git a/packages/streaming-image-volume-loader/src/helpers/splitImageIdsBy4DTags.ts b/packages/streaming-image-volume-loader/src/helpers/splitImageIdsBy4DTags.ts new file mode 100644 index 000000000..901991fe0 --- /dev/null +++ b/packages/streaming-image-volume-loader/src/helpers/splitImageIdsBy4DTags.ts @@ -0,0 +1,75 @@ +import { metaData } from '@cornerstonejs/core'; + +// TODO: add support for other 4D tags as listed below +// Supported 4D Tags +// (0018,1060) Trigger Time [NOK] +// (0018,0081) Echo Time [NOK] +// (0018,0086) Echo Number [NOK] +// (0020,0100) Temporal Position Identifier [NOK] +// (0054,1300) FrameReferenceTime [OK] + +interface MappedFrameReferenceTime { + imageId: string; + frameReferenceTime: number; +} + +const groupBy = (array, key) => { + return array.reduce((rv, x) => { + (rv[x[key]] = rv[x[key]] || []).push(x); + return rv; + }, {}); +}; + +function splitFramesByFrameReferenceTime(imageIds: string[]): string[][] { + const framesMetadata: Array = imageIds.map( + (imageId: string): MappedFrameReferenceTime => { + const petImageModule = metaData.get('petImageModule', imageId); + const { frameReferenceTime = 0 } = petImageModule ?? {}; + return { imageId, frameReferenceTime }; + } + ); + + const framesGroups = groupBy(framesMetadata, 'frameReferenceTime'); + const sortedFrameReferenceTimes = Object.keys(framesGroups) + .map(Number.parseFloat) + .sort((a, b) => a - b); + + const imageIdsGroups = sortedFrameReferenceTimes.map((key) => + framesGroups[key].map((item) => item.imageId) + ); + + return imageIdsGroups; +} + +/** + * Split the imageIds array by 4D tags into groups. Each group must have the + * same number of imageIds or the same imageIds array passed in is returned. + * @param imageIds - array of imageIds + * @returns imageIds grouped by 4D tags + */ +function splitImageIdsBy4DTags(imageIds: string[]): string[][] { + const fncList = [splitFramesByFrameReferenceTime]; + + for (let i = 0; i < fncList.length; i++) { + const framesGroups = fncList[i](imageIds); + + if (!framesGroups || framesGroups.length <= 1) { + // imageIds could not be split into groups + continue; + } + + const framesPerGroup = framesGroups[0].length; + const groupsHaveSameLength = framesGroups.every( + (g) => g.length === framesPerGroup + ); + + if (groupsHaveSameLength) { + return framesGroups; + } + } + + // return the same imagesIds for non-4D volumes + return [imageIds]; +} + +export default splitImageIdsBy4DTags; diff --git a/packages/streaming-image-volume-loader/src/index.ts b/packages/streaming-image-volume-loader/src/index.ts index fe27a65b9..ececf3777 100644 --- a/packages/streaming-image-volume-loader/src/index.ts +++ b/packages/streaming-image-volume-loader/src/index.ts @@ -1,4 +1,11 @@ import cornerstoneStreamingImageVolumeLoader from './cornerstoneStreamingImageVolumeLoader'; +import cornerstoneStreamingDynamicImageVolumeLoader from './cornerstoneStreamingDynamicImageVolumeLoader'; import StreamingImageVolume from './StreamingImageVolume'; +import StreamingDynamicImageVolume from './StreamingDynamicImageVolume'; -export { cornerstoneStreamingImageVolumeLoader, StreamingImageVolume }; +export { + cornerstoneStreamingImageVolumeLoader, + cornerstoneStreamingDynamicImageVolumeLoader, + StreamingImageVolume, + StreamingDynamicImageVolume, +}; diff --git a/packages/tools/examples/dynamicPetCt/index.ts b/packages/tools/examples/dynamicPetCt/index.ts new file mode 100644 index 000000000..ac9af4c00 --- /dev/null +++ b/packages/tools/examples/dynamicPetCt/index.ts @@ -0,0 +1,800 @@ +import cornerstoneWADOImageLoader from 'cornerstone-wado-image-loader'; +import { + RenderingEngine, + Types, + Enums, + setVolumesForViewports, + volumeLoader, + getRenderingEngine, + cache, +} from '@cornerstonejs/core'; +import { + initDemo, + createImageIdsAndCacheMetaData, + setTitleAndDescription, + setPetColorMapTransferFunctionForVolumeActor, + setPetTransferFunctionForVolumeActor, + setCtTransferFunctionForVolumeActor, + addDropdownToToolbar, + addSliderToToolbar, +} from '../../../../utils/demo/helpers'; +import * as cornerstoneTools from '@cornerstonejs/tools'; + +const { + ToolGroupManager, + Enums: csToolsEnums, + WindowLevelTool, + PanTool, + ZoomTool, + StackScrollMouseWheelTool, + synchronizers, + MIPJumpToClickTool, + VolumeRotateMouseWheelTool, + CrosshairsTool, +} = cornerstoneTools; + +const { MouseBindings } = csToolsEnums; +const { ViewportType, BlendModes } = Enums; + +const { createCameraPositionSynchronizer, createVOISynchronizer } = + synchronizers; + +let renderingEngine; +const renderingEngineId = 'myRenderingEngine'; +const volumeLoaderScheme = 'cornerstoneStreamingDynamicImageVolume'; // Loader id which defines which volume loader to use +const ctVolumeName = 'CT_VOLUME_ID'; // Id of the volume less loader prefix +const ctVolumeId = `${volumeLoaderScheme}:${ctVolumeName}`; // VolumeId with loader id + volume id +const ptVolumeName = 'PT_VOLUME_ID'; +const ptVolumeId = `${volumeLoaderScheme}:${ptVolumeName}`; +const ctToolGroupId = 'CT_TOOLGROUP_ID'; +const ptToolGroupId = 'PT_TOOLGROUP_ID'; +const fusionToolGroupId = 'FUSION_TOOLGROUP_ID'; +const mipToolGroupUID = 'MIP_TOOLGROUP_ID'; + +const viewportIds = { + CT: { AXIAL: 'CT_AXIAL', SAGITTAL: 'CT_SAGITTAL', CORONAL: 'CT_CORONAL' }, + PT: { AXIAL: 'PT_AXIAL', SAGITTAL: 'PT_SAGITTAL', CORONAL: 'PT_CORONAL' }, + FUSION: { + AXIAL: 'FUSION_AXIAL', + SAGITTAL: 'FUSION_SAGITTAL', + CORONAL: 'FUSION_CORONAL', + }, + PETMIP: { + CORONAL: 'PET_MIP_CORONAL', + }, +}; + +// ======== Set up page ======== // +const description = [ + 'PT-CT fusion layout with Crosshairs, and synchronized cameras, CT W/L and PET threshold', + '', + 'DataSets:', + ' - CT 972 x 972 images / 565 images total', + ' - PET 255 x 255 images / 40 time points / 235 images per time point / 9,400 images total', +].join('\n'); + +setTitleAndDescription('PET-CT', description); + +const optionsValues = [WindowLevelTool.toolName, CrosshairsTool.toolName]; + +// ============================= // +addDropdownToToolbar({ + options: { values: optionsValues, defaultValue: WindowLevelTool.toolName }, + onSelectedValueChange: (toolNameAsStringOrNumber) => { + const toolName = String(toolNameAsStringOrNumber); + + [ctToolGroupId, ptToolGroupId, fusionToolGroupId].forEach((toolGroupId) => { + const toolGroup = ToolGroupManager.getToolGroup(toolGroupId); + + // Set the other tools disabled so we don't get conflicts. + // Note we only strictly need to change the one which is currently active. + + if (toolName === WindowLevelTool.toolName) { + // Set crosshairs passive so they are still interactable + toolGroup.setToolPassive(CrosshairsTool.toolName); + toolGroup.setToolActive(WindowLevelTool.toolName, { + bindings: [{ mouseButton: MouseBindings.Primary }], + }); + } else { + toolGroup.setToolDisabled(WindowLevelTool.toolName); + toolGroup.setToolActive(CrosshairsTool.toolName, { + bindings: [{ mouseButton: MouseBindings.Primary }], + }); + } + }); + }, +}); + +function addTimePointSlider(volume) { + addSliderToToolbar({ + title: 'Time Point', + range: [0, volume.numTimePoints - 1], + defaultValue: 0, + onSelectedValueChange: (value) => { + volume.timePointIndex = Number(value); + }, + }); +} + +const resizeObserver = new ResizeObserver(() => { + renderingEngine = getRenderingEngine(renderingEngineId); + + if (renderingEngine) { + renderingEngine.resize(true, false); + } +}); + +const viewportGrid = document.createElement('div'); + +viewportGrid.style.display = 'grid'; +viewportGrid.style.gridTemplateRows = `[row1-start] 33% [row2-start] 33% [row3-start] 33% [end]`; +viewportGrid.style.gridTemplateColumns = `[col1-start] 20% [col2-start] 20% [col3-start] 20% [col4-start] 20% [col5-start] 20%[end]`; +viewportGrid.style.width = '95vw'; +viewportGrid.style.height = '80vh'; + +const content = document.getElementById('content'); + +content.appendChild(viewportGrid); + +const element1_1 = document.createElement('div'); +const element1_2 = document.createElement('div'); +const element1_3 = document.createElement('div'); +const element2_1 = document.createElement('div'); +const element2_2 = document.createElement('div'); +const element2_3 = document.createElement('div'); +const element3_1 = document.createElement('div'); +const element3_2 = document.createElement('div'); +const element3_3 = document.createElement('div'); +const element_mip = document.createElement('div'); + +// Place main 3x3 viewports +element1_1.style.gridColumnStart = '1'; +element1_1.style.gridRowStart = '1'; +element1_2.style.gridColumnStart = '2'; +element1_2.style.gridRowStart = '1'; +element1_3.style.gridColumnStart = '3'; +element1_3.style.gridRowStart = '1'; +element2_1.style.gridColumnStart = '1'; +element2_1.style.gridRowStart = '2'; +element2_2.style.gridColumnStart = '2'; +element2_2.style.gridRowStart = '2'; +element2_3.style.gridColumnStart = '3'; +element2_3.style.gridRowStart = '2'; +element3_1.style.gridColumnStart = '1'; +element3_1.style.gridRowStart = '3'; +element3_2.style.gridColumnStart = '2'; +element3_2.style.gridRowStart = '3'; +element3_3.style.gridColumnStart = '3'; +element3_3.style.gridRowStart = '3'; + +// Place MIP viewport +element_mip.style.gridColumnStart = '4'; +element_mip.style.gridRowStart = '1'; +element_mip.style.gridRowEnd = 'span 3'; + +viewportGrid.appendChild(element1_1); +viewportGrid.appendChild(element1_2); +viewportGrid.appendChild(element1_3); +viewportGrid.appendChild(element2_1); +viewportGrid.appendChild(element2_2); +viewportGrid.appendChild(element2_3); +viewportGrid.appendChild(element3_1); +viewportGrid.appendChild(element3_2); +viewportGrid.appendChild(element3_3); +viewportGrid.appendChild(element_mip); + +const elements = [ + element1_1, + element1_2, + element1_3, + element2_1, + element2_2, + element2_3, + element3_1, + element3_2, + element3_3, +]; + +elements.forEach((element) => { + element.style.width = '100%'; + element.style.height = '100%'; + + // Disable right click context menu so we can have right click tools + element.oncontextmenu = (e) => e.preventDefault(); + + resizeObserver.observe(element); +}); + +element_mip.style.width = '100%'; +element_mip.style.height = '100%'; +element_mip.oncontextmenu = (e) => e.preventDefault(); +resizeObserver.observe(element_mip); + +const instructions = document.createElement('p'); + +instructions.innerText = ` + Basic Controls: + - Left click: Use selected tool + - Middle click: Pan + - Right click: Zoom + - Mouse Wheel: Stack Scroll + + Window Level Tool: + - Drag to set the window level for the CT and threshold for the PET. + + Crosshairs: + - When the tool is active: Click/Drag anywhere in the viewport to move the center of the crosshairs. + - Drag a reference line to move it, scrolling the other views. + - Square (closest to center): Drag these to change the thickness of the MIP slab in that plane. + - Circle (further from center): Drag these to rotate the axes. + + PET MIP: + - Mouse Wheel: Rotate PET + - Left click: Jump all views to the point of highest SUV in the region clicked. + `; + +instructions.style.gridColumnStart = '5'; +instructions.style.gridRowStart = '1'; +instructions.style.gridRowEnd = 'span 3'; + +viewportGrid.append(instructions); + +// ============================= // + +const viewportColors = { + [viewportIds.CT.AXIAL]: 'rgb(200, 0, 0)', + [viewportIds.CT.SAGITTAL]: 'rgb(200, 200, 0)', + [viewportIds.CT.CORONAL]: 'rgb(0, 200, 0)', + [viewportIds.PT.AXIAL]: 'rgb(200, 0, 0)', + [viewportIds.PT.SAGITTAL]: 'rgb(200, 200, 0)', + [viewportIds.PT.CORONAL]: 'rgb(0, 200, 0)', + [viewportIds.FUSION.AXIAL]: 'rgb(200, 0, 0)', + [viewportIds.FUSION.SAGITTAL]: 'rgb(200, 200, 0)', + [viewportIds.FUSION.CORONAL]: 'rgb(0, 200, 0)', +}; + +const viewportReferenceLineControllable = [ + viewportIds.CT.AXIAL, + viewportIds.CT.SAGITTAL, + viewportIds.CT.CORONAL, + viewportIds.PT.AXIAL, + viewportIds.PT.SAGITTAL, + viewportIds.PT.CORONAL, + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, +]; + +const viewportReferenceLineDraggableRotatable = [ + viewportIds.CT.AXIAL, + viewportIds.CT.SAGITTAL, + viewportIds.CT.CORONAL, + viewportIds.PT.AXIAL, + viewportIds.PT.SAGITTAL, + viewportIds.PT.CORONAL, + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, +]; + +const viewportReferenceLineSlabThicknessControlsOn = [ + viewportIds.CT.AXIAL, + viewportIds.CT.SAGITTAL, + viewportIds.CT.CORONAL, + viewportIds.PT.AXIAL, + viewportIds.PT.SAGITTAL, + viewportIds.PT.CORONAL, + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, +]; + +function getReferenceLineColor(viewportId) { + return viewportColors[viewportId]; +} + +function getReferenceLineControllable(viewportId) { + const index = viewportReferenceLineControllable.indexOf(viewportId); + return index !== -1; +} + +function getReferenceLineDraggableRotatable(viewportId) { + const index = viewportReferenceLineDraggableRotatable.indexOf(viewportId); + return index !== -1; +} + +function getReferenceLineSlabThicknessControlsOn(viewportId) { + const index = + viewportReferenceLineSlabThicknessControlsOn.indexOf(viewportId); + return index !== -1; +} + +function setUpToolGroups() { + // Add tools to Cornerstone3D + cornerstoneTools.addTool(WindowLevelTool); + cornerstoneTools.addTool(PanTool); + cornerstoneTools.addTool(ZoomTool); + cornerstoneTools.addTool(StackScrollMouseWheelTool); + cornerstoneTools.addTool(MIPJumpToClickTool); + cornerstoneTools.addTool(VolumeRotateMouseWheelTool); + cornerstoneTools.addTool(CrosshairsTool); + + // Define tool groups for the main 9 viewports. + // Crosshairs currently only supports 3 viewports for a toolgroup due to the + // way it is constructed, but its configuration input allows us to synchronize + // multiple sets of 3 viewports. + const ctToolGroup = ToolGroupManager.createToolGroup(ctToolGroupId); + const ptToolGroup = ToolGroupManager.createToolGroup(ptToolGroupId); + const fusionToolGroup = ToolGroupManager.createToolGroup(fusionToolGroupId); + + ctToolGroup.addViewport(viewportIds.CT.AXIAL, renderingEngineId); + ctToolGroup.addViewport(viewportIds.CT.SAGITTAL, renderingEngineId); + ctToolGroup.addViewport(viewportIds.CT.CORONAL, renderingEngineId); + ptToolGroup.addViewport(viewportIds.PT.AXIAL, renderingEngineId); + ptToolGroup.addViewport(viewportIds.PT.SAGITTAL, renderingEngineId); + ptToolGroup.addViewport(viewportIds.PT.CORONAL, renderingEngineId); + fusionToolGroup.addViewport(viewportIds.FUSION.AXIAL, renderingEngineId); + fusionToolGroup.addViewport(viewportIds.FUSION.SAGITTAL, renderingEngineId); + fusionToolGroup.addViewport(viewportIds.FUSION.CORONAL, renderingEngineId); + + // Manipulation Tools + [ctToolGroup, ptToolGroup].forEach((toolGroup) => { + toolGroup.addTool(PanTool.toolName); + toolGroup.addTool(ZoomTool.toolName); + toolGroup.addTool(StackScrollMouseWheelTool.toolName); + toolGroup.addTool(CrosshairsTool.toolName, { + getReferenceLineColor, + getReferenceLineControllable, + getReferenceLineDraggableRotatable, + getReferenceLineSlabThicknessControlsOn, + }); + }); + + fusionToolGroup.addTool(PanTool.toolName); + fusionToolGroup.addTool(ZoomTool.toolName); + fusionToolGroup.addTool(StackScrollMouseWheelTool.toolName); + fusionToolGroup.addTool(CrosshairsTool.toolName, { + getReferenceLineColor, + getReferenceLineControllable, + getReferenceLineDraggableRotatable, + getReferenceLineSlabThicknessControlsOn, + // Only set CT volume to MIP in the fusion viewport + filterActorUIDsToSetSlabThickness: [ctVolumeId], + }); + + // Here is the difference in the toolGroups used, that we need to specify the + // volume to use for the WindowLevelTool for the fusion viewports + ctToolGroup.addTool(WindowLevelTool.toolName); + ptToolGroup.addTool(WindowLevelTool.toolName); + fusionToolGroup.addTool(WindowLevelTool.toolName, { + volumeId: ptVolumeId, + }); + + [ctToolGroup, ptToolGroup, fusionToolGroup].forEach((toolGroup) => { + toolGroup.setToolActive(WindowLevelTool.toolName, { + bindings: [ + { + mouseButton: MouseBindings.Primary, // Left Click + }, + ], + }); + toolGroup.setToolActive(PanTool.toolName, { + bindings: [ + { + mouseButton: MouseBindings.Auxiliary, // Middle Click + }, + ], + }); + toolGroup.setToolActive(ZoomTool.toolName, { + bindings: [ + { + mouseButton: MouseBindings.Secondary, // Right Click + }, + ], + }); + + toolGroup.setToolActive(StackScrollMouseWheelTool.toolName); + toolGroup.setToolPassive(CrosshairsTool.toolName); + }); + + // MIP Tool Groups + const mipToolGroup = ToolGroupManager.createToolGroup(mipToolGroupUID); + + mipToolGroup.addTool('VolumeRotateMouseWheel'); + mipToolGroup.addTool('MIPJumpToClickTool', { + targetViewportIds: [ + viewportIds.CT.AXIAL, + viewportIds.CT.SAGITTAL, + viewportIds.CT.CORONAL, + viewportIds.PT.AXIAL, + viewportIds.PT.SAGITTAL, + viewportIds.PT.CORONAL, + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, + ], + }); + + // Set the initial state of the tools, here we set one tool active on left click. + // This means left click will draw that tool. + mipToolGroup.setToolActive('MIPJumpToClickTool', { + bindings: [ + { + mouseButton: MouseBindings.Primary, // Left Click + }, + ], + }); + // As the Stack Scroll mouse wheel is a tool using the `mouseWheelCallback` + // hook instead of mouse buttons, it does not need to assign any mouse button. + mipToolGroup.setToolActive('VolumeRotateMouseWheel'); + + mipToolGroup.addViewport(viewportIds.PETMIP.CORONAL, renderingEngineId); +} + +function setUpSynchronizers() { + const axialCameraSynchronizerId = 'AXIAL_CAMERA_SYNCHRONIZER_ID'; + const sagittalCameraSynchronizerId = 'SAGITTAL_CAMERA_SYNCHRONIZER_ID'; + const coronalCameraSynchronizerId = 'CORONAL_CAMERA_SYNCHRONIZER_ID'; + const ctVoiSynchronizerId = 'CT_VOI_SYNCHRONIZER_ID'; + const ptVoiSynchronizerId = 'PT_VOI_SYNCHRONIZER_ID'; + + const axialCameraPositionSynchronizer = createCameraPositionSynchronizer( + axialCameraSynchronizerId + ); + const sagittalCameraPositionSynchronizer = createCameraPositionSynchronizer( + sagittalCameraSynchronizerId + ); + const coronalCameraPositionSynchronizer = createCameraPositionSynchronizer( + coronalCameraSynchronizerId + ); + const ctVoiSynchronizer = createVOISynchronizer(ctVoiSynchronizerId); + const ptVoiSynchronizer = createVOISynchronizer(ptVoiSynchronizerId); + + // Add viewports to camera synchronizers + [ + viewportIds.CT.AXIAL, + viewportIds.PT.AXIAL, + viewportIds.FUSION.AXIAL, + ].forEach((viewportId) => { + axialCameraPositionSynchronizer.add({ + renderingEngineId, + viewportId, + }); + }); + [ + viewportIds.CT.SAGITTAL, + viewportIds.PT.SAGITTAL, + viewportIds.FUSION.SAGITTAL, + ].forEach((viewportId) => { + sagittalCameraPositionSynchronizer.add({ + renderingEngineId, + viewportId, + }); + }); + [ + viewportIds.CT.CORONAL, + viewportIds.PT.CORONAL, + viewportIds.FUSION.CORONAL, + ].forEach((viewportId) => { + coronalCameraPositionSynchronizer.add({ + renderingEngineId, + viewportId, + }); + }); + + // Add viewports to VOI synchronizers + [ + viewportIds.CT.AXIAL, + viewportIds.CT.SAGITTAL, + viewportIds.CT.CORONAL, + ].forEach((viewportId) => { + ctVoiSynchronizer.add({ + renderingEngineId, + viewportId, + }); + }); + [ + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, + ].forEach((viewportId) => { + // In this example, the fusion viewports are only targets for CT VOI + // synchronization, not sources + ctVoiSynchronizer.addTarget({ + renderingEngineId, + viewportId, + }); + }); + [ + viewportIds.PT.AXIAL, + viewportIds.PT.SAGITTAL, + viewportIds.PT.CORONAL, + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, + viewportIds.PETMIP.CORONAL, + ].forEach((viewportId) => { + ptVoiSynchronizer.add({ + renderingEngineId, + viewportId, + }); + }); +} + +async function setUpDisplay() { + const { metaDataManager } = cornerstoneWADOImageLoader.wadors; + const wadoRsRoot = 'https://d28o5kq0jsoob5.cloudfront.net/dicomweb'; + const StudyInstanceUID = + '1.3.6.1.4.1.12842.1.1.14.3.20220915.105557.468.2963630849'; + + // Get Cornerstone imageIds and fetch metadata into RAM + let ctImageIds = await createImageIdsAndCacheMetaData({ + StudyInstanceUID, + SeriesInstanceUID: + '1.3.6.1.4.1.12842.1.1.14.4.20220915.121025.435.2500855592', + wadoRsRoot, + }); + + const ptImageIds = await createImageIdsAndCacheMetaData({ + StudyInstanceUID, + SeriesInstanceUID: + '1.3.6.1.4.1.12842.1.1.22.4.20220915.124758.560.4125514885', + wadoRsRoot, + }); + + // Limit to 565 images because it is currently crashing when all 763 images + // are loaded. There is a work in progress to fix that. + ctImageIds = ctImageIds.filter((imageId) => { + const instanceMetaData = metaDataManager.get(imageId); + const instanceTag = instanceMetaData['00200013']; + const instanceNumber = parseInt(instanceTag.Value[0]); + + return instanceNumber <= 565; + }); + + // Define a volume in memory + const ctVolume = await volumeLoader.createAndCacheVolume(ctVolumeId, { + imageIds: ctImageIds, + }); + // Define a volume in memory + const ptVolume = await volumeLoader.createAndCacheVolume(ptVolumeId, { + imageIds: ptImageIds, + }); + + // Create the viewports + + const viewportInputArray = [ + { + viewportId: viewportIds.CT.AXIAL, + type: ViewportType.ORTHOGRAPHIC, + element: element1_1, + defaultOptions: { + orientation: Enums.OrientationAxis.AXIAL, + }, + }, + { + viewportId: viewportIds.CT.SAGITTAL, + type: ViewportType.ORTHOGRAPHIC, + element: element1_2, + defaultOptions: { + orientation: Enums.OrientationAxis.SAGITTAL, + }, + }, + { + viewportId: viewportIds.CT.CORONAL, + type: ViewportType.ORTHOGRAPHIC, + element: element1_3, + defaultOptions: { + orientation: Enums.OrientationAxis.CORONAL, + }, + }, + { + viewportId: viewportIds.PT.AXIAL, + type: ViewportType.ORTHOGRAPHIC, + element: element2_1, + defaultOptions: { + orientation: Enums.OrientationAxis.AXIAL, + background: [1, 1, 1], + }, + }, + { + viewportId: viewportIds.PT.SAGITTAL, + type: ViewportType.ORTHOGRAPHIC, + element: element2_2, + defaultOptions: { + orientation: Enums.OrientationAxis.SAGITTAL, + background: [1, 1, 1], + }, + }, + { + viewportId: viewportIds.PT.CORONAL, + type: ViewportType.ORTHOGRAPHIC, + element: element2_3, + defaultOptions: { + orientation: Enums.OrientationAxis.CORONAL, + background: [1, 1, 1], + }, + }, + { + viewportId: viewportIds.FUSION.AXIAL, + type: ViewportType.ORTHOGRAPHIC, + element: element3_1, + defaultOptions: { + orientation: Enums.OrientationAxis.AXIAL, + }, + }, + { + viewportId: viewportIds.FUSION.SAGITTAL, + type: ViewportType.ORTHOGRAPHIC, + element: element3_2, + defaultOptions: { + orientation: Enums.OrientationAxis.SAGITTAL, + }, + }, + { + viewportId: viewportIds.FUSION.CORONAL, + type: ViewportType.ORTHOGRAPHIC, + element: element3_3, + defaultOptions: { + orientation: Enums.OrientationAxis.CORONAL, + }, + }, + { + viewportId: viewportIds.PETMIP.CORONAL, + type: ViewportType.ORTHOGRAPHIC, + element: element_mip, + defaultOptions: { + orientation: Enums.OrientationAxis.CORONAL, + background: [1, 1, 1], + }, + }, + ]; + + renderingEngine.setViewports(viewportInputArray); + + // Set the volumes to load + ctVolume.load(); + ptVolume.load(); + + addTimePointSlider(ptVolume); + + // Set volumes on the viewports + await setVolumesForViewports( + renderingEngine, + [ + { + volumeId: ctVolumeId, + callback: setCtTransferFunctionForVolumeActor, + }, + ], + [viewportIds.CT.AXIAL, viewportIds.CT.SAGITTAL, viewportIds.CT.CORONAL] + ); + + await setVolumesForViewports( + renderingEngine, + [ + { + volumeId: ptVolumeId, + callback: setPetTransferFunctionForVolumeActor, + }, + ], + [viewportIds.PT.AXIAL, viewportIds.PT.SAGITTAL, viewportIds.PT.CORONAL] + ); + + await setVolumesForViewports( + renderingEngine, + [ + { + volumeId: ctVolumeId, + callback: setCtTransferFunctionForVolumeActor, + }, + { + volumeId: ptVolumeId, + callback: setPetColorMapTransferFunctionForVolumeActor, + }, + ], + [ + viewportIds.FUSION.AXIAL, + viewportIds.FUSION.SAGITTAL, + viewportIds.FUSION.CORONAL, + ] + ); + + // Calculate size of fullBody pet mip + const ptVolumeDimensions = ptVolume.dimensions; + + // Only make the MIP as large as it needs to be. + const slabThickness = Math.sqrt( + ptVolumeDimensions[0] * ptVolumeDimensions[0] + + ptVolumeDimensions[1] * ptVolumeDimensions[1] + + ptVolumeDimensions[2] * ptVolumeDimensions[2] + ); + + setVolumesForViewports( + renderingEngine, + [ + { + volumeId: ptVolumeId, + callback: setPetTransferFunctionForVolumeActor, + blendMode: BlendModes.MAXIMUM_INTENSITY_BLEND, + slabThickness, + }, + ], + [viewportIds.PETMIP.CORONAL] + ); + + initializeCameraSync(renderingEngine); + + // Render the viewports + renderingEngine.render(); +} + +function initializeCameraSync(renderingEngine) { + // The fusion scene is the target as it is scaled to both volumes. + // TODO -> We should have a more generic way to do this, + // So that when all data is added we can synchronize zoom/position before interaction. + + const axialCtViewport = renderingEngine.getViewport(viewportIds.CT.AXIAL); + const sagittalCtViewport = renderingEngine.getViewport( + viewportIds.CT.SAGITTAL + ); + const coronalCtViewport = renderingEngine.getViewport(viewportIds.CT.CORONAL); + + const axialPtViewport = renderingEngine.getViewport(viewportIds.PT.AXIAL); + const sagittalPtViewport = renderingEngine.getViewport( + viewportIds.PT.SAGITTAL + ); + const coronalPtViewport = renderingEngine.getViewport(viewportIds.PT.CORONAL); + + const axialFusionViewport = renderingEngine.getViewport( + viewportIds.FUSION.AXIAL + ); + const sagittalFusionViewport = renderingEngine.getViewport( + viewportIds.FUSION.SAGITTAL + ); + const coronalFusionViewport = renderingEngine.getViewport( + viewportIds.FUSION.CORONAL + ); + + initCameraSynchronization(axialFusionViewport, axialCtViewport); + initCameraSynchronization(axialFusionViewport, axialPtViewport); + + initCameraSynchronization(sagittalFusionViewport, sagittalCtViewport); + initCameraSynchronization(sagittalFusionViewport, sagittalPtViewport); + + initCameraSynchronization(coronalFusionViewport, coronalCtViewport); + initCameraSynchronization(coronalFusionViewport, coronalPtViewport); + + renderingEngine.render(); +} + +function initCameraSynchronization(sViewport, tViewport) { + // Initialise the sync as they viewports will have + // Different initial zoom levels for viewports of different sizes. + + const camera = sViewport.getCamera(); + + tViewport.setCamera(camera); +} + +/** + * Runs the demo + */ +async function run() { + // Init Cornerstone and related libraries + await initDemo(); + + // Increases cache size to 4GB to be able to store all PET/CT images + cache.setMaxCacheSize(4 * 1024 * 1024 * 1024); + + // Instantiate a rendering engine + renderingEngine = new RenderingEngine(renderingEngineId); + + // Display needs to be set up first so that we have viewport to reference for tools and synchronizers. + await setUpDisplay(); + // Tools and synchronizers can be set up in any order. + setUpToolGroups(); + setUpSynchronizers(); +} + +run(); diff --git a/packages/tools/src/tools/WindowLevelTool.ts b/packages/tools/src/tools/WindowLevelTool.ts index b710651ff..dd9f7df0a 100644 --- a/packages/tools/src/tools/WindowLevelTool.ts +++ b/packages/tools/src/tools/WindowLevelTool.ts @@ -150,7 +150,8 @@ class WindowLevelTool extends BaseTool { if (volumeId) { const imageVolume = cache.getVolume(volumeId); - const { dimensions, scalarData } = imageVolume; + const { dimensions } = imageVolume; + const scalarData = imageVolume.getScalarData(); imageDynamicRange = this._getImageDynamicRangeFromMiddleSlice( scalarData, dimensions diff --git a/packages/tools/src/tools/annotation/ProbeTool.ts b/packages/tools/src/tools/annotation/ProbeTool.ts index 10d1d5e98..f56e10d4b 100644 --- a/packages/tools/src/tools/annotation/ProbeTool.ts +++ b/packages/tools/src/tools/annotation/ProbeTool.ts @@ -608,7 +608,9 @@ class ProbeTool extends AnnotationTool { continue; } - const { dimensions, scalarData, imageData, metadata } = image; + const { dimensions, imageData, metadata } = image; + const scalarData = + 'getScalarData' in image ? image.getScalarData() : image.scalarData; const modality = metadata.Modality; const index = transformWorldToIndex(imageData, worldPos); diff --git a/packages/tools/src/tools/annotation/RectangleROITool.ts b/packages/tools/src/tools/annotation/RectangleROITool.ts index 96e83d209..a3e68719d 100644 --- a/packages/tools/src/tools/annotation/RectangleROITool.ts +++ b/packages/tools/src/tools/annotation/RectangleROITool.ts @@ -898,8 +898,9 @@ class RectangleROITool extends AnnotationTool { continue; } - const { dimensions, scalarData, imageData, metadata, hasPixelSpacing } = - image; + const { dimensions, imageData, metadata, hasPixelSpacing } = image; + const scalarData = + 'getScalarData' in image ? image.getScalarData() : image.scalarData; const worldPos1Index = transformWorldToIndex(imageData, worldPos1); diff --git a/packages/tools/src/tools/segmentation/PaintFillTool.ts b/packages/tools/src/tools/segmentation/PaintFillTool.ts index d91cb7be0..282c8e438 100644 --- a/packages/tools/src/tools/segmentation/PaintFillTool.ts +++ b/packages/tools/src/tools/segmentation/PaintFillTool.ts @@ -85,7 +85,8 @@ class PaintFillTool extends BaseTool { const { volumeId } = representationData[type] as LabelmapSegmentationData; const segmentation = cache.getVolume(volumeId); - const { scalarData, dimensions, direction } = segmentation; + const { dimensions, direction } = segmentation; + const scalarData = segmentation.getScalarData(); const index = transformWorldToIndex(segmentation.imageData, worldPos); diff --git a/packages/tools/src/tools/segmentation/strategies/eraseRectangle.ts b/packages/tools/src/tools/segmentation/strategies/eraseRectangle.ts index c5fdd5d3c..a3df8236e 100644 --- a/packages/tools/src/tools/segmentation/strategies/eraseRectangle.ts +++ b/packages/tools/src/tools/segmentation/strategies/eraseRectangle.ts @@ -26,7 +26,8 @@ function eraseRectangle( segmentsLocked, segmentationId, } = operationData; - const { imageData, dimensions, scalarData } = segmentation; + const { imageData, dimensions } = segmentation; + const scalarData = segmentation.getScalarData(); const rectangleCornersIJK = points.map((world) => { return transformWorldToIndex(imageData, world); diff --git a/packages/tools/src/tools/segmentation/strategies/fillCircle.ts b/packages/tools/src/tools/segmentation/strategies/fillCircle.ts index dee89df66..691023fd2 100644 --- a/packages/tools/src/tools/segmentation/strategies/fillCircle.ts +++ b/packages/tools/src/tools/segmentation/strategies/fillCircle.ts @@ -39,7 +39,8 @@ function fillCircle( segmentationId, strategySpecificConfiguration, } = operationData; - const { imageData, dimensions, scalarData } = segmentationVolume; + const { imageData, dimensions } = segmentationVolume; + const scalarData = segmentationVolume.getScalarData(); const { viewport } = enabledElement; // Average the points to get the center of the ellipse @@ -127,7 +128,7 @@ function isWithinThreshold( ) { const { THRESHOLD_INSIDE_CIRCLE } = strategySpecificConfiguration; - const voxelValue = imageVolume.scalarData[index]; + const voxelValue = imageVolume.getScalarData()[index]; const { threshold } = THRESHOLD_INSIDE_CIRCLE; return threshold[0] <= voxelValue && voxelValue <= threshold[1]; diff --git a/packages/tools/src/tools/segmentation/strategies/fillRectangle.ts b/packages/tools/src/tools/segmentation/strategies/fillRectangle.ts index c2a776fa8..27762595e 100644 --- a/packages/tools/src/tools/segmentation/strategies/fillRectangle.ts +++ b/packages/tools/src/tools/segmentation/strategies/fillRectangle.ts @@ -38,7 +38,8 @@ function fillRectangle( segmentationId, constraintFn, } = operationData; - const { imageData, dimensions, scalarData } = segmentation; + const { imageData, dimensions } = segmentation; + const scalarData = segmentation.getScalarData(); let rectangleCornersIJK = points.map((world) => { return transformWorldToIndex(imageData, world); diff --git a/packages/tools/src/tools/segmentation/strategies/fillSphere.ts b/packages/tools/src/tools/segmentation/strategies/fillSphere.ts index 92fefa9ec..d066f18b4 100644 --- a/packages/tools/src/tools/segmentation/strategies/fillSphere.ts +++ b/packages/tools/src/tools/segmentation/strategies/fillSphere.ts @@ -28,7 +28,8 @@ function fillSphere( points, } = operationData; - const { scalarData, imageData, dimensions } = segmentation; + const { imageData, dimensions } = segmentation; + const scalarData = segmentation.getScalarData(); const scalarIndex = []; const callback = ({ index, value }) => { diff --git a/packages/tools/src/utilities/segmentation/createMergedLabelmapForIndex.ts b/packages/tools/src/utilities/segmentation/createMergedLabelmapForIndex.ts index d0308bafa..962f3e94d 100644 --- a/packages/tools/src/utilities/segmentation/createMergedLabelmapForIndex.ts +++ b/packages/tools/src/utilities/segmentation/createMergedLabelmapForIndex.ts @@ -30,11 +30,11 @@ function createMergedLabelmapForIndex( const labelmap = labelmaps[0]; - const arrayType = labelmap.scalarData.constructor; - const outputData = new arrayType(labelmap.scalarData.length); + const arrayType = (labelmap.getScalarData() as any).constructor; + const outputData = new arrayType(labelmap.getScalarData().length); labelmaps.forEach((labelmap) => { - const { scalarData } = labelmap; + const scalarData = labelmap.getScalarData(); for (let i = 0; i < scalarData.length; i++) { if (scalarData[i] === segmentIndex) { outputData[i] = segmentIndex; diff --git a/packages/tools/src/utilities/segmentation/rectangleROIThresholdVolumeByRange.ts b/packages/tools/src/utilities/segmentation/rectangleROIThresholdVolumeByRange.ts index 8a00f5383..fef01a3df 100644 --- a/packages/tools/src/utilities/segmentation/rectangleROIThresholdVolumeByRange.ts +++ b/packages/tools/src/utilities/segmentation/rectangleROIThresholdVolumeByRange.ts @@ -57,8 +57,9 @@ function rectangleROIThresholdVolumeByRange( let boundsIJK; for (let i = 0; i < thresholdVolumeInformation.length; i++) { // make sure that the boundsIJK are generated by the correct volume - const volumeSize = thresholdVolumeInformation[i].volume.scalarData.length; - if (volumeSize === segmentationVolume.scalarData.length || i === 0) { + const volumeSize = + thresholdVolumeInformation[i].volume.getScalarData().length; + if (volumeSize === segmentationVolume.getScalarData().length || i === 0) { boundsIJK = getBoundsIJKFromRectangleAnnotations( annotations, thresholdVolumeInformation[i].volume, diff --git a/packages/tools/src/utilities/segmentation/thresholdVolumeByRange.ts b/packages/tools/src/utilities/segmentation/thresholdVolumeByRange.ts index 7fedacff6..6bb99115d 100644 --- a/packages/tools/src/utilities/segmentation/thresholdVolumeByRange.ts +++ b/packages/tools/src/utilities/segmentation/thresholdVolumeByRange.ts @@ -47,11 +47,9 @@ function thresholdVolumeByRange( thresholdVolumeInformation: ThresholdInformation[], options: ThresholdRangeOptions ): Types.IImageVolume { - const { - scalarData, - spacing: segmentationSpacing, - imageData: segmentationImageData, - } = segmentationVolume; + const { spacing: segmentationSpacing, imageData: segmentationImageData } = + segmentationVolume; + const scalarData = segmentationVolume.getScalarData(); const { overwrite, boundsIJK } = options; const overlapType = options?.overlapType || 0; @@ -70,7 +68,8 @@ function thresholdVolumeByRange( const { imageData, spacing, dimensions } = thresholdVolumeInformation[i].volume; - const volumeSize = thresholdVolumeInformation[i].volume.scalarData.length; + const volumeSize = + thresholdVolumeInformation[i].volume.getScalarData().length; // discover the index of the volume the segmentation data is based on if ( volumeSize === scalarData.length && diff --git a/utils/ExampleRunner/example-info.json b/utils/ExampleRunner/example-info.json index a0c6efdac..bd213207f 100644 --- a/utils/ExampleRunner/example-info.json +++ b/utils/ExampleRunner/example-info.json @@ -81,6 +81,10 @@ "webLoader": { "name": "Load Web images of PNG or JPG format", "description": "Demonstrates how render web images in a stack viewport" + }, + "dynamicVolume": { + "name": "Load a dynamic 4D data", + "description": "Demonstrates how you can render 4D data with cornerstone 3d" } }, "core-advanced": { @@ -197,6 +201,10 @@ "doubleClickWithStackAnnotationTools": { "name": "Double Click With Stack Annotation Tools", "description": "Demonstrates double click detection before/during/after using various annotation tools on a stack viewport." + }, + "dynamicPetCt": { + "name": "Load a petCT data where PT series is 4D", + "description": "Demonstrates how to render a 4D data into multiple viewports and fuse them" } }, "tools-advanced": { diff --git a/utils/demo/helpers/createImageIdsAndCacheMetaData.js b/utils/demo/helpers/createImageIdsAndCacheMetaData.js index 3f0edfb4b..c85dfeef6 100644 --- a/utils/demo/helpers/createImageIdsAndCacheMetaData.js +++ b/utils/demo/helpers/createImageIdsAndCacheMetaData.js @@ -8,6 +8,7 @@ import cornerstoneWADOImageLoader from 'cornerstone-wado-image-loader'; import ptScalingMetaDataProvider from './ptScalingMetaDataProvider'; import getPixelSpacingInformation from './getPixelSpacingInformation'; import { convertMultiframeImageIds } from './convertMultiframeImageIds'; +import removeInvalidTags from './removeInvalidTags'; const { DicomMetaDictionary } = dcmjs.data; const { calibratedPixelSpacingMetadataProvider } = utilities; @@ -64,13 +65,18 @@ export default async function createImageIdsAndCacheMetaData({ ); return imageId; }); + // if the image ids represent multiframe information, creates a new list with one image id per frame // if not multiframe data available, just returns the same list given imageIds = convertMultiframeImageIds(imageIds); + imageIds.forEach((imageId) => { let instanceMetaData = cornerstoneWADOImageLoader.wadors.metaDataManager.get(imageId); - instanceMetaData = JSON.parse(JSON.stringify(instanceMetaData)); + + // It was using JSON.parse(JSON.stringify(...)) before but it is 8x slower + instanceMetaData = removeInvalidTags(instanceMetaData); + if (instanceMetaData) { // Add calibrated pixel spacing const metadata = DicomMetaDictionary.naturalizeDataset(instanceMetaData); diff --git a/utils/demo/helpers/initVolumeLoader.js b/utils/demo/helpers/initVolumeLoader.js index 3133f2e85..22a81d6e4 100644 --- a/utils/demo/helpers/initVolumeLoader.js +++ b/utils/demo/helpers/initVolumeLoader.js @@ -1,5 +1,8 @@ -import { imageLoader, volumeLoader } from '@cornerstonejs/core'; -import { cornerstoneStreamingImageVolumeLoader } from '@cornerstonejs/streaming-image-volume-loader'; +import { volumeLoader } from '@cornerstonejs/core'; +import { + cornerstoneStreamingImageVolumeLoader, + cornerstoneStreamingDynamicImageVolumeLoader, +} from '@cornerstonejs/streaming-image-volume-loader'; export default function initVolumeLoader() { volumeLoader.registerUnknownVolumeLoader( @@ -9,4 +12,8 @@ export default function initVolumeLoader() { 'cornerstoneStreamingImageVolume', cornerstoneStreamingImageVolumeLoader ); + volumeLoader.registerVolumeLoader( + 'cornerstoneStreamingDynamicImageVolume', + cornerstoneStreamingDynamicImageVolumeLoader + ); } diff --git a/utils/demo/helpers/removeInvalidTags.js b/utils/demo/helpers/removeInvalidTags.js new file mode 100644 index 000000000..abf765262 --- /dev/null +++ b/utils/demo/helpers/removeInvalidTags.js @@ -0,0 +1,33 @@ +/** + * Remove invalid tags from a metadata and return a new object. + * + * At this time it is only removing tags that has `null` or `undefined` values + * which is our main goal because that breaks when `naturalizeDataset(...)` is + * called. + * + * Validating the tag id using regex like /^[a-fA-F0-9]{8}$/ make it run + * +50% slower and looping through all characteres (split+every+Set or simple + * FOR+Set) double the time it takes to run. It is currently taking +12ms/1k + * images on average which can change depending on the machine. + * + * @param srcMetadata - source metadata + * @returns new metadata object without invalid tags + */ +function removeInvalidTags(srcMetadata) { + // Object.create(null) make it ~9% faster + const dstMetadata = Object.create(null); + const tagIds = Object.keys(srcMetadata); + let tagValue; + + tagIds.forEach((tagId) => { + tagValue = srcMetadata[tagId]; + + if (tagValue !== undefined && tagValue !== null) { + dstMetadata[tagId] = tagValue; + } + }); + + return dstMetadata; +} + +export { removeInvalidTags as default, removeInvalidTags }; diff --git a/yarn.lock b/yarn.lock index 883d713c2..7d4cbd68b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2524,21 +2524,41 @@ resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-charls/-/codec-charls-0.1.1.tgz#e55d4aa908732d0cc902888b7f3856c5a996df7f" integrity sha512-Y250DGVzmownJ7WgpHxNqWvfTnv4/malaKm/tWm0xE1FxhQE8iErMWFpKxpNDk3MdfXO4/98piVsUwmJMiWoDQ== +"@cornerstonejs/codec-charls@^1.2.3": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-charls/-/codec-charls-1.2.3.tgz#6952c420486822ac8404409ae0ed5a559aff6e25" + integrity sha512-qKUe6DN0dnGzhhfZLYhH9UZacMcudjxcaLXCrpxJImT/M/PQvZCT2rllu6VGJbWKJWG+dMVV2zmmleZcdJ7/cA== + "@cornerstonejs/codec-libjpeg-turbo-8bit@^0.0.7": version "0.0.7" resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-libjpeg-turbo-8bit/-/codec-libjpeg-turbo-8bit-0.0.7.tgz#2ea9b575eed19e6e7e3701b7a50a4ae0ffbef0c4" integrity sha512-qgm6BuVAy5mNP8SJ+A6+VbmPnqgj8jPvJrw4HbUoAzndmf9/VHjTYwawn3kmZWya5ErFAsXQ6c0U0noB1LKAiA== +"@cornerstonejs/codec-libjpeg-turbo-8bit@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-libjpeg-turbo-8bit/-/codec-libjpeg-turbo-8bit-1.2.2.tgz#ae384b149d6655e3dd6e18b9891fab479ab5e144" + integrity sha512-aAUMK2958YNpOb/7G6e2/aG7hExTiFTASlMt/v90XA0pRHdWiNg5ny4S5SAju0FbIw4zcMnR0qfY+yW3VG2ivg== + "@cornerstonejs/codec-openjpeg@^0.1.0": version "0.1.1" resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-openjpeg/-/codec-openjpeg-0.1.1.tgz#5bd1c52a33a425299299e970312731fa0cc2711b" integrity sha512-HOMMOLV6xy8O/agNGGvrl0a8DwShpBvWxAzEzv2pqq12d3r5z/3MyIgNA3Oj/8bIBVvvVXxh9RX7rMDRHJdowg== +"@cornerstonejs/codec-openjpeg@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-openjpeg/-/codec-openjpeg-1.2.2.tgz#f0b524235b5551426b46db197a37b06f8ac805d7" + integrity sha512-b1O7lZacKXelgeV9n8XWZ7pTw3i4Bq4qQ26G5ahBjWoOw4QNcCrb5hPxWBxNB/I8AoNbJxAe+lyLtyQGfdrTbw== + "@cornerstonejs/codec-openjph@^1.0.3": version "1.0.3" resolved "https://registry.npmjs.org/@cornerstonejs/codec-openjph/-/codec-openjph-1.0.3.tgz#4c82642e8a6beb0d263f5a263723aec4225e5300" integrity sha512-PK+9N/JL7ZMGum6OKuReRGGhWXpWjRC9WnltQ6aEzRKEQPMg+3WFiQPZowKCCG1I4whv35DYFRi6wu7RyRaEMQ== +"@cornerstonejs/codec-openjph@^2.4.2": + version "2.4.2" + resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-openjph/-/codec-openjph-2.4.2.tgz#e96721d56f6ec96f7f95c16321d88cc8467d8d81" + integrity sha512-lgdvBvvNezleY+4pIe2ceUsJzlZe/0PipdeubQ3vZZOz3xxtHHMR1XFCl4fgd8gosR8COHuD7h6q+MwgrwBsng== + "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" @@ -8851,6 +8871,22 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cornerstone-wado-image-loader@^4.10.0: + version "4.10.0" + resolved "https://registry.yarnpkg.com/cornerstone-wado-image-loader/-/cornerstone-wado-image-loader-4.10.0.tgz#25c367cfc54a2c92ebbb5c64dba4fe38439112a1" + integrity sha512-XZcgB8DpUxnsTA3vU/zbPtB2uFLL4ght70BPrQHykkX/Jlg/r6Ob7ztX2dEEAAb4ELE/d4icfGuSy10Acg/kyw== + dependencies: + "@babel/eslint-parser" "^7.19.1" + "@cornerstonejs/codec-charls" "^1.2.3" + "@cornerstonejs/codec-libjpeg-turbo-8bit" "^1.2.2" + "@cornerstonejs/codec-openjpeg" "^1.2.2" + "@cornerstonejs/codec-openjph" "^2.4.2" + coverage-istanbul-loader "^3.0.5" + date-format "^4.0.14" + dicom-parser "^1.8.9" + pako "^2.0.4" + uuid "^9.0.0" + cornerstone-wado-image-loader@^4.8.0: version "4.8.0" resolved "https://registry.yarnpkg.com/cornerstone-wado-image-loader/-/cornerstone-wado-image-loader-4.8.0.tgz#64237a807227815bd117af839db4c05ab028d5f0"