feat(RT): add dicom RT support via volume viewports (#3310)

* feat: initial RT support

* make the segmentation service work with representation data

* feat: make segmentation service work with representations

* fix rtss vis

* fix: rt hydration

* fix the rendering of rt names

* fix imports

* refactor: Modify status and click handling for hydration of RTStructures

Modify status and click handling for hydration of RTStructures by renaming `onPillClick` to `onStatusClick` in `OHIFCornerstoneRTViewport.tsx` and `_getStatusComponent.tsx` files. Also, update initial segmentation configurations in `PanelSegmentation.tsx` and simplify configuration changes and values for segmentation service in `SegmentationService.ts`. Finally, remove console debug in `CornerstoneViewportService.ts`.

* wip for highlighting contours

* refactor rt displayset code

* review code update

* update cornerstone dependencies

* refactor: Update license year, version number, and minor code cleanup

This commit updates the license year in several files, updates the version number in package.json, and contains minor code cleanup in two files.

* add bulkdataURI retrieve for RT

* fix package version

* apply review comments

* apply review comments

* apply review comments

* feat(panels): refactor and streamline segmentation configuration and inputs

Rewrote state hooks and streamlined the configuration input for `PanelSegmentation` to be more verbose and reusable. Included several new input types, including the `InputRange` component which now shows a fixed floating value based on the step provided. The `SegmentationConfig` component now works with dynamic values controlled by `initialConfig`. These changes should improve function usability and make the code more maintainable going forward.

* fix various bugs

* fix contour delete by upgrade cs3d version

* feat(viewport, inputNumber, segmentationConfig, orthanc): Implement minimum and maximum values for input number components, and useBulkDataURI for Orthanc configuration. Compare measurement view planes with absolute viewport view planes in Cornerstone viewport.

* update yarn lock

extensions/cornerstone/package.json

@@ -50,8 +50,8 @@
     "@babel/runtime": "^7.20.13",
     "@cornerstonejs/adapters": "^0.6.0",
     "@cornerstonejs/core": "^0.42.2",
-    "@cornerstonejs/streaming-image-volume-loader": "^0.16.0",
-    "@cornerstonejs/tools": "^0.61.11",
+    "@cornerstonejs/streaming-image-volume-loader": "^0.16.2",
+    "@cornerstonejs/tools": "^0.63.2",
     "@kitware/vtk.js": "26.5.6",
     "html2canvas": "^1.4.1",
     "lodash.debounce": "4.0.8",

extensions/cornerstone/src/Viewport/OHIFCornerstoneViewport.tsx

@@ -630,19 +630,19 @@ function _jumpToMeasurement(
     } else {
       // for volume viewport we can't rely on the imageIdIndex since it can be
       // a reconstructed view that doesn't match the original slice numbers etc.
-      const { viewPlaneNormal } = measurement.metadata;
+      const { viewPlaneNormal: measurementViewPlane } = measurement.metadata;
       imageIdIndex = referencedDisplaySet.images.findIndex(
         i => i.SOPInstanceUID === SOPInstanceUID
       );
 
-      const { orientation } = viewportInfo.getViewportOptions();
+      const { viewPlaneNormal: viewportViewPlane } = viewport.getCamera();
 
+      // should compare abs for both planes since the direction can be flipped
       if (
-        orientation &&
-        viewPlaneNormal &&
+        measurementViewPlane &&
         !csUtils.isEqual(
-          CONSTANTS.MPR_CAMERA_VALUES[orientation]?.viewPlaneNormal,
-          viewPlaneNormal
+          measurementViewPlane.map(Math.abs),
+          viewportViewPlane.map(Math.abs)
         )
       ) {
         viewportCameraDirectionMatch = false;

extensions/cornerstone/src/getCustomizationModule.ts

@@ -0,0 +1,37 @@
+import { Enums } from '@cornerstonejs/tools';
+import { toolNames } from './initCornerstoneTools';
+
+const tools = {
+  active: [
+    {
+      toolName: toolNames.WindowLevel,
+      bindings: [{ mouseButton: Enums.MouseBindings.Primary }],
+    },
+    {
+      toolName: toolNames.Pan,
+      bindings: [{ mouseButton: Enums.MouseBindings.Auxiliary }],
+    },
+    {
+      toolName: toolNames.Zoom,
+      bindings: [{ mouseButton: Enums.MouseBindings.Secondary }],
+    },
+    { toolName: toolNames.StackScrollMouseWheel, bindings: [] },
+  ],
+  enabled: [{ toolName: toolNames.SegmentationDisplay }],
+};
+
+function getCustomizationModule() {
+  return [
+    {
+      name: 'default',
+      value: [
+        {
+          id: 'cornerstone.overlayViewportTools',
+          tools,
+        },
+      ],
+    },
+  ];
+}
+
+export default getCustomizationModule;

extensions/cornerstone/src/index.tsx

@@ -10,6 +10,7 @@ import { Enums as cs3DToolsEnums } from '@cornerstonejs/tools';
 import { ServicesManager, Types } from '@ohif/core';
 
 import init from './init';
+import getCustomizationModule from './getCustomizationModule';
 import getCommandsModule from './commandsModule';
 import getHangingProtocolModule from './getHangingProtocolModule';
 import ToolGroupService from './services/ToolGroupService';
@@ -108,6 +109,7 @@ const cornerstoneExtension: Types.Extensions.Extension = {
     ];
   },
   getCommandsModule,
+  getCustomizationModule,
   getUtilityModule({ servicesManager }) {
     return [
       {
@@ -139,5 +141,5 @@ const cornerstoneExtension: Types.Extensions.Extension = {
 };
 
 export type { PublicViewportOptions };
-export { measurementMappingUtils, CornerstoneExtensionTypes };
+export { measurementMappingUtils, CornerstoneExtensionTypes, toolNames };
 export default cornerstoneExtension;

extensions/cornerstone/src/init.tsx

@@ -1,4 +1,4 @@
-import OHIF from '@ohif/core';
+import OHIF, { Types } from '@ohif/core';
 import React from 'react';
 
 import * as cornerstone from '@cornerstonejs/core';

extensions/cornerstone/src/services/SegmentationService/RTSTRUCT/mapROIContoursToRTStructData.ts

@@ -0,0 +1,38 @@
+/**
+ * Maps a DICOM RT Struct ROI Contour to a RTStruct data that can be used
+ * in Segmentation Service
+ *
+ * @param structureSet - A DICOM RT Struct ROI Contour
+ * @param rtDisplaySetUID - A CornerstoneTools DisplaySet UID
+ * @returns An array of object that includes data, id, segmentIndex, color
+ * and geometry Id
+ */
+export function mapROIContoursToRTStructData(
+  structureSet: unknown,
+  rtDisplaySetUID: unknown
+) {
+  return structureSet.ROIContours.map(
+    ({ contourPoints, ROINumber, ROIName, colorArray }) => {
+      const data = contourPoints.map(({ points, ...rest }) => {
+        const newPoints = points.map(({ x, y, z }) => {
+          return [x, y, z];
+        });
+
+        return {
+          ...rest,
+          points: newPoints,
+        };
+      });
+
+      const id = ROIName || ROINumber;
+
+      return {
+        data,
+        id,
+        segmentIndex: ROINumber,
+        color: colorArray,
+        geometryId: `${rtDisplaySetUID}:${id}:segmentIndex-${ROINumber}`,
+      };
+    }
+  );
+}

extensions/cornerstone/src/services/SegmentationService/SegmentationServiceTypes.ts

@@ -29,6 +29,8 @@ type Segmentation = {
   colorLUTIndex: number;
   // if segmentation contains any data (often calculated from labelmap)
   cachedStats: Record<string, any>;
+  // displaySetInstanceUID
+  displaySetInstanceUID: string;
   // displayText is the text that is displayed on the segmentation panel (often derived from the data)
   displayText?: string[];
   // the id of the segmentation
@@ -45,44 +47,22 @@ type Segmentation = {
   segments: Array<Segment>;
   // the set of segments that are locked
   segmentsLocked: Array<number>;
-  // the segmentation representation type
-  type: csToolsEnums.SegmentationRepresentations;
-  // if labelmap, the id of the volume that the labelmap is associated with
-  volumeId?: string;
   // whether the segmentation is hydrated or not (non-hydrated SEG -> temporary segmentation for display in SEG Viewport
   // but hydrated SEG -> segmentation that is persisted in the store)
   hydrated: boolean;
-};
-
-// Schema to generate a segmentation
-type SegmentationSchema = {
-  // active segment index for the segmentation
-  activeSegmentIndex: number;
-  // statistics that are derived from the segmentation
-  cachedStats: Record<string, any>;
-  // the displayText for the segmentation in the panels
-  displayText?: string[];
-  // segmentation id
-  id: string;
-  // displaySetInstanceUID
-  displaySetInstanceUID: string;
-  // segmentation label
-  label: string;
-  // segment indices that are locked for the segmentation
-  segmentsLocked: Array<number>;
   // the type of the segmentation (e.g., Labelmap etc.)
   type: csToolsEnums.SegmentationRepresentations;
-  // the volume id of the volume that the labelmap is associated with, this only exists for the labelmap representation
-  volumeId: string;
-  // the referenced volumeURI for the segmentation
-  referencedVolumeURI: string;
-  // whether the segmentation is hydrated or not (non-hydrated SEG -> temporary segmentation for display in SEG Viewport
-  // but hydrated SEG -> segmentation that is persisted in the store)
-  hydrated: boolean;
-  // the number of segments in the segmentation
-  segmentCount: number;
-  // the array of segments with their details
-  segments: Array<Segment>;
+  // the segmentation representation data
+  representationData: SegmentationRepresentationData;
 };
 
-export { SegmentationConfig, Segment, Segmentation, SegmentationSchema };
+type LabelmapSegmentationData = {
+  volumeId: string;
+  referencedVolumeId?: string;
+};
+
+type SegmentationRepresentationData = {
+  LABELMAP?: LabelmapSegmentationData;
+};
+
+export { SegmentationConfig, Segment, Segmentation };

extensions/cornerstone/src/services/ViewportService/CornerstoneViewportService.ts

@@ -11,9 +11,13 @@ import {
   cache,
   utilities,
   CONSTANTS,
+  Enums as csEnums,
 } from '@cornerstonejs/core';
 
-import { utilities as csToolsUtils } from '@cornerstonejs/tools';
+import {
+  utilities as csToolsUtils,
+  Enums as csToolsEnums,
+} from '@cornerstonejs/tools';
 import { IViewportService } from './IViewportService';
 import { RENDERING_ENGINE_ID } from './constants';
 import ViewportInfo, {
@@ -386,7 +390,7 @@ class CornerstoneViewportService extends PubSubService
       initialImageIndexToUse === null
     ) {
       initialImageIndexToUse =
-        this._getInitialImageIndexForStackViewport(viewportInfo, imageIds) || 0;
+        this._getInitialImageIndexForViewport(viewportInfo, imageIds) || 0;
     }
 
     const properties = { ...presentations.lutPresentation?.properties };
@@ -412,7 +416,7 @@ class CornerstoneViewportService extends PubSubService
     });
   }
 
-  private _getInitialImageIndexForStackViewport(
+  private _getInitialImageIndexForViewport(
     viewportInfo: ViewportInfo,
     imageIds?: string[]
   ): number {
@@ -423,7 +427,29 @@ class CornerstoneViewportService extends PubSubService
     }
 
     const { index, preset } = initialImageOptions;
-    return this._getInitialImageIndex(imageIds.length, index, preset);
+    const viewportType = viewportInfo.getViewportType();
+
+    let numberOfSlices;
+    if (viewportType === csEnums.ViewportType.STACK) {
+      numberOfSlices = imageIds.length;
+    } else if (viewportType === csEnums.ViewportType.ORTHOGRAPHIC) {
+      const viewport = this.getCornerstoneViewport(
+        viewportInfo.getViewportId()
+      );
+      const imageSliceData = csUtils.getImageSliceDataForVolumeViewport(
+        viewport
+      );
+
+      if (!imageSliceData) {
+        return;
+      }
+
+      ({ numberOfSlices } = imageSliceData);
+    } else {
+      return;
+    }
+
+    return this._getInitialImageIndex(numberOfSlices, index, preset);
   }
 
   _getInitialImageIndex(
@@ -548,7 +574,6 @@ class CornerstoneViewportService extends PubSubService
   ) {
     const {
       displaySetService,
-      segmentationService,
       toolGroupService,
     } = this.servicesManager.services;
 
@@ -558,110 +583,32 @@ class CornerstoneViewportService extends PubSubService
     // load any secondary displaySets
     const displaySetInstanceUIDs = this.viewportsDisplaySets.get(viewport.id);
 
-    const segDisplaySet = displaySetInstanceUIDs
+    // can be SEG or RTSTRUCT for now
+    const overlayDisplaySet = displaySetInstanceUIDs
       .map(displaySetService.getDisplaySetByUID)
-      .find(displaySet => displaySet && displaySet.Modality === 'SEG');
+      .find(displaySet => displaySet?.isOverlayDisplaySet);
 
-    if (segDisplaySet) {
-      const { referencedVolumeId } = segDisplaySet;
-      const referencedVolume = cache.getVolume(referencedVolumeId);
-      const segmentationId = segDisplaySet.displaySetInstanceUID;
-
-      const toolGroup = toolGroupService.getToolGroupForViewport(viewport.id);
-
-      if (referencedVolume) {
-        segmentationService.addSegmentationRepresentationToToolGroup(
-          toolGroup.id,
-          segmentationId
-        );
-      }
+    if (overlayDisplaySet) {
+      this.addOverlayRepresentationForDisplaySet(overlayDisplaySet, viewport);
     } else {
-      const toolGroup = toolGroupService.getToolGroupForViewport(viewport.id);
-      const toolGroupSegmentationRepresentations =
-        segmentationService.getSegmentationRepresentationsForToolGroup(
-          toolGroup.id
-        ) || [];
-
-      // csToolsUtils.segmentation.triggerSegmentationRender(toolGroup.id);
       // If the displaySet is not a SEG displaySet we assume it is a primary displaySet
       // and we can look into hydrated segmentations to check if any of them are
       // associated with the primary displaySet
+
       // get segmentations only returns the hydrated segmentations
-      const segmentations = segmentationService.getSegmentations();
-
-      for (const segmentation of segmentations) {
-        // if there is already a segmentation representation for this segmentation
-        // for this toolGroup, don't bother at all
-        if (
-          toolGroupSegmentationRepresentations.find(
-            representation => representation.segmentationId === segmentation.id
-          )
-        ) {
-          continue;
-        }
-
-        // otherwise, check if the hydrated segmentations are in the same FOR
-        // as the primary displaySet, if so add the representation (since it was not there)
-        const { id: segDisplaySetInstanceUID } = segmentation;
-
-        const segFrameOfReferenceUID = this._getFrameOfReferenceUID(
-          segDisplaySetInstanceUID
-        );
-
-        let shouldDisplaySeg = false;
-
-        for (const displaySetInstanceUID of displaySetInstanceUIDs) {
-          const primaryFrameOfReferenceUID = this._getFrameOfReferenceUID(
-            displaySetInstanceUID
-          );
-
-          if (segFrameOfReferenceUID === primaryFrameOfReferenceUID) {
-            shouldDisplaySeg = true;
-            break;
-          }
-        }
-
-        if (shouldDisplaySeg) {
-          const toolGroup = toolGroupService.getToolGroupForViewport(
-            viewport.id
-          );
-
-          segmentationService.addSegmentationRepresentationToToolGroup(
-            toolGroup.id,
-            segmentation.id
-          );
-        }
-      }
+      this._addSegmentationRepresentationToToolGroupIfNecessary(
+        displaySetInstanceUIDs,
+        viewport
+      );
     }
 
     const viewportInfo = this.getViewportInfo(viewport.id);
-
-    if (!viewportInfo) {
-      console.warn('Viewport info not defined for', viewport.id);
-    }
-
     const toolGroup = toolGroupService.getToolGroupForViewport(viewport.id);
     csToolsUtils.segmentation.triggerSegmentationRender(toolGroup.id);
 
-    const initialImageOptions = viewportInfo.getInitialImageOptions();
-
-    if (
-      initialImageOptions &&
-      (initialImageOptions.preset !== undefined ||
-        initialImageOptions.index !== undefined)
-    ) {
-      const { index, preset } = initialImageOptions;
-
-      const { numberOfSlices } = csUtils.getImageSliceDataForVolumeViewport(
-        viewport
-      );
-
-      const imageIndex = this._getInitialImageIndex(
-        numberOfSlices,
-        index,
-        preset
-      );
+    const imageIndex = this._getInitialImageIndexForViewport(viewportInfo);
 
+    if (imageIndex !== undefined) {
       csToolsUtils.jumpToSlice(viewport.element, {
         imageIndex,
       });
@@ -670,6 +617,96 @@ class CornerstoneViewportService extends PubSubService
     viewport.render();
   }
 
+  private _addSegmentationRepresentationToToolGroupIfNecessary(
+    displaySetInstanceUIDs: string[],
+    viewport: any
+  ) {
+    const {
+      segmentationService,
+      toolGroupService,
+    } = this.servicesManager.services;
+
+    const toolGroup = toolGroupService.getToolGroupForViewport(viewport.id);
+
+    // this only returns hydrated segmentations
+    const segmentations = segmentationService.getSegmentations();
+
+    for (const segmentation of segmentations) {
+      const toolGroupSegmentationRepresentations =
+        segmentationService.getSegmentationRepresentationsForToolGroup(
+          toolGroup.id
+        ) || [];
+
+      // if there is already a segmentation representation for this segmentation
+      // for this toolGroup, don't bother at all
+      const isSegmentationInToolGroup = toolGroupSegmentationRepresentations.find(
+        representation => representation.segmentationId === segmentation.id
+      );
+
+      if (isSegmentationInToolGroup) {
+        continue;
+      }
+
+      // otherwise, check if the hydrated segmentations are in the same FOR
+      // as the primary displaySet, if so add the representation (since it was not there)
+      const { id: segDisplaySetInstanceUID, type } = segmentation;
+      const segFrameOfReferenceUID = this._getFrameOfReferenceUID(
+        segDisplaySetInstanceUID
+      );
+
+      let shouldDisplaySeg = false;
+
+      for (const displaySetInstanceUID of displaySetInstanceUIDs) {
+        const primaryFrameOfReferenceUID = this._getFrameOfReferenceUID(
+          displaySetInstanceUID
+        );
+
+        if (segFrameOfReferenceUID === primaryFrameOfReferenceUID) {
+          shouldDisplaySeg = true;
+          break;
+        }
+      }
+
+      if (!shouldDisplaySeg) {
+        return;
+      }
+
+      segmentationService.addSegmentationRepresentationToToolGroup(
+        toolGroup.id,
+        segmentation.id,
+        false, // already hydrated,
+        segmentation.type
+      );
+    }
+  }
+
+  private addOverlayRepresentationForDisplaySet(
+    displaySet: any,
+    viewport: any
+  ) {
+    const {
+      segmentationService,
+      toolGroupService,
+    } = this.servicesManager.services;
+
+    const { referencedVolumeId } = displaySet;
+    const segmentationId = displaySet.displaySetInstanceUID;
+
+    const toolGroup = toolGroupService.getToolGroupForViewport(viewport.id);
+
+    const representationType =
+      referencedVolumeId && cache.getVolume(referencedVolumeId) !== undefined
+        ? csToolsEnums.SegmentationRepresentations.Labelmap
+        : csToolsEnums.SegmentationRepresentations.Contour;
+
+    segmentationService.addSegmentationRepresentationToToolGroup(
+      toolGroup.id,
+      segmentationId,
+      false,
+      representationType
+    );
+  }
+
   // Todo: keepCamera is an interim solution until we have a better solution for
   // keeping the camera position when the viewport data is changed
   public updateViewport(
@@ -859,6 +896,11 @@ class CornerstoneViewportService extends PubSubService
       return instance.FrameOfReferenceUID;
     }
 
+    if (displaySet.Modality === 'RTSTRUCT') {
+      const { instance } = displaySet;
+      return instance.ReferencedFrameOfReferenceSequence.FrameOfReferenceUID;
+    }
+
     const { images } = displaySet;
     if (images && images.length) {
       return images[0].FrameOfReferenceUID;

extensions/cornerstone/src/utils/transitions.ts

@@ -20,3 +20,17 @@ export function easeInOutBell(x: number, baseline: number): number {
     return (- 4 * Math.pow(2 * x - 2, 3)) * alpha + baseline;
   }
 }
+
+/**
+ * A reversed bell curved function that starts from 1 and goes to baseline and
+ * come back to 1 again. It uses ease in out quadratic for css transition
+ * timing function for each side of the curve.
+ *
+ * @param {number} x - The current time, in the range [0, 1].
+ * @param {number} baseline - The baseline value to start from and return to.
+ * @returns the value of the transition at time x.
+ */
+export function reverseEaseInOutBell(x: number, baseline: number): number {
+  const y = easeInOutBell(x, baseline);
+  return -y + 1 + baseline;
+}