feat: Add CS3D WSI and Video Viewports and add annotation navigation for MPR (#4182)

extensions/cornerstone/package.json

@@ -38,7 +38,7 @@
     "@cornerstonejs/codec-libjpeg-turbo-8bit": "^1.2.2",
     "@cornerstonejs/codec-openjpeg": "^1.2.2",
     "@cornerstonejs/codec-openjph": "^2.4.2",
-    "@cornerstonejs/dicom-image-loader": "^1.82.0",
+    "@cornerstonejs/dicom-image-loader": "^1.82.4",
     "@icr/polyseg-wasm": "^0.4.0",
     "@ohif/core": "3.9.0-beta.77",
     "@ohif/ui": "3.9.0-beta.77",
@@ -55,10 +55,10 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^1.82.0",
-    "@cornerstonejs/core": "^1.82.0",
-    "@cornerstonejs/streaming-image-volume-loader": "^1.82.0",
-    "@cornerstonejs/tools": "^1.82.0",
+    "@cornerstonejs/adapters": "^1.82.4",
+    "@cornerstonejs/core": "^1.82.4",
+    "@cornerstonejs/streaming-image-volume-loader": "^1.82.4",
+    "@cornerstonejs/tools": "^1.82.4",
     "@icr/polyseg-wasm": "^0.4.0",
     "@kitware/vtk.js": "30.4.1",
     "html2canvas": "^1.4.1",

extensions/cornerstone/src/Viewport/OHIFCornerstoneViewport.tsx

@@ -2,24 +2,17 @@ import React, { useEffect, useRef, useCallback, useState } from 'react';
 import { useResizeDetector } from 'react-resize-detector';
 import PropTypes from 'prop-types';
 import * as cs3DTools from '@cornerstonejs/tools';
-import {
-  Enums,
-  eventTarget,
-  getEnabledElement,
-  StackViewport,
-  utilities as csUtils,
-} from '@cornerstonejs/core';
+import { Enums, eventTarget, getEnabledElement } from '@cornerstonejs/core';
 import { MeasurementService } from '@ohif/core';
 import { Notification, useViewportDialog, AllInOneMenu } from '@ohif/ui';
-import { IStackViewport, IVolumeViewport } from '@cornerstonejs/core/dist/esm/types';
+import type { Types as csTypes } from '@cornerstonejs/core';
 
 import { setEnabledElement } from '../state';
 
 import './OHIFCornerstoneViewport.css';
 import CornerstoneOverlays from './Overlays/CornerstoneOverlays';
-import getSOPInstanceAttributes from '../utils/measurementServiceMappings/utils/getSOPInstanceAttributes';
 import CinePlayer from '../components/CinePlayer';
-import { Types } from '@ohif/core';
+import type { Types } from '@ohif/core';
 
 import OHIFViewportActionCorners from '../components/OHIFViewportActionCorners';
 import { getWindowLevelActionMenu } from '../components/WindowLevelActionMenu/getWindowLevelActionMenu';
@@ -106,6 +99,7 @@ const OHIFCornerstoneViewport = React.memo((props: withAppTypes) => {
     viewportOptions,
     displaySetOptions,
     servicesManager,
+    commandsManager,
     onElementEnabled,
     // eslint-disable-next-line react/prop-types
     onElementDisabled,
@@ -149,14 +143,12 @@ const OHIFCornerstoneViewport = React.memo((props: withAppTypes) => {
   const [appConfig] = useAppConfig();
 
   const {
-    measurementService,
     displaySetService,
     toolbarService,
     toolGroupService,
     syncGroupService,
     cornerstoneViewportService,
     cornerstoneCacheService,
-    viewportGridService,
     stateSyncService,
     viewportActionCornersService,
   } = servicesManager.services;
@@ -370,24 +362,12 @@ const OHIFCornerstoneViewport = React.memo((props: withAppTypes) => {
     }
 
     const unsubscribeFromJumpToMeasurementEvents = _subscribeToJumpToMeasurementEvents(
-      measurementService,
-      displaySetService,
       elementRef,
       viewportId,
-      displaySets,
-      viewportGridService,
-      cornerstoneViewportService
+      servicesManager
     );
 
-    _checkForCachedJumpToMeasurementEvents(
-      measurementService,
-      displaySetService,
-      elementRef,
-      viewportId,
-      displaySets,
-      viewportGridService,
-      cornerstoneViewportService
-    );
+    _checkForCachedJumpToMeasurementEvents(elementRef, viewportId, displaySets, servicesManager);
 
     return () => {
       unsubscribeFromJumpToMeasurementEvents();
@@ -471,15 +451,9 @@ const OHIFCornerstoneViewport = React.memo((props: withAppTypes) => {
   );
 }, areEqual);
 
-function _subscribeToJumpToMeasurementEvents(
-  measurementService,
-  displaySetService,
-  elementRef,
-  viewportId,
-  displaySets,
-  viewportGridService,
-  cornerstoneViewportService
-) {
+function _subscribeToJumpToMeasurementEvents(elementRef, viewportId, servicesManager) {
+  const { measurementService, cornerstoneViewportService } = servicesManager.services;
+
   const { unsubscribe } = measurementService.subscribe(
     MeasurementService.EVENTS.JUMP_TO_MEASUREMENT_VIEWPORT,
     props => {
@@ -491,27 +465,17 @@ function _subscribeToJumpToMeasurementEvents(
       if (cacheJumpToMeasurementEvent.cornerstoneViewport === undefined) {
         // Decide on which viewport should handle this
         cacheJumpToMeasurementEvent.cornerstoneViewport =
-          cornerstoneViewportService.getViewportIdToJump(
-            jumpId,
-            measurement.displaySetInstanceUID,
-            {
-              referencedImageId:
-                measurement.referencedImageId || measurement.metadata?.referencedImageId,
-            }
-          );
+          cornerstoneViewportService.getViewportIdToJump(jumpId, {
+            displaySetInstanceUID: measurement.displaySetInstanceUID,
+            ...measurement.metadata,
+            referencedImageId:
+              measurement.referencedImageId || measurement.metadata?.referencedImageId,
+          });
       }
       if (cacheJumpToMeasurementEvent.cornerstoneViewport !== viewportId) {
         return;
       }
-      _jumpToMeasurement(
-        measurement,
-        elementRef,
-        viewportId,
-        measurementService,
-        displaySetService,
-        viewportGridService,
-        cornerstoneViewportService
-      );
+      _jumpToMeasurement(measurement, elementRef, viewportId, servicesManager);
     }
   );
 
@@ -520,13 +484,10 @@ function _subscribeToJumpToMeasurementEvents(
 
 // Check if there is a queued jumpToMeasurement event
 function _checkForCachedJumpToMeasurementEvents(
-  measurementService,
-  displaySetService,
   elementRef,
   viewportId,
   displaySets,
-  viewportGridService,
-  cornerstoneViewportService
+  servicesManager
 ) {
   if (!cacheJumpToMeasurementEvent) {
     return;
@@ -544,37 +505,15 @@ function _checkForCachedJumpToMeasurementEvents(
   const { measurement } = cacheJumpToMeasurementEvent;
   if (measurement && elementRef) {
     if (displaysUIDs.includes(measurement?.displaySetInstanceUID)) {
-      _jumpToMeasurement(
-        measurement,
-        elementRef,
-        viewportId,
-        measurementService,
-        displaySetService,
-        viewportGridService,
-        cornerstoneViewportService
-      );
+      _jumpToMeasurement(measurement, elementRef, viewportId, servicesManager);
     }
   }
 }
 
-function _jumpToMeasurement(
-  measurement,
-  targetElementRef,
-  viewportId,
-  measurementService,
-  displaySetService,
-  viewportGridService,
-  cornerstoneViewportService
-) {
+function _jumpToMeasurement(measurement, targetElementRef, viewportId, servicesManager) {
+  const { viewportGridService } = servicesManager.services;
+
   const targetElement = targetElementRef.current;
-  const { displaySetInstanceUID, SOPInstanceUID, frameNumber } = measurement;
-
-  if (!SOPInstanceUID) {
-    console.warn('cannot jump in a non-acquisition plane measurements yet');
-    return;
-  }
-
-  const referencedDisplaySet = displaySetService.getDisplaySetByUID(displaySetInstanceUID);
 
   // Todo: setCornerstoneMeasurementActive should be handled by the toolGroupManager
   //  to set it properly
@@ -586,47 +525,15 @@ function _jumpToMeasurement(
 
   if (enabledElement) {
     // See how the jumpToSlice() of Cornerstone3D deals with imageIdx param.
-    const viewport = enabledElement.viewport as IStackViewport | IVolumeViewport;
+    const viewport = enabledElement.viewport as csTypes.IStackViewport | csTypes.IVolumeViewport;
 
-    let imageIdIndex = 0;
-    let viewportCameraDirectionMatch = true;
-
-    if (viewport instanceof StackViewport) {
-      const imageIds = viewport.getImageIds();
-      imageIdIndex = imageIds.findIndex(imageId => {
-        const { SOPInstanceUID: aSOPInstanceUID, frameNumber: aFrameNumber } =
-          getSOPInstanceAttributes(imageId);
-        return aSOPInstanceUID === SOPInstanceUID && (!frameNumber || frameNumber === aFrameNumber);
-      });
-    } else {
-      // for volume viewport we can't rely on the imageIdIndex since it can be
-      // a reconstructed view that doesn't match the original slice numbers etc.
-      const { viewPlaneNormal: measurementViewPlane } = measurement.metadata;
-      imageIdIndex = referencedDisplaySet.images.findIndex(
-        i => i.SOPInstanceUID === SOPInstanceUID
-      );
-
-      // the index is reversed in the volume viewport
-      // imageIdIndex = referencedDisplaySet.images.length - 1 - imageIdIndex;
-
-      const { viewPlaneNormal: viewportViewPlane } = viewport.getCamera();
-
-      // should compare abs for both planes since the direction can be flipped
-      if (
-        measurementViewPlane &&
-        !csUtils.isEqual(measurementViewPlane.map(Math.abs), viewportViewPlane.map(Math.abs))
-      ) {
-        viewportCameraDirectionMatch = false;
-      }
-    }
-
-    if (!viewportCameraDirectionMatch || imageIdIndex === -1) {
+    const { metadata } = measurement;
+    if (!viewport.isReferenceViewable(metadata, { withNavigation: true, withOrientation: true })) {
+      console.log("Reference isn't viewable, postponing until updated");
       return;
     }
 
-    cs3DTools.utilities.jumpToSlice(targetElement, {
-      imageIndex: imageIdIndex,
-    });
+    viewport.setViewReference(metadata);
 
     cs3DTools.annotation.selection.setAnnotationSelected(measurement.uid);
     // Jump to measurement consumed, remove.

extensions/cornerstone/src/Viewport/Overlays/ViewportImageScrollbar.tsx

@@ -43,67 +43,40 @@ function CornerstoneImageScrollbar({
       return;
     }
 
-    if (viewportData.viewportType === Enums.ViewportType.STACK) {
-      const imageIndex = viewport.getCurrentImageIdIndex();
+    const imageIndex = viewport.getCurrentImageIdIndex();
+    const numberOfSlices = viewport.getNumberOfSlices();
 
-      setImageSliceData({
-        imageIndex: imageIndex,
-        numberOfSlices: viewportData.data[0].imageIds.length,
-      });
-
-      return;
-    }
-
-    if (viewportData.viewportType === Enums.ViewportType.ORTHOGRAPHIC) {
-      const sliceData = utilities.getImageSliceDataForVolumeViewport(
-        viewport as Types.IVolumeViewport
-      );
-
-      if (!sliceData) {
-        return;
-      }
-
-      const { imageIndex, numberOfSlices } = sliceData;
-      setImageSliceData({ imageIndex, numberOfSlices });
-    }
+    setImageSliceData({
+      imageIndex: imageIndex,
+      numberOfSlices,
+    });
   }, [viewportId, viewportData]);
 
   useEffect(() => {
-    if (viewportData?.viewportType !== Enums.ViewportType.STACK) {
+    if (!viewportData) {
       return;
     }
+    const { viewportType } = viewportData;
+    const eventId =
+      (viewportType === Enums.ViewportType.STACK && Enums.Events.STACK_VIEWPORT_SCROLL) ||
+      (viewportType === Enums.ViewportType.ORTHOGRAPHIC && Enums.Events.VOLUME_NEW_IMAGE) ||
+      Enums.Events.IMAGE_RENDERED;
 
-    const updateStackIndex = event => {
-      const { newImageIdIndex } = event.detail;
+    const updateIndex = event => {
+      const viewport = cornerstoneViewportService.getCornerstoneViewport(viewportId);
+      const { imageIndex, newImageIdIndex = imageIndex } = event.detail;
+      const numberOfSlices = viewport.getNumberOfSlices();
       // find the index of imageId in the imageIds
       setImageSliceData({
         imageIndex: newImageIdIndex,
-        numberOfSlices: viewportData.data[0].imageIds.length,
+        numberOfSlices,
       });
     };
 
-    element.addEventListener(Enums.Events.STACK_VIEWPORT_SCROLL, updateStackIndex);
+    element.addEventListener(eventId, updateIndex);
 
     return () => {
-      element.removeEventListener(Enums.Events.STACK_VIEWPORT_SCROLL, updateStackIndex);
-    };
-  }, [viewportData, element]);
-
-  useEffect(() => {
-    if (viewportData?.viewportType !== Enums.ViewportType.ORTHOGRAPHIC) {
-      return;
-    }
-
-    const updateVolumeIndex = event => {
-      const { imageIndex, numberOfSlices } = event.detail;
-      // find the index of imageId in the imageIds
-      setImageSliceData({ imageIndex, numberOfSlices });
-    };
-
-    element.addEventListener(Enums.Events.VOLUME_NEW_IMAGE, updateVolumeIndex);
-
-    return () => {
-      element.removeEventListener(Enums.Events.VOLUME_NEW_IMAGE, updateVolumeIndex);
+      element.removeEventListener(eventId, updateIndex);
     };
   }, [viewportData, element]);
 
@@ -112,7 +85,7 @@ function CornerstoneImageScrollbar({
       onChange={evt => onImageScrollbarChange(evt, viewportId)}
       max={imageSliceData.numberOfSlices ? imageSliceData.numberOfSlices - 1 : 0}
       height={scrollbarHeight}
-      value={imageSliceData.imageIndex}
+      value={imageSliceData.imageIndex || 0}
     />
   );
 }

extensions/cornerstone/src/getSopClassHandlerModule.js

@@ -0,0 +1,170 @@
+import OHIF from '@ohif/core';
+import { utilities as csUtils, Enums as csEnums } from '@cornerstonejs/core';
+import dcmjs from 'dcmjs';
+import { dicomWebUtils } from '@ohif/extension-default';
+
+const { MetadataModules } = csEnums;
+const { utils } = OHIF;
+const { denaturalizeDataset } = dcmjs.data.DicomMetaDictionary;
+const { transferDenaturalizedDataset, fixMultiValueKeys } = dicomWebUtils;
+
+const SOP_CLASS_UIDS = {
+  VL_WHOLE_SLIDE_MICROSCOPY_IMAGE_STORAGE: '1.2.840.10008.5.1.4.1.1.77.1.6',
+};
+
+const SOPClassHandlerId =
+  '@ohif/extension-cornerstone.sopClassHandlerModule.DicomMicroscopySopClassHandler';
+
+function _getDisplaySetsFromSeries(instances, servicesManager, extensionManager) {
+  // If the series has no instances, stop here
+  if (!instances || !instances.length) {
+    throw new Error('No instances were provided');
+  }
+
+  const instance = instances[0];
+
+  let singleFrameInstance = instance;
+  let currentFrames = +singleFrameInstance.NumberOfFrames || 1;
+  for (const instanceI of instances) {
+    const framesI = +instanceI.NumberOfFrames || 1;
+    if (framesI < currentFrames) {
+      singleFrameInstance = instanceI;
+      currentFrames = framesI;
+    }
+  }
+  let imageIdForThumbnail = null;
+  const dataSource = extensionManager.getActiveDataSource()[0];
+  if (singleFrameInstance) {
+    if (currentFrames == 1) {
+      // Not all DICOM server implementations support thumbnail service,
+      // So if we have a single-frame image, we will prefer it.
+      imageIdForThumbnail = singleFrameInstance.imageId;
+    }
+    if (!imageIdForThumbnail) {
+      // use the thumbnail service provided by DICOM server
+      imageIdForThumbnail = dataSource.getImageIdsForInstance({
+        instance: singleFrameInstance,
+        thumbnail: true,
+      });
+    }
+  }
+
+  const {
+    FrameOfReferenceUID,
+    SeriesDescription,
+    ContentDate,
+    ContentTime,
+    SeriesNumber,
+    StudyInstanceUID,
+    SeriesInstanceUID,
+    SOPInstanceUID,
+    SOPClassUID,
+  } = instance;
+
+  instances = instances.map(inst => {
+    // NOTE: According to DICOM standard a series should have a FrameOfReferenceUID
+    // When the Microscopy file was built by certain tool from multiple image files,
+    // each instance's FrameOfReferenceUID is sometimes different.
+    // Even though this means the file was not well formatted DICOM VL Whole Slide Microscopy Image,
+    // the case is so often, so let's override this value manually here.
+    //
+    // https://dicom.nema.org/medical/dicom/current/output/chtml/part03/sect_C.7.4.html#sect_C.7.4.1.1.1
+
+    inst.FrameOfReferenceUID = instance.FrameOfReferenceUID;
+
+    return inst;
+  });
+
+  const othersFrameOfReferenceUID = instances
+    .filter(v => v)
+    .map(inst => inst.FrameOfReferenceUID)
+    .filter((value, index, array) => array.indexOf(value) === index);
+  if (othersFrameOfReferenceUID.length > 1) {
+    console.warn(
+      'Expected FrameOfReferenceUID of difference instances within a series to be the same, found multiple different values',
+      othersFrameOfReferenceUID
+    );
+  }
+
+  const displaySet = {
+    plugin: 'microscopy',
+    Modality: 'SM',
+    viewportType: csEnums.ViewportType.WholeSlide,
+    altImageText: 'Microscopy',
+    displaySetInstanceUID: utils.guid(),
+    SOPInstanceUID,
+    SeriesInstanceUID,
+    StudyInstanceUID,
+    FrameOfReferenceUID,
+    SOPClassHandlerId,
+    SOPClassUID,
+    SeriesDescription: SeriesDescription || 'Microscopy Data',
+    // Map ContentDate/Time to SeriesTime for series list sorting.
+    SeriesDate: ContentDate,
+    SeriesTime: ContentTime,
+    SeriesNumber,
+    firstInstance: singleFrameInstance, // top level instance in the image Pyramid
+    instance,
+    numImageFrames: 0,
+    numInstances: 1,
+    imageIdForThumbnail, // thumbnail image
+    others: instances, // all other level instances in the image Pyramid
+    instances,
+    othersFrameOfReferenceUID,
+    imageIds: instances.map(instance => instance.imageId),
+  };
+  // The microscopy viewer directly accesses the metadata already loaded, and
+  // uses the DICOMweb client library directly for loading, so it has to be
+  // provided here.
+  const dicomWebClient = dataSource.retrieve.getWadoDicomWebClient?.();
+  const instanceMap = new Map();
+  instances.forEach(instance => instanceMap.set(instance.imageId, instance));
+  if (dicomWebClient) {
+    const webClient = Object.create(dicomWebClient);
+    // This replaces just the dicom web metadata call with one which retrieves
+    // internally.
+    webClient.getDICOMwebMetadata = getDICOMwebMetadata.bind(webClient, instanceMap);
+
+    csUtils.genericMetadataProvider.addRaw(displaySet.imageIds[0], {
+      type: MetadataModules.WADO_WEB_CLIENT,
+      metadata: webClient,
+    });
+  } else {
+    // Might have some other way of getting the data in the future or internally?
+    // throw new Error('Unable to provide a DICOMWeb client library, microscopy will fail to view');
+  }
+
+  return [displaySet];
+}
+
+/**
+ * This method provides access to the internal DICOMweb metadata, used to avoid
+ * refetching the DICOMweb data.  It gets assigned as a member function to the
+ * dicom web client.
+ */
+function getDICOMwebMetadata(instanceMap, imageId) {
+  const instance = instanceMap.get(imageId);
+  if (!instance) {
+    console.warn('Metadata not already found for', imageId, 'in', instanceMap);
+    return this.super.getDICOMwebMetadata(imageId);
+  }
+  return transferDenaturalizedDataset(
+    denaturalizeDataset(fixMultiValueKeys(instanceMap.get(imageId)))
+  );
+}
+
+export function getDicomMicroscopySopClassHandler({ servicesManager, extensionManager }) {
+  const getDisplaySetsFromSeries = instances => {
+    return _getDisplaySetsFromSeries(instances, servicesManager, extensionManager);
+  };
+
+  return {
+    name: 'DicomMicroscopySopClassHandler',
+    sopClassUids: [SOP_CLASS_UIDS.VL_WHOLE_SLIDE_MICROSCOPY_IMAGE_STORAGE],
+    getDisplaySetsFromSeries,
+  };
+}
+
+export function getSopClassHandlerModule(params) {
+  return [getDicomMicroscopySopClassHandler(params)];
+}

extensions/cornerstone/src/index.tsx

@@ -39,6 +39,7 @@ import ActiveViewportWindowLevel from './components/ActiveViewportWindowLevel';
 import getSOPInstanceAttributes from './utils/measurementServiceMappings/utils/getSOPInstanceAttributes';
 import { findNearbyToolData } from './utils/findNearbyToolData';
 import { createFrameViewSynchronizer } from './synchronizers/frameViewSynchronizer';
+import { getSopClassHandlerModule } from './getSopClassHandlerModule';
 
 const { helpers: volumeLoaderHelpers } = csStreamingImageVolumeLoader;
 const { getDynamicVolumeInfo } = volumeLoaderHelpers ?? {};
@@ -218,6 +219,7 @@ const cornerstoneExtension: Types.Extensions.Extension = {
       },
     ];
   },
+  getSopClassHandlerModule,
 };
 
 export type { PublicViewportOptions };

extensions/cornerstone/src/services/CornerstoneCacheService/CornerstoneCacheService.ts

@@ -32,7 +32,7 @@ class CornerstoneCacheService {
   }
 
   public async createViewportData(
-    displaySets: unknown[],
+    displaySets: Types.DisplaySet[],
     viewportOptions: Record<string, unknown>,
     dataSource: unknown,
     initialImageIndex?: number
@@ -55,23 +55,29 @@ class CornerstoneCacheService {
       viewportOptions.viewportType = viewportType;
     }
 
-    const cs3DViewportType = getCornerstoneViewportType(viewportType);
+    const cs3DViewportType = getCornerstoneViewportType(viewportType, displaySets);
     let viewportData: StackViewportData | VolumeViewportData;
 
-    if (cs3DViewportType === Enums.ViewportType.STACK) {
-      viewportData = await this._getStackViewportData(
-        dataSource,
-        displaySets,
-        initialImageIndex,
-        cs3DViewportType
-      );
-    }
-
     if (
       cs3DViewportType === Enums.ViewportType.ORTHOGRAPHIC ||
       cs3DViewportType === Enums.ViewportType.VOLUME_3D
     ) {
       viewportData = await this._getVolumeViewportData(dataSource, displaySets, cs3DViewportType);
+    } else if (cs3DViewportType === Enums.ViewportType.STACK) {
+      // Everything else looks like a stack
+      viewportData = await this._getStackViewportData(
+        dataSource,
+        displaySets,
+        initialImageIndex,
+        cs3DViewportType
+      );
+    } else {
+      viewportData = await this._getOtherViewportData(
+        dataSource,
+        displaySets,
+        initialImageIndex,
+        cs3DViewportType
+      );
     }
 
     viewportData.viewportType = cs3DViewportType;
@@ -143,6 +149,25 @@ class CornerstoneCacheService {
     return newViewportData;
   }
 
+  private async _getOtherViewportData(
+    dataSource,
+    displaySets,
+    _initialImageIndex,
+    viewportType: Enums.ViewportType
+  ): Promise<StackViewportData> {
+    // TODO - handle overlays and secondary display sets, but for now assume
+    // the 1st display set is the one of interest
+    const [displaySet] = displaySets;
+    if (!displaySet.imageIds) {
+      displaySet.imagesIds = this._getCornerstoneStackImageIds(displaySet, dataSource);
+    }
+    const { imageIds: data, viewportType: dsViewportType } = displaySet;
+    return {
+      viewportType: dsViewportType || viewportType,
+      data: displaySets,
+    };
+  }
+
   private async _getStackViewportData(
     dataSource,
     displaySets,

extensions/cornerstone/src/services/ViewportService/CornerstoneViewportService.ts

@@ -65,6 +65,14 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
     this.viewportGridResizeObserver = null;
     this.servicesManager = servicesManager;
   }
+  hangingProtocolService: unknown;
+  viewportsInfo: unknown;
+  sceneVolumeInputs: unknown;
+  viewportDivElements: unknown;
+  ViewportPropertiesMap: unknown;
+  volumeUIDs: unknown;
+  displaySetsNeedRerendering: unknown;
+  viewportDisplaySets: unknown;
 
   /**
    * Adds the HTML element to the viewportService
@@ -166,17 +174,18 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
    *
    * @param viewportId - The ID of the viewport.
    * @param presentations - The presentations to apply to the viewport.
+   * @param viewportInfo - Contains a view reference for immediate application
    */
-  public setPresentations(viewportId: string, presentations?: Presentations): void {
+  public setPresentations(
+    viewportId: string,
+    presentations?: Presentations,
+    viewportInfo?: ViewportInfo
+  ): void {
     const viewport = this.getCornerstoneViewport(viewportId) as
       | Types.IStackViewport
       | Types.IVolumeViewport;
 
-    if (!viewport) {
-      return;
-    }
-
-    if (!presentations) {
+    if (!viewport || !presentations) {
       return;
     }
 
@@ -196,17 +205,12 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
       }
     }
 
-    if (positionPresentation) {
-      const { viewPlaneNormal, viewUp, zoom, pan } = positionPresentation.presentation;
-      viewport.setCamera({ viewPlaneNormal, viewUp });
-
-      if (zoom !== undefined) {
-        viewport.setZoom(zoom);
-      }
-
-      if (pan !== undefined) {
-        viewport.setPan(pan);
-      }
+    const viewRef = viewportInfo?.getViewReference() || positionPresentation?.viewReference;
+    if (viewRef) {
+      viewport.setViewReference(viewRef);
+    }
+    if (positionPresentation?.position) {
+      viewport.setViewPresentation(positionPresentation.position);
     }
   }
 
@@ -235,21 +239,11 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
       return;
     }
 
-    const { viewPlaneNormal, viewUp } = csViewport.getCamera();
-    const initialImageIndex = csViewport.getCurrentImageIdIndex() || 0;
-    const zoom = csViewport.getZoom();
-    const pan = csViewport.getPan();
-
     return {
       id: positionPresentationId,
       viewportType: viewportInfo.getViewportType(),
-      presentation: {
-        initialImageIndex,
-        viewUp,
-        viewPlaneNormal,
-        zoom,
-        pan,
-      },
+      viewReference: csViewport.getViewReference(),
+      position: csViewport.getViewPresentation({ pan: true, zoom: true }),
     };
   }
 
@@ -428,7 +422,14 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
     // override the viewportOptions and displaySetOptions with the public ones
     // since those are the newly set ones, we set them here so that it handles defaults
     const displaySetOptions = viewportInfo.setPublicDisplaySetOptions(publicDisplaySetOptions);
-    const viewportOptions = viewportInfo.setPublicViewportOptions(publicViewportOptions);
+    // Specify an over-ride for the viewport type, even though it is in the public
+    // viewport options, because the one in the public viewport options is a suggestion
+    // for initial view, whereas the one in viewportData is a requirement based on the
+    // type of data being displayed.
+    const viewportOptions = viewportInfo.setPublicViewportOptions(
+      publicViewportOptions,
+      viewportData.viewportType
+    );
 
     const element = viewportInfo.getElement();
     const type = viewportInfo.getViewportType();
@@ -518,35 +519,80 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
 
   /**
    * Looks through the viewports to see if the specified measurement can be
-   * displayed in one of the viewports.
+   * displayed in one of the viewports. This function tries to get a "best fit"
+   * viewport to display the image in where it matches, in order:
+   *   * Active viewport that can be navigated to the given image without orientation change
+   *   * Other viewport that can be navigated to the given image without orientation change
+   *   * Active viewport that can change orientation to display the image
+   *   * Other viewport that can change orientation to display the image
    *
-   * @param measurement
-   *          The measurement that is desired to view.
+   * It returns `null` otherwise, indicating that a viewport needs display set/type
+   * changes in order to display the image.
+   *
+   * Notes:
+   *   * If the display set is displayed in multiple viewports all needing orientation change,
+   *     then the active one or first one listed will be modified.  This can create unexpected
+   *     behaviour for MPR views.
+   *   * If the image is contained in multiple display sets, then the first one
+   *     found will be navigated (active first, followed by first found)
+   *
+   * @param measurement - The measurement that is desired to view.
    * @param activeViewportId - the index that was active at the time the jump
    *          was initiated.
    * @return the viewportId that the measurement should be displayed in.
    */
-  public getViewportIdToJump(
-    activeViewportId: string,
-    displaySetInstanceUID: string,
-    cameraProps: unknown
-  ): string {
-    const viewportInfo = this.getViewportInfo(activeViewportId);
-
-    if (viewportInfo.getViewportType() === csEnums.ViewportType.VOLUME_3D) {
-      return null;
-    }
-
-    const { referencedImageId } = cameraProps;
-    if (viewportInfo?.contains(displaySetInstanceUID, referencedImageId)) {
+  public getViewportIdToJump(activeViewportId: string, metadata): string {
+    // First check if the active viewport can just be navigated to show the given item
+    const activeViewport = this.getCornerstoneViewport(activeViewportId);
+    if (activeViewport.isReferenceViewable(metadata, { withNavigation: true })) {
       return activeViewportId;
     }
 
-    return (
-      [...this.viewportsById.values()].find(viewportInfo =>
-        viewportInfo.contains(displaySetInstanceUID, referencedImageId)
-      )?.viewportId ?? null
-    );
+    // Next, see if any viewport could be navigated to show the given item,
+    // without considering orientation changes.
+    for (const id of this.viewportsById.keys()) {
+      const viewport = this.getCornerstoneViewport(id);
+      if (viewport?.isReferenceViewable(metadata, { withNavigation: true })) {
+        return id;
+      }
+    }
+
+    // No viewport is in the right display set/orientation to show this, so see if
+    // the active viewport could change orientations to show this
+    if (
+      activeViewport.isReferenceViewable(metadata, { withNavigation: true, withOrientation: true })
+    ) {
+      return activeViewportId;
+    }
+
+    // See if any viewport could show this with an orientation change
+    for (const id of this.viewportsById.keys()) {
+      const viewport = this.getCornerstoneViewport(id);
+      if (
+        viewport?.isReferenceViewable(metadata, { withNavigation: true, withOrientation: true })
+      ) {
+        return id;
+      }
+    }
+
+    // No luck, need to update the viewport itself
+    return null;
+  }
+
+  /**
+   * Sets the image data for the given viewport.
+   */
+  private async _setOtherViewport(
+    viewport: Types.IStackViewport,
+    viewportData: StackViewportData,
+    viewportInfo: ViewportInfo,
+    _presentations: Presentations = {}
+  ): Promise<void> {
+    const [displaySet] = viewportData.data;
+    return viewport.setDataIds(displaySet.imageIds, {
+      groupId: displaySet.displaySetInstanceUID,
+      viewReference: viewportInfo.getViewReference(),
+    });
   }
 
   private async _setStackViewport(
@@ -566,6 +612,9 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
 
     const { initialImageIndex, imageIds } = viewportData.data[0];
 
+    // Use the slice index from any provided view reference, as the view reference
+    // is being used to navigate to the initial view position for measurement
+    // navigation and other navigation forcing specific views.
     let initialImageIndexToUse =
       presentations?.positionPresentation?.initialImageIndex ?? initialImageIndex;
 
@@ -586,20 +635,15 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
         properties.voiRange = { lower, upper };
       }
 
-      if (voiInverted !== undefined) {
-        properties.invert = voiInverted;
-      }
-
-      if (colormap !== undefined) {
-        properties.colormap = colormap;
-      }
+      properties.invert = voiInverted ?? properties.invert;
+      properties.colormap = colormap ?? properties.colormap;
     }
 
     this._handleOverlays(viewport);
 
     return viewport.setStack(imageIds, initialImageIndexToUse).then(() => {
       viewport.setProperties({ ...properties });
-      this.setPresentations(viewport.id, presentations);
+      this.setPresentations(viewport.id, presentations, viewportInfo);
       if (displayArea) {
         viewport.setDisplayArea(displayArea);
       }
@@ -750,7 +794,6 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
     const displaySetUIDs = viewportGridService.getDisplaySetsUIDsForViewport(viewport.id);
     const displaySet = displaySetService.getDisplaySetByUID(displaySetUIDs[0]);
     const displaySetModality = displaySet?.Modality;
-
     // Todo: use presentations states
     const volumesProperties = volumeInputArray.map((volumeInput, index) => {
       const { volumeId } = volumeInput;
@@ -786,7 +829,7 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
       viewport.setProperties(properties, volumeId);
     });
 
-    this.setPresentations(viewport.id, presentations);
+    this.setPresentations(viewport.id, presentations, viewportInfo);
 
     this._handleOverlays(viewport);
 
@@ -976,7 +1019,12 @@ class CornerstoneViewportService extends PubSubService implements IViewportServi
       );
     }
 
-    throw new Error('Unknown viewport type');
+    return this._setOtherViewport(
+      viewport as Types.IViewport,
+      viewportData as StackViewportData,
+      viewportInfo,
+      presentations
+    );
   }
 
   /**

extensions/cornerstone/src/services/ViewportService/Viewport.ts

@@ -31,6 +31,9 @@ export type ViewportOptions = {
   displayArea?: Types.DisplayArea;
   syncGroups?: SyncGroup[];
   initialImageOptions?: InitialImageOptions;
+  rotation?: number;
+  flipHorizontal?: boolean;
+  viewReference?: Types.ViewReference;
   customViewportProps?: Record<string, unknown>;
   /*
    * Allows drag and drop of display sets not matching viewport options, but
@@ -49,6 +52,8 @@ export type PublicViewportOptions = {
   background?: Types.Point3;
   displayArea?: Types.DisplayArea;
   syncGroups?: SyncGroup[];
+  rotation?: number;
+  flipHorizontal?: boolean;
   initialImageOptions?: InitialImageOptions;
   customViewportProps?: Record<string, unknown>;
   allowUnmatchedView?: boolean;
@@ -130,6 +135,7 @@ class ViewportInfo {
   private displaySetOptions: Array<DisplaySetOptions>;
   private viewportData: StackViewportData | VolumeViewportData;
   private renderingEngineId: string;
+  private viewReference: Types.ViewReference;
 
   constructor(viewportId: string) {
     this.viewportId = viewportId;
@@ -201,6 +207,10 @@ class ViewportInfo {
     return this.viewportId;
   }
 
+  public getViewReference(): Types.ViewReference {
+    return this.viewportOptions?.viewReference;
+  }
+
   public setPublicDisplaySetOptions(
     publicDisplaySetOptions: PublicDisplaySetOptions[] | DisplaySetSelector[]
   ): Array<DisplaySetOptions> {
@@ -232,21 +242,22 @@ class ViewportInfo {
     return viewportData.data.displaySetInstanceUID === displaySetInstanceUID;
   }
 
-  public setPublicViewportOptions(viewportOptionsEntry: PublicViewportOptions): ViewportOptions {
-    let viewportType = viewportOptionsEntry.viewportType;
-    const { toolGroupId = DEFAULT_TOOLGROUP_ID, presentationIds } = viewportOptionsEntry;
-    let orientation;
+  /**
+   *
+   * @param viewportOptionsEntry - the base values for the options
+   * @param viewportTypeDisplaySet  - allows overriding the viewport type
+   */
+  public setPublicViewportOptions(
+    viewportOptionsEntry: PublicViewportOptions,
+    viewportTypeDisplaySet?: string
+  ): ViewportOptions {
+    const ohifViewportType = viewportTypeDisplaySet || viewportOptionsEntry.viewportType || STACK;
+    const { presentationIds } = viewportOptionsEntry;
+    let { toolGroupId = DEFAULT_TOOLGROUP_ID } = viewportOptionsEntry;
+    // Just assign the orientation for any viewport type and let the viewport deal with it
+    const orientation = getCornerstoneOrientation(viewportOptionsEntry.orientation);
 
-    if (!viewportType) {
-      viewportType = getCornerstoneViewportType(STACK);
-    } else {
-      viewportType = getCornerstoneViewportType(viewportOptionsEntry.viewportType);
-    }
-
-    // map SAGITTAL, AXIAL, CORONAL orientation to be used by cornerstone
-    if (viewportOptionsEntry.viewportType?.toLowerCase() !== STACK) {
-      orientation = getCornerstoneOrientation(viewportOptionsEntry.orientation);
-    }
+    const viewportType = getCornerstoneViewportType(ohifViewportType);
 
     if (!toolGroupId) {
       toolGroupId = DEFAULT_TOOLGROUP_ID;

extensions/cornerstone/src/types/CornerstoneCacheService.ts

@@ -9,7 +9,6 @@ type StackData = {
   isCompositeStack?: boolean;
   imageIds: string[];
   frameRate?: number;
-  isClip?: boolean;
   initialImageIndex?: number | string | null;
 };
 

extensions/cornerstone/src/types/Presentation.ts

@@ -7,13 +7,10 @@ import type { Types } from '@cornerstonejs/core';
 export type PositionPresentation = {
   id: string;
   viewportType: string;
-  presentation: {
-    initialImageIndex: number;
-    viewUp: Types.Point3;
-    viewPlaneNormal: Types.Point3;
-    zoom?: number;
-    pan?: Types.Point2;
-  };
+  // The view reference has the basic information as to what image orientation/slice is shown
+  viewReference: Types.ViewReference;
+  // The position information has the zoom/pan and possibly other related information, but not LUT
+  position: Types.ViewPresentation;
 };
 
 /**

extensions/cornerstone/src/utils/getCornerstoneViewportType.ts

@@ -1,16 +1,30 @@
+import type { Types } from '@ohif/core';
 import { Enums } from '@cornerstonejs/core';
 
 const STACK = 'stack';
 const VOLUME = 'volume';
 const ORTHOGRAPHIC = 'orthographic';
 const VOLUME_3D = 'volume3d';
+const VIDEO = 'video';
+const WHOLESLIDE = 'wholeslide';
 
-export default function getCornerstoneViewportType(viewportType: string): Enums.ViewportType {
-  const lowerViewportType = viewportType.toLowerCase();
+export default function getCornerstoneViewportType(
+  viewportType: string,
+  displaySets?: Types.DisplaySet[]
+): Enums.ViewportType {
+  const lowerViewportType =
+    displaySets?.[0]?.viewportType?.toLowerCase() || viewportType.toLowerCase();
   if (lowerViewportType === STACK) {
     return Enums.ViewportType.STACK;
   }
 
+  if (lowerViewportType === VIDEO) {
+    return Enums.ViewportType.VIDEO;
+  }
+  if (lowerViewportType === WHOLESLIDE) {
+    return Enums.ViewportType.WholeSlide;
+  }
+
   if (lowerViewportType === VOLUME || lowerViewportType === ORTHOGRAPHIC) {
     return Enums.ViewportType.ORTHOGRAPHIC;
   }
@@ -19,5 +33,7 @@ export default function getCornerstoneViewportType(viewportType: string): Enums.
     return Enums.ViewportType.VOLUME_3D;
   }
 
-  throw new Error(`Invalid viewport type: ${viewportType}. Valid types are: stack, volume`);
+  throw new Error(
+    `Invalid viewport type: ${viewportType}. Valid types are: stack, volume, video, wholeslide`
+  );
 }

extensions/cornerstone/src/utils/measurementServiceMappings/Angle.ts

@@ -168,7 +168,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { angle, unit, SeriesNumber, SOPInstanceUID, frameNumber } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/ArrowAnnotate.ts

@@ -117,7 +117,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { SeriesNumber, SOPInstanceUID, frameNumber } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/Bidirectional.ts

@@ -165,7 +165,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   const roundedLength = utils.roundNumber(length, 2);
   const roundedWidth = utils.roundNumber(width, 2);
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/CircleROI.ts

@@ -171,7 +171,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { area, SOPInstanceUID, frameNumber, areaUnit } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/CobbAngle.ts

@@ -168,7 +168,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { angle, unit, SeriesNumber, SOPInstanceUID, frameNumber } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/EllipticalROI.ts

@@ -171,7 +171,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { area, SOPInstanceUID, frameNumber, areaUnit } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/Length.ts

@@ -168,7 +168,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { length, SeriesNumber, SOPInstanceUID, frameNumber, unit } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/LivewireContour.ts

@@ -132,7 +132,7 @@ function getDisplayText(annotation, displaySet, customizationService) {
 
   const displayText = [];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
   let InstanceNumber;
   if (instance) {
     InstanceNumber = instance.InstanceNumber;

extensions/cornerstone/src/utils/measurementServiceMappings/PlanarFreehandROI.ts

@@ -139,7 +139,7 @@ function getDisplayText(annotation, displaySet, customizationService, displaySet
 
   const displayTextArray = [];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
   let InstanceNumber;
   if (instance) {
     InstanceNumber = instance.InstanceNumber;

extensions/cornerstone/src/utils/measurementServiceMappings/Probe.ts

@@ -163,7 +163,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
 
   const { value, unit, SeriesNumber, SOPInstanceUID, frameNumber } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/RectangleROI.ts

@@ -172,7 +172,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   // Area is the same for all series
   const { area, SOPInstanceUID, frameNumber, areaUnit } = mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {

extensions/cornerstone/src/utils/measurementServiceMappings/SplineROI.ts

@@ -138,7 +138,7 @@ function getDisplayText(annotation, displaySet, customizationService) {
 
   const displayTextArray = [];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
   let InstanceNumber;
   if (instance) {
     InstanceNumber = instance.InstanceNumber;

extensions/cornerstone/src/utils/measurementServiceMappings/UltrasoundDirectional.ts

@@ -176,7 +176,7 @@ function getDisplayText(mappedAnnotations, displaySet, customizationService) {
   const { xValues, yValues, units, isUnitless, SeriesNumber, SOPInstanceUID, frameNumber } =
     mappedAnnotations[0];
 
-  const instance = displaySet.images.find(image => image.SOPInstanceUID === SOPInstanceUID);
+  const instance = displaySet.instances.find(image => image.SOPInstanceUID === SOPInstanceUID);
 
   let InstanceNumber;
   if (instance) {