feat(thumbnails): Enhance DICOM Thumbnail Rendering with different strategies (#5004)

Co-authored-by: Martin Bellehumeur, M. Eng. <23396581+mbellehumeur@users.noreply.github.com>

extensions/default/src/DicomWebDataSource/index.ts

@@ -37,7 +37,15 @@ export type DicomWebConfig = {
   wadoUri?: string; // - Base URL to use for WADO URI requests
   qidoSupportsIncludeField?: boolean; // - Whether QIDO supports the "Include" option to request additional fields in response
   imageRendering?: string; // - wadors | ? (unsure of where/how this is used)
-  thumbnailRendering?: string; // - wadors | ? (unsure of where/how this is used)
+  thumbnailRendering?: string;
+  /**
+   wadors - render using the wadors fetch.  The full image is retrieved and rendered in cornerstone to thumbnail size  png and returned as binary data to the src attribute of the  image tag.
+           for example,  <img  src=data:image/png;base64,sdlfk;adkfadfk....asldfjkl;asdkf>
+   thumbnailDirect -  get the direct url endpoint for the thumbnail as the image src (eg not authentication required).
+           for example, <img src=http://server:port/wadors/studies/1.2.3/thumbnail?accept=image/jpeg>
+   thumbnail - render using the thumbnail endpoint on wadors using bulkDataURI, passing authentication params  to the url.
+    rendered - should use the rendered endpoint instead of the thumbnail endpoint
+*/
   /** Whether the server supports reject calls (i.e. DCM4CHEE) */
   supportsReject?: boolean;
   /** Request series meta async instead of blocking */
@@ -225,6 +233,71 @@ function createDicomWebApi(dicomWebConfig: DicomWebConfig, servicesManager) {
        * @returns an absolute URL to the resource, if the absolute URL can be retrieved as singlepart,
        *    or is already retrieved, or a promise to a URL for such use if a BulkDataURI
        */
+
+      getGetThumbnailSrc: function (instance, imageId) {
+        if (dicomWebConfig.thumbnailRendering === 'wadors') {
+          return function getThumbnailSrc(options) {
+            if (!imageId) {
+              return null;
+            }
+            if (!options?.getImageSrc) {
+              return null;
+            }
+            return options.getImageSrc(imageId);
+          };
+        }
+        if (dicomWebConfig.thumbnailRendering === 'thumbnailDirect') {
+          return function getThumbnailSrc() {
+            return this.directURL({
+              instance: instance,
+              defaultPath: '/thumbnail',
+              defaultType: 'image/jpeg',
+              singlepart: true,
+              tag: 'Absent',
+            });
+          }.bind(this);
+        }
+
+        if (dicomWebConfig.thumbnailRendering === 'thumbnail') {
+          return async function getThumbnailSrc() {
+            const { StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID } = instance;
+            const bulkDataURI = `${dicomWebConfig.wadoRoot}/studies/${StudyInstanceUID}/series/${SeriesInstanceUID}/instances/${SOPInstanceUID}/thumbnail?accept=image/jpeg`;
+            return URL.createObjectURL(
+              new Blob(
+                [
+                  await this.bulkDataURI({
+                    BulkDataURI: bulkDataURI.replace('wadors:', ''),
+                    defaultType: 'image/jpeg',
+                    mediaTypes: ['image/jpeg'],
+                    thumbnail: true,
+                  }),
+                ],
+                { type: 'image/jpeg' }
+              )
+            );
+          }.bind(this);
+        }
+        if (dicomWebConfig.thumbnailRendering === 'rendered') {
+          return async function getThumbnailSrc() {
+            const { StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID } = instance;
+            const bulkDataURI = `${dicomWebConfig.wadoRoot}/studies/${StudyInstanceUID}/series/${SeriesInstanceUID}/instances/${SOPInstanceUID}/rendered?accept=image/jpeg`;
+            return URL.createObjectURL(
+              new Blob(
+                [
+                  await this.bulkDataURI({
+                    BulkDataURI: bulkDataURI.replace('wadors:', ''),
+                    defaultType: 'image/jpeg',
+                    mediaTypes: ['image/jpeg'],
+                    thumbnail: true,
+                  }),
+                ],
+                { type: 'image/jpeg' }
+              )
+            );
+          }.bind(this);
+        }
+      },
+
       directURL: params => {
         return getDirectURL(
           {

extensions/default/src/Panels/StudyBrowser/PanelStudyBrowser.tsx

@@ -10,17 +10,7 @@ import { CallbackCustomization } from 'platform/core/src/types';
 
 const { sortStudyInstances, formatDate, createStudyBrowserTabs } = utils;
 
-const thumbnailNoImageModalities = [
-  'SR',
-  'SEG',
-  'SM',
-  'RTSTRUCT',
-  'RTPLAN',
-  'RTDOSE',
-  'DOC',
-  'OT',
-  'PMAP',
-];
+const thumbnailNoImageModalities = ['SR', 'SEG', 'RTSTRUCT', 'RTPLAN', 'RTDOSE', 'DOC', 'PMAP'];
 
 /**
  * Study Browser component that displays and manages studies and their display sets
@@ -178,7 +168,7 @@ function PanelStudyBrowser({
     let currentDisplaySets = displaySetService.activeDisplaySets;
     // filter non based on the list of modalities that are supported by cornerstone
     currentDisplaySets = currentDisplaySets.filter(
-      ds => !thumbnailNoImageModalities.includes(ds.Modality)
+      ds => !thumbnailNoImageModalities.includes(ds.Modality) || ds.thumbnailSrc === null
     );
 
     if (!currentDisplaySets.length) {
@@ -188,20 +178,20 @@ function PanelStudyBrowser({
     currentDisplaySets.forEach(async dSet => {
       const newImageSrcEntry = {};
       const displaySet = displaySetService.getDisplaySetByUID(dSet.displaySetInstanceUID);
-      const imageIds = dataSource.getImageIdsForDisplaySet(displaySet);
+      const imageIds = dataSource.getImageIdsForDisplaySet(dSet);
 
       const imageId = getImageIdForThumbnail(displaySet, imageIds);
 
       // TODO: Is it okay that imageIds are not returned here for SR displaySets?
-      if (!imageId || displaySet?.unsupported) {
+      if (displaySet?.unsupported) {
         return;
       }
       // When the image arrives, render it and store the result in the thumbnailImgSrcMap
       let { thumbnailSrc } = displaySet;
       if (!thumbnailSrc && displaySet.getThumbnailSrc) {
-        thumbnailSrc = await displaySet.getThumbnailSrc();
+        thumbnailSrc = await displaySet.getThumbnailSrc({ getImageSrc });
       }
-      if (!thumbnailSrc) {
+      if (!thumbnailSrc && imageId) {
         const thumbnailSrc = await getImageSrc(imageId);
         displaySet.thumbnailSrc = thumbnailSrc;
       }
@@ -253,13 +243,11 @@ function PanelStudyBrowser({
         const { displaySetsAdded, options } = data;
         displaySetsAdded.forEach(async dSet => {
           const displaySetInstanceUID = dSet.displaySetInstanceUID;
-
           const newImageSrcEntry = {};
           const displaySet = displaySetService.getDisplaySetByUID(displaySetInstanceUID);
           if (displaySet?.unsupported) {
             return;
           }
-
           if (options?.madeInClient) {
             setJumpToDisplaySet(displaySetInstanceUID);
           }
@@ -275,7 +263,7 @@ function PanelStudyBrowser({
           // When the image arrives, render it and store the result in the thumbnailImgSrcMap
           let { thumbnailSrc } = displaySet;
           if (!thumbnailSrc && displaySet.getThumbnailSrc) {
-            thumbnailSrc = await displaySet.getThumbnailSrc();
+            thumbnailSrc = await displaySet.getThumbnailSrc({ getImageSrc });
           }
           if (!thumbnailSrc) {
             thumbnailSrc = await getImageSrc(imageId);
@@ -512,7 +500,11 @@ function _mapDisplaySets(displaySets, displaySetLoadingState, thumbnailImageSrcM
 }
 
 function _getComponentType(ds) {
-  if (thumbnailNoImageModalities.includes(ds.Modality) || ds?.unsupported) {
+  if (
+    thumbnailNoImageModalities.includes(ds.Modality) ||
+    ds?.unsupported ||
+    ds.thumbnailSrc === null
+  ) {
     return 'thumbnailNoImage';
   }
 

extensions/default/src/getSopClassHandlerModule.js

@@ -5,7 +5,13 @@ import getDisplaySetMessages from './getDisplaySetMessages';
 import getDisplaySetsFromUnsupportedSeries from './getDisplaySetsFromUnsupportedSeries';
 import { chartHandler } from './SOPClassHandlers/chartSOPClassHandler';
 
-const { isImage, sopClassDictionary, isDisplaySetReconstructable } = utils;
+const {
+  isImage,
+  sortStudyInstances,
+  instancesSortCriteria,
+  sopClassDictionary,
+  isDisplaySetReconstructable,
+} = utils;
 const { ImageSet } = classes;
 
 const DEFAULT_VOLUME_LOADER_SCHEME = 'cornerstoneStreamingImageVolume';
@@ -62,9 +68,12 @@ function getDisplaySetInfo(instances) {
 }
 
 const makeDisplaySet = instances => {
+  // Need to sort the instances in order to get a consistent instance/thumbnail
+  sortStudyInstances(instances);
   const instance = instances[0];
   const imageSet = new ImageSet(instances);
-
+  const { extensionManager } = appContext;
+  const dataSource = extensionManager.getActiveDataSource()[0];
   const {
     isDynamicVolume,
     value: isReconstructable,
@@ -79,6 +88,16 @@ const makeDisplaySet = instances => {
   // set appropriate attributes to image set...
   const messages = getDisplaySetMessages(instances, isReconstructable, isDynamicVolume);
 
+  const imageIds = dataSource.getImageIdsForDisplaySet(imageSet);
+  let imageId = imageIds[Math.floor(imageIds.length / 2)];
+  let thumbnailInstance = instances[Math.floor(instances.length / 2)];
+  if (isDynamicVolume) {
+    const timePoints = dynamicVolumeInfo.timePoints;
+    const middleIndex = Math.floor(timePoints.length / 2);
+    const middleTimePointImageIds = timePoints[middleIndex];
+    imageId = middleTimePointImageIds[Math.floor(middleTimePointImageIds.length / 2)];
+  }
+
   imageSet.setAttributes({
     volumeLoaderSchema,
     displaySetInstanceUID: imageSet.uid, // create a local alias for the imageSet UID
@@ -100,6 +119,7 @@ const makeDisplaySet = instances => {
     averageSpacingBetweenFrames: averageSpacingBetweenFrames || null,
     isDynamicVolume,
     dynamicVolumeInfo,
+    getThumbnailSrc: dataSource.retrieve.getGetThumbnailSrc?.(thumbnailInstance, imageId),
     supportsWindowLevel: true,
     label:
       instance.SeriesDescription ||
@@ -107,14 +127,7 @@ const makeDisplaySet = instances => {
     FrameOfReferenceUID: instance.FrameOfReferenceUID,
   });
 
-  // Sort the images in this series if needed
-  const shallSort = true; //!OHIF.utils.ObjectPath.get(Meteor, 'settings.public.ui.sortSeriesByIncomingOrder');
-  if (shallSort) {
-    imageSet.sortBy((a, b) => {
-      // Sort by InstanceNumber (0020,0013)
-      return (parseInt(a.InstanceNumber) || 0) - (parseInt(b.InstanceNumber) || 0);
-    });
-  }
+  imageSet.sortBy(instancesSortCriteria.default);
 
   // Include the first image instance number (after sorted)
   /*imageSet.setAttribute(
@@ -178,10 +191,8 @@ function getDisplaySetsFromSeries(instances) {
     }
 
     let displaySet;
-
     if (isMultiFrame(instance)) {
       displaySet = makeDisplaySet([instance]);
-
       displaySet.setAttributes({
         sopClassUids,
         numImageFrames: instance.NumberOfFrames,