From 29fceacee97d51f1952a0f6b574c66596d32c201 Mon Sep 17 00:00:00 2001 From: Davide Punzo Date: Thu, 10 Dec 2020 11:04:27 +0100 Subject: [PATCH] feat: visualize overlapping segments in cornerstone (#2185) * feat: visualize overlapping segments load multiple labelMaps (splitted by dcmjs because overlapping) in a segmentations [IDC2164] Dynamically update active labelmap [IDC2164] Co-authored-by: Davide Punzo Co-authored-by: Igor Octaviano --- extensions/cornerstone/package.json | 2 +- extensions/dicom-html/package.json | 2 +- extensions/dicom-rt/package.json | 2 +- extensions/dicom-segmentation/package.json | 2 +- .../SegmentationPanel/SegmentationPanel.js | 32 +++- .../src/getOHIFDicomSegSopClassHandler.js | 84 +++++++++- .../src/loadSegmentation.js | 157 +++++------------- extensions/dicom-tag-browser/package.json | 2 +- extensions/vtk/package.json | 2 +- extensions/vtk/src/OHIFVTKViewport.js | 13 +- platform/core/package.json | 2 +- .../src/contextProviders/SnackbarProvider.js | 4 + platform/viewer/package.json | 2 +- .../components/ViewportGrid/ViewportGrid.js | 3 +- yarn.lock | 9 +- 15 files changed, 182 insertions(+), 136 deletions(-) diff --git a/extensions/cornerstone/package.json b/extensions/cornerstone/package.json index 0eee700c8..023a58387 100644 --- a/extensions/cornerstone/package.json +++ b/extensions/cornerstone/package.json @@ -36,7 +36,7 @@ "cornerstone-math": "^0.1.8", "cornerstone-tools": "^4.20.1", "cornerstone-wado-image-loader": "^3.1.0", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "dicom-parser": "^1.8.3", "hammerjs": "^2.0.8", "prop-types": "^15.6.2", diff --git a/extensions/dicom-html/package.json b/extensions/dicom-html/package.json index 318d4add9..6143a47f4 100644 --- a/extensions/dicom-html/package.json +++ b/extensions/dicom-html/package.json @@ -29,7 +29,7 @@ }, "peerDependencies": { "@ohif/core": "^0.50.0", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "prop-types": "^15.6.2", "react": "^16.8.6", "react-dom": "^16.8.6" diff --git a/extensions/dicom-rt/package.json b/extensions/dicom-rt/package.json index 53603aba0..d1a5f5e42 100644 --- a/extensions/dicom-rt/package.json +++ b/extensions/dicom-rt/package.json @@ -31,7 +31,7 @@ "@ohif/core": "^0.50.0", "cornerstone-core": "^2.2.8", "cornerstone-tools": "^4.20.1", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "gl-matrix": "^3.3.0", "prop-types": "^15.6.2", "react": "^16.8.6", diff --git a/extensions/dicom-segmentation/package.json b/extensions/dicom-segmentation/package.json index 6b872fbef..3a35a893b 100644 --- a/extensions/dicom-segmentation/package.json +++ b/extensions/dicom-segmentation/package.json @@ -31,7 +31,7 @@ "@ohif/core": "^0.50.0", "cornerstone-core": "^2.2.8", "cornerstone-tools": "^4.20.1", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "prop-types": "^15.6.2", "react": "^16.8.6", "react-dom": "^16.8.6" diff --git a/extensions/dicom-segmentation/src/components/SegmentationPanel/SegmentationPanel.js b/extensions/dicom-segmentation/src/components/SegmentationPanel/SegmentationPanel.js index 7fb9d6bc9..1cd292442 100644 --- a/extensions/dicom-segmentation/src/components/SegmentationPanel/SegmentationPanel.js +++ b/extensions/dicom-segmentation/src/components/SegmentationPanel/SegmentationPanel.js @@ -159,12 +159,12 @@ const SegmentationPanel = ({ const labelmapList = getLabelmapList( brushStackState, firstImageId, - activeViewport + activeViewport, ); const segmentList = getSegmentList( labelmap3D, firstImageId, - brushStackState + brushStackState, ); setState(state => ({ ...state, @@ -292,10 +292,13 @@ const SegmentationPanel = ({ const sameSegment = state.selectedSegment === segmentNumber; const setCurrentSelectedSegment = () => { + const activeViewport = viewports[activeIndex]; + _setActiveSegment( firstImageId, segmentNumber, - labelmap3D.activeSegmentIndex + labelmap3D.activeSegmentIndex, + activeViewport ); updateState('selectedSegment', sameSegment ? null : segmentNumber); @@ -640,7 +643,7 @@ const _getReferencedSegDisplaysets = (StudyInstanceUID, SeriesInstanceUID) => { * @param {*} activeSegmentIndex * @returns */ -const _setActiveSegment = (firstImageId, segmentIndex, activeSegmentIndex) => { +const _setActiveSegment = (firstImageId, segmentIndex, activeSegmentIndex, activeViewport) => { if (segmentIndex === activeSegmentIndex) { log.info(`${activeSegmentIndex} is already the active segment`); return; @@ -653,8 +656,27 @@ const _setActiveSegment = (firstImageId, segmentIndex, activeSegmentIndex) => { brushStackState.labelmaps3D[brushStackState.activeLabelmapIndex]; labelmap3D.activeSegmentIndex = segmentIndex; - refreshViewports(); + /** + * Activates the correct label map if clicked segment + * does not belong to the active labelmap + */ + const { StudyInstanceUID } = activeViewport; + const studyMetadata = studyMetadataManager.get(StudyInstanceUID); + const allDisplaySets = studyMetadata.getDisplaySets(); + let newLabelmapIndex = brushStackState.activeLabelmapIndex; + allDisplaySets.forEach(displaySet => { + if (displaySet.labelmapSegments) { + Object.keys(displaySet.labelmapSegments).forEach(labelmapIndex => { + if (displaySet.labelmapSegments[labelmapIndex].includes(segmentIndex)) { + newLabelmapIndex = labelmapIndex; + } + }); + } + }); + brushStackState.activeLabelmapIndex = newLabelmapIndex; + + refreshViewports(); return segmentIndex; }; diff --git a/extensions/dicom-segmentation/src/getOHIFDicomSegSopClassHandler.js b/extensions/dicom-segmentation/src/getOHIFDicomSegSopClassHandler.js index 2a3c5ad30..f21c080f4 100644 --- a/extensions/dicom-segmentation/src/getOHIFDicomSegSopClassHandler.js +++ b/extensions/dicom-segmentation/src/getOHIFDicomSegSopClassHandler.js @@ -1,6 +1,11 @@ import { MODULE_TYPES, utils } from '@ohif/core'; import loadSegmentation from './loadSegmentation'; import getSourceDisplaySet from './getSourceDisplaySet'; +import OHIF from '@ohif/core'; +import dcmjs from 'dcmjs'; + +const { DicomLoaderService } = OHIF.utils; +const { DicomMessage, DicomMetaDictionary } = dcmjs.data; // TODO: Should probably use dcmjs for this const SOP_CLASS_UIDS = { @@ -16,7 +21,7 @@ export default function getSopClassHandlerModule({ servicesManager }) { id: 'OHIFDicomSegSopClassHandler', type: MODULE_TYPES.SOP_CLASS_HANDLER, sopClassUIDs, - getDisplaySetFromSeries: function( + getDisplaySetFromSeries: function ( series, study, dicomWebClient, @@ -46,7 +51,6 @@ export default function getSopClassHandlerModule({ servicesManager }) { StudyInstanceUID, FrameOfReferenceUID, authorizationHeaders, - metadata, isDerived: true, referencedDisplaySetUID: null, // Assigned when loaded. labelmapIndex: null, // Assigned when loaded. @@ -58,15 +62,85 @@ export default function getSopClassHandlerModule({ servicesManager }) { metadata, }; - segDisplaySet.getSourceDisplaySet = function(studies) { + segDisplaySet.getSourceDisplaySet = function (studies) { return getSourceDisplaySet(studies, segDisplaySet); }; - segDisplaySet.load = function(referencedDisplaySet, studies) { - return loadSegmentation(segDisplaySet, referencedDisplaySet, studies); + segDisplaySet.load = async function (referencedDisplaySet, studies) { + segDisplaySet.isLoaded = true; + const { StudyInstanceUID } = referencedDisplaySet; + const segArrayBuffer = await DicomLoaderService.findDicomDataPromise( + segDisplaySet, + studies + ); + const dicomData = DicomMessage.readFile(segArrayBuffer); + const dataset = DicomMetaDictionary.naturalizeDataset(dicomData.dict); + dataset._meta = DicomMetaDictionary.namifyDataset(dicomData.meta); + const imageIds = _getImageIdsForDisplaySet( + studies, + StudyInstanceUID, + referencedDisplaySet.SeriesInstanceUID + ); + return new Promise((resolve, reject) => { + let results; + try { + results = _parseSeg(segArrayBuffer, imageIds); + } catch (error) { + segDisplaySet.isLoaded = false; + segDisplaySet.loadError = true; + reject(error); + } + const { labelmapBufferArray, segMetadata, segmentsOnFrame, segmentsOnFrameArray } = results; + + if (labelmapBufferArray.length > 1) { + for (let i = 0; i < labelmapBufferArray.length; ++i) { + loadSegmentation(imageIds, segDisplaySet, labelmapBufferArray[i], segMetadata, segmentsOnFrame, segmentsOnFrameArray[i]); + } + } else { + loadSegmentation(imageIds, segDisplaySet, labelmapBufferArray[0], segMetadata, segmentsOnFrame, []); + } + segDisplaySet.labelmapIndex = 0; + + resolve(); + }); }; return segDisplaySet; }, }; } + +function _parseSeg(arrayBuffer, imageIds) { + return dcmjs.adapters.Cornerstone.Segmentation.generateToolState( + imageIds, + arrayBuffer, + cornerstone.metaData + ); +} + +function _getImageIdsForDisplaySet( + studies, + StudyInstanceUID, + SeriesInstanceUID +) { + const study = studies.find( + study => study.StudyInstanceUID === StudyInstanceUID + ); + + const displaySets = study.displaySets.filter(displaySet => { + return displaySet.SeriesInstanceUID === SeriesInstanceUID; + }); + + if (displaySets.length > 1) { + console.warn( + 'More than one display set with the same SeriesInstanceUID. This is not supported yet...' + ); + // TODO -> We could make check the instance list and see if any match? + // Do we split the segmentation into two cornerstoneTools segmentations if there are images in both series? + // ^ Will that even happen? + } + + const referencedDisplaySet = displaySets[0]; + + return referencedDisplaySet.images.map(image => image.getImageId()); +} diff --git a/extensions/dicom-segmentation/src/loadSegmentation.js b/extensions/dicom-segmentation/src/loadSegmentation.js index 052126180..beb4f80ba 100644 --- a/extensions/dicom-segmentation/src/loadSegmentation.js +++ b/extensions/dicom-segmentation/src/loadSegmentation.js @@ -1,92 +1,62 @@ -import OHIF from '@ohif/core'; import dcmjs from 'dcmjs'; import cornerstone from 'cornerstone-core'; import cornerstoneTools from 'cornerstone-tools'; -const { DicomLoaderService } = OHIF.utils; - export default async function loadSegmentation( + imageIds, segDisplaySet, - referencedDisplaySet, - studies + labelmapBuffer, + segMetadata, + segmentsOnFrame, + labelmapSegments ) { - const { StudyInstanceUID } = referencedDisplaySet; + const { setters } = cornerstoneTools.getModule('segmentation'); - // Set here is loading is asynchronous. - // If this function throws its set back to false. - segDisplaySet.isLoaded = true; + // TODO: Could define a color LUT based on colors in the SEG. + const labelmapIndex = _getNextLabelmapIndex(imageIds[0]); + const colorLUTIndex = _makeColorLUTAndGetIndex(segMetadata); - const segArrayBuffer = await DicomLoaderService.findDicomDataPromise( - segDisplaySet, - studies + setters.labelmap3DByFirstImageId( + imageIds[0], + labelmapBuffer, + labelmapIndex, + segMetadata, + imageIds.length, + segmentsOnFrame, + colorLUTIndex ); - const dicomData = dcmjs.data.DicomMessage.readFile(segArrayBuffer); - const dataset = dcmjs.data.DicomMetaDictionary.naturalizeDataset( - dicomData.dict - ); + if (!segDisplaySet.labelmapSegments) { + segDisplaySet.labelmapSegments = {}; + } - dataset._meta = dcmjs.data.DicomMetaDictionary.namifyDataset(dicomData.meta); + /** + * Cache each labelmap segments. + * This data is used to determine the active label map when a given segment is activated/clicked. + */ + segDisplaySet.labelmapSegments[labelmapIndex] = labelmapSegments.length ? Array.from(new Set(labelmapSegments.filter(a => !!a).reduce((a, b) => a.concat(b)))) : []; + segDisplaySet.labelmapIndex = labelmapIndex; - const imageIds = _getImageIdsForDisplaySet( - studies, - StudyInstanceUID, - referencedDisplaySet.SeriesInstanceUID - ); - - return new Promise((resolve, reject) => { - let results; - - try { - results = _parseSeg(segArrayBuffer, imageIds); - } catch (error) { - segDisplaySet.isLoaded = false; - segDisplaySet.loadError = true; - reject(error); - } - - const { labelmapBuffer, segMetadata, segmentsOnFrame } = results; - const { setters } = cornerstoneTools.getModule('segmentation'); - - // TODO: Could define a color LUT based on colors in the SEG. - const labelmapIndex = _getNextLabelmapIndex(imageIds[0]); - const colorLUTIndex = _makeColorLUTAndGetIndex(segMetadata); - - setters.labelmap3DByFirstImageId( - imageIds[0], - labelmapBuffer, - labelmapIndex, - segMetadata, - imageIds.length, - segmentsOnFrame, - colorLUTIndex - ); - - segDisplaySet.labelmapIndex = labelmapIndex; - - /* - * TODO: Improve the way we notify parts of the app that depends on segs to be loaded. - * - * Currently we are using a non-ideal implementation through a custom event to notify the segmentation panel - * or other components that could rely on loaded segmentations that - * the segments were loaded so that e.g. when the user opens the panel - * before the segments are fully loaded, the panel can subscribe to this custom event - * and update itself with the new segments. - * - * This limitation is due to the fact that the cs segmentation module is an object (which will be - * updated after the segments are loaded) that React its not aware of its changes - * because the module object its not passed in to the panel component as prop but accessed externally. - * - * Improving this event approach to something reactive that can be tracked inside the react lifecycle, - * allows us to easily watch the module or the segmentations loading process in any other component - * without subscribing to external events. - */ - console.log('Segmentation loaded.'); - const event = new CustomEvent('extensiondicomsegmentationsegloaded'); - document.dispatchEvent(event); - - resolve(labelmapIndex); - }); + /* + * TODO: Improve the way we notify parts of the app that depends on segs to be loaded. + * + * Currently we are using a non-ideal implementation through a custom event to notify the segmentation panel + * or other components that could rely on loaded segmentations that + * the segments were loaded so that e.g. when the user opens the panel + * before the segments are fully loaded, the panel can subscribe to this custom event + * and update itself with the new segments. + * + * This limitation is due to the fact that the cs segmentation module is an object (which will be + * updated after the segments are loaded) that React its not aware of its changes + * because the module object its not passed in to the panel component as prop but accessed externally. + * + * Improving this event approach to something reactive that can be tracked inside the react lifecycle, + * allows us to easily watch the module or the segmentations loading process in any other component + * without subscribing to external events. + */ + console.log('Segmentation loaded.'); + const event = new CustomEvent('extensiondicomsegmentationsegloaded'); + document.dispatchEvent(event); } function _getNextLabelmapIndex(firstImageId) { @@ -172,38 +142,3 @@ function _getNextColorLUTIndex() { return colorLUTIndex; } - -function _parseSeg(arrayBuffer, imageIds) { - return dcmjs.adapters.Cornerstone.Segmentation.generateToolState( - imageIds, - arrayBuffer, - cornerstone.metaData - ); -} - -function _getImageIdsForDisplaySet( - studies, - StudyInstanceUID, - SeriesInstanceUID -) { - const study = studies.find( - study => study.StudyInstanceUID === StudyInstanceUID - ); - - const displaySets = study.displaySets.filter(displaySet => { - return displaySet.SeriesInstanceUID === SeriesInstanceUID; - }); - - if (displaySets.length > 1) { - console.warn( - 'More than one display set with the same SeriesInstanceUID. This is not supported yet...' - ); - // TODO -> We could make check the instance list and see if any match? - // Do we split the segmentation into two cornerstoneTools segmentations if there are images in both series? - // ^ Will that even happen? - } - - const referencedDisplaySet = displaySets[0]; - - return referencedDisplaySet.images.map(image => image.getImageId()); -} diff --git a/extensions/dicom-tag-browser/package.json b/extensions/dicom-tag-browser/package.json index 173762552..7658fac67 100644 --- a/extensions/dicom-tag-browser/package.json +++ b/extensions/dicom-tag-browser/package.json @@ -28,7 +28,7 @@ }, "peerDependencies": { "@ohif/core": "^2.6.0", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "react": "^16.8.6" }, "dependencies": { diff --git a/extensions/vtk/package.json b/extensions/vtk/package.json index 2902f8d70..6d0ad1eff 100644 --- a/extensions/vtk/package.json +++ b/extensions/vtk/package.json @@ -35,7 +35,7 @@ "cornerstone-core": "^2.2.8", "cornerstone-tools": "^4.20.1", "cornerstone-wado-image-loader": "^3.1.0", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "dicom-parser": "^1.8.3", "i18next": "^17.0.3", "i18next-browser-languagedetector": "^3.0.1", diff --git a/extensions/vtk/src/OHIFVTKViewport.js b/extensions/vtk/src/OHIFVTKViewport.js index 852dcf27f..19e557143 100644 --- a/extensions/vtk/src/OHIFVTKViewport.js +++ b/extensions/vtk/src/OHIFVTKViewport.js @@ -72,7 +72,7 @@ class OHIFVTKViewport extends Component { }; static defaultProps = { - onScroll: () => {}, + onScroll: () => { }, }; static id = 'OHIFVTKViewport'; @@ -159,6 +159,15 @@ class OHIFVTKViewport extends Component { const { activeLabelmapIndex } = brushStackState; const labelmap3D = brushStackState.labelmaps3D[activeLabelmapIndex]; + if (brushStackState.labelmaps3D.length > 1 && this.props.viewportIndex === 0) { + const { UINotificationService } = this.props.servicesManager.services; + UINotificationService.show({ + title: 'Overlapping Segmentation Found', + message: 'Overlapping segmentations cannot be displayed when in MPR mode', + type: 'info', + }); + } + this.segmentsDefaultProperties = labelmap3D.segmentsHidden.map( isHidden => { return { visible: !isHidden }; @@ -354,7 +363,7 @@ class OHIFVTKViewport extends Component { if ( displaySet.displaySetInstanceUID !== - prevDisplaySet.displaySetInstanceUID || + prevDisplaySet.displaySetInstanceUID || displaySet.SOPInstanceUID !== prevDisplaySet.SOPInstanceUID || displaySet.frameIndex !== prevDisplaySet.frameIndex ) { diff --git a/platform/core/package.json b/platform/core/package.json index b9a8fcc52..64cd15afd 100644 --- a/platform/core/package.json +++ b/platform/core/package.json @@ -39,7 +39,7 @@ "dependencies": { "@babel/runtime": "^7.5.5", "ajv": "^6.10.0", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "dicomweb-client": "^0.6.0", "immer": "6.0.2", "isomorphic-base64": "^1.0.2", diff --git a/platform/ui/src/contextProviders/SnackbarProvider.js b/platform/ui/src/contextProviders/SnackbarProvider.js index e3c7cb363..4b4c6bcbd 100644 --- a/platform/ui/src/contextProviders/SnackbarProvider.js +++ b/platform/ui/src/contextProviders/SnackbarProvider.js @@ -37,6 +37,10 @@ const SnackbarProvider = ({ children, service }) => { return null; } + if (options.type === 'error') { + console.error(options.error); + } + const newItem = { ...DEFAULT_OPTIONS, ...options, diff --git a/platform/viewer/package.json b/platform/viewer/package.json index baf80c8ec..e189ee599 100644 --- a/platform/viewer/package.json +++ b/platform/viewer/package.json @@ -68,7 +68,7 @@ "cornerstone-math": "^0.1.8", "cornerstone-tools": "^4.20.1", "cornerstone-wado-image-loader": "^3.1.0", - "dcmjs": "0.16.5", + "dcmjs": "0.17.0", "dicom-parser": "^1.8.3", "dicomweb-client": "^0.4.4", "hammerjs": "^2.0.8", diff --git a/platform/viewer/src/components/ViewportGrid/ViewportGrid.js b/platform/viewer/src/components/ViewportGrid/ViewportGrid.js index 7f0449403..862c251fb 100644 --- a/platform/viewer/src/components/ViewportGrid/ViewportGrid.js +++ b/platform/viewer/src/components/ViewportGrid/ViewportGrid.js @@ -12,7 +12,7 @@ import EmptyViewport from './EmptyViewport.js'; const { loadAndCacheDerivedDisplaySets } = utils; -const ViewportGrid = function(props) { +const ViewportGrid = function (props) { const { activeViewportIndex, availablePlugins, @@ -48,6 +48,7 @@ const ViewportGrid = function(props) { title: 'Error loading derived display set:', message: error.message, type: 'error', + error, autoClose: false, }); }); diff --git a/yarn.lock b/yarn.lock index e6ab2188e..b260f856a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6619,14 +6619,15 @@ dateformat@^3.0.0: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q== -dcmjs@0.16.5: - version "0.16.5" - resolved "https://registry.yarnpkg.com/dcmjs/-/dcmjs-0.16.5.tgz#05aa0c8d539a4d5777f996b9b796932d1f848993" - integrity sha512-Y92KAx4JSfev79xjYhaukQZjWgyJlaHnXSTdbtkGkJAHERjIakgHcCIPmACuVmtW/nHViTu04zIGAB0pmZ4GaA== +dcmjs@0.17.0: + version "0.17.0" + resolved "https://registry.yarnpkg.com/dcmjs/-/dcmjs-0.17.0.tgz#dbfd6db3efd2936f9bead9c902b7db396f64e9f5" + integrity sha512-7JnNqNWXqjGyspLRwKM4nn+TU/+w3iMl9boU4eHrr831M4crMWWIiSQDejpOnfDT9+opqyeaYcNkiekr7p6TFg== dependencies: "@babel/polyfill" "^7.8.3" "@babel/runtime" "^7.8.4" gl-matrix "^3.1.0" + lodash.clonedeep "^4.5.0" loglevelnext "^3.0.1" ndarray "^1.0.19"