feat(segmentation): Enhance Segmentation with New AI and Once Click Tools (#4910)

.github/workflows/playwright.yml

@@ -9,7 +9,7 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     container:
-      image: mcr.microsoft.com/playwright:v1.50.0-noble
+      image: mcr.microsoft.com/playwright:v1.51.1-noble
     strategy:
       fail-fast: false
       matrix:

extensions/cornerstone-dicom-pmap/package.json

@@ -46,8 +46,8 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^3.7.17",
+    "@cornerstonejs/adapters": "^3.8.2",
-    "@cornerstonejs/core": "^3.7.17",
+    "@cornerstonejs/core": "^3.8.2",
     "@kitware/vtk.js": "32.12.0",
     "react-color": "^2.19.3"
   }

extensions/cornerstone-dicom-seg/package.json

@@ -46,8 +46,8 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^3.7.17",
+    "@cornerstonejs/adapters": "^3.8.2",
-    "@cornerstonejs/core": "^3.7.17",
+    "@cornerstonejs/core": "^3.8.2",
     "@kitware/vtk.js": "32.12.0",
     "react-color": "^2.19.3"
   }

extensions/cornerstone-dicom-seg/src/getToolbarModule.ts

@@ -45,7 +45,7 @@ export function getToolbarModule({ servicesManager }: withAppTypes) {
         if (!toolNames) {
           return {
             disabled: false,
-            // isActive: true,
+            // isActive: false,
           };
         }
 

extensions/cornerstone-dicom-sr/package.json

@@ -46,9 +46,9 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^3.7.17",
+    "@cornerstonejs/adapters": "^3.8.2",
-    "@cornerstonejs/core": "^3.7.17",
+    "@cornerstonejs/core": "^3.8.2",
-    "@cornerstonejs/tools": "^3.7.17",
+    "@cornerstonejs/tools": "^3.8.2",
     "classnames": "^2.3.2"
   }
 }

extensions/cornerstone-dynamic-volume/package.json

@@ -42,8 +42,8 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/core": "^3.7.17",
+    "@cornerstonejs/core": "^3.8.2",
-    "@cornerstonejs/tools": "^3.7.17",
+    "@cornerstonejs/tools": "^3.8.2",
     "classnames": "^2.3.2"
   }
 }

extensions/cornerstone/package.json

@@ -38,7 +38,7 @@
     "@cornerstonejs/codec-libjpeg-turbo-8bit": "^1.2.2",
     "@cornerstonejs/codec-openjpeg": "^1.2.4",
     "@cornerstonejs/codec-openjph": "^2.4.5",
-    "@cornerstonejs/dicom-image-loader": "^3.7.17",
+    "@cornerstonejs/dicom-image-loader": "^3.8.2",
     "@ohif/core": "3.10.0-beta.143",
     "@ohif/ui": "3.10.0-beta.143",
     "dcmjs": "*",
@@ -51,11 +51,12 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^3.7.17",
+    "@cornerstonejs/adapters": "^3.8.2",
-    "@cornerstonejs/core": "^3.7.17",
+    "@cornerstonejs/core": "^3.8.2",
-    "@cornerstonejs/labelmap-interpolation": "^3.7.17",
+    "@cornerstonejs/labelmap-interpolation": "^3.8.2",
-    "@cornerstonejs/polymorphic-segmentation": "^3.7.17",
+    "@cornerstonejs/ai": "^3.8.2",
-    "@cornerstonejs/tools": "^3.7.17",
+    "@cornerstonejs/polymorphic-segmentation": "^3.8.2",
+    "@cornerstonejs/tools": "^3.8.2",
     "@itk-wasm/morphological-contour-interpolation": "1.1.0",
     "@kitware/vtk.js": "32.12.0",
     "html2canvas": "^1.4.1",

extensions/cornerstone/src/commandsModule.ts

@@ -16,6 +16,7 @@ import {
 } from '@cornerstonejs/tools';
 import * as cornerstoneTools from '@cornerstonejs/tools';
 import * as labelmapInterpolation from '@cornerstonejs/labelmap-interpolation';
+import { ONNXSegmentationController } from '@cornerstonejs/ai';
 
 import { Types as OhifTypes, utils } from '@ohif/core';
 import i18n from '@ohif/i18n';
@@ -59,6 +60,28 @@ const getLabelmapTools = ({ toolGroupService }) => {
   return labelmapTools;
 };
 
+const segmentAI = new ONNXSegmentationController({
+  autoSegmentMode: true,
+  models: {
+    sam_b: [
+      {
+        name: 'sam-b-encoder',
+        url: 'https://huggingface.co/schmuell/sam-b-fp16/resolve/main/sam_vit_b_01ec64.encoder-fp16.onnx',
+        size: 180,
+        key: 'encoder',
+      },
+      {
+        name: 'sam-b-decoder',
+        url: 'https://huggingface.co/schmuell/sam-b-fp16/resolve/main/sam_vit_b_01ec64.decoder.onnx',
+        size: 17,
+        key: 'decoder',
+      },
+    ],
+  },
+  modelName: 'sam_b',
+});
+let segmentAIEnabled = false;
+
 function commandsModule({
   servicesManager,
   commandsManager,
@@ -979,7 +1002,7 @@ function commandsModule({
         });
       }
     },
-    setSourceViewportForReferenceLinesTool: ({ viewportId }) => {
+    setViewportForToolConfiguration: ({ viewportId, toolName }) => {
       if (!viewportId) {
         const { activeViewportId } = viewportGridService.getState();
         viewportId = activeViewportId ?? 'default';
@@ -987,9 +1010,11 @@ function commandsModule({
 
       const toolGroup = toolGroupService.getToolGroupForViewport(viewportId);
 
+      const prevConfig = toolGroup?.getToolConfiguration(toolName);
       toolGroup?.setToolConfiguration(
-        ReferenceLinesTool.toolName,
+        toolName,
         {
+          ...prevConfig,
           sourceViewportId: viewportId,
         },
         true // overwrite
@@ -1525,17 +1550,85 @@ function commandsModule({
         }
       });
     },
-    acceptPreview: () => {
+    toggleUseCenterSegmentIndex: ({ toggle }) => {
       const labelmapTools = getLabelmapTools({ toolGroupService });
       labelmapTools.forEach(tool => {
-        tool.acceptPreview();
+        tool.configuration = {
+          ...tool.configuration,
+          useCenterSegmentIndex: toggle,
+        };
       });
     },
-    rejectPreview: () => {
+    _handlePreviewAction: action => {
       const labelmapTools = getLabelmapTools({ toolGroupService });
-      labelmapTools.forEach(tool => {
+      const { viewport } = _getActiveViewportEnabledElement();
-        tool.rejectPreview();
+      const activeTools = labelmapTools.filter(
+        tool => tool.mode === 'Active' || tool.mode === 'Enabled'
+      );
+
+      activeTools.forEach(tool => {
+        tool[`${action}Preview`]();
       });
+
+      if (segmentAI.enabled) {
+        segmentAI[`${action}Preview`](viewport.element);
+      }
+    },
+    acceptPreview: () => {
+      actions._handlePreviewAction('accept');
+    },
+    rejectPreview: () => {
+      actions._handlePreviewAction('reject');
+    },
+    clearMarkersForMarkerLabelmap: () => {
+      const { viewport } = _getActiveViewportEnabledElement();
+      const toolGroup = cornerstoneTools.ToolGroupManager.getToolGroupForViewport(viewport.id);
+      const toolInstance = toolGroup.getToolInstance('MarkerLabelmap');
+
+      if (!toolInstance) {
+        return;
+      }
+
+      toolInstance.clearMarkers(viewport);
+    },
+    interpolateScrollForMarkerLabelmap: () => {
+      const { viewport } = _getActiveViewportEnabledElement();
+      const toolGroup = cornerstoneTools.ToolGroupManager.getToolGroupForViewport(viewport.id);
+      const toolInstance = toolGroup.getToolInstance('MarkerLabelmap');
+
+      if (!toolInstance) {
+        return;
+      }
+
+      toolInstance.interpolateScroll(viewport, 1);
+    },
+    toggleLabelmapAssist: async () => {
+      const { viewport } = _getActiveViewportEnabledElement();
+      const newState = !segmentAI.enabled;
+      segmentAI.enabled = newState;
+
+      if (!segmentAIEnabled) {
+        await segmentAI.initModel();
+        segmentAIEnabled = true;
+      }
+
+      // set the brush tool to active
+      const toolGroupIds = toolGroupService.getToolGroupIds();
+      if (newState) {
+        actions.setToolActiveToolbar({
+          toolName: 'CircularBrushForAutoSegmentAI',
+          toolGroupIds: toolGroupIds,
+        });
+      } else {
+        toolGroupIds.forEach(toolGroupId => {
+          const toolGroup = cornerstoneTools.ToolGroupManager.getToolGroup(toolGroupId);
+          toolGroup.setToolPassive('CircularBrushForAutoSegmentAI');
+        });
+      }
+
+      if (segmentAI.enabled) {
+        segmentAI.initViewport(viewport);
+      }
     },
   };
 
@@ -1680,8 +1773,8 @@ function commandsModule({
     setViewportColormap: {
       commandFn: actions.setViewportColormap,
     },
-    setSourceViewportForReferenceLinesTool: {
+    setViewportForToolConfiguration: {
-      commandFn: actions.setSourceViewportForReferenceLinesTool,
+      commandFn: actions.setViewportForToolConfiguration,
     },
     storePresentation: {
       commandFn: actions.storePresentation,
@@ -1806,6 +1899,10 @@ function commandsModule({
     toggleSegmentSelect: actions.toggleSegmentSelect,
     acceptPreview: actions.acceptPreview,
     rejectPreview: actions.rejectPreview,
+    toggleUseCenterSegmentIndex: actions.toggleUseCenterSegmentIndex,
+    toggleLabelmapAssist: actions.toggleLabelmapAssist,
+    interpolateScrollForMarkerLabelmap: actions.interpolateScrollForMarkerLabelmap,
+    clearMarkersForMarkerLabelmap: actions.clearMarkersForMarkerLabelmap,
   };
 
   return {

extensions/cornerstone/src/customizations/segmentationPanelCustomization.tsx

@@ -36,7 +36,7 @@ export default function getSegmentationPanelCustomization({ commandsManager, ser
       // Get initial states based on current configuration
       const [previewEdits, setPreviewEdits] = useState(false);
       const [toggleSegmentEnabled, setToggleSegmentEnabled] = useState(false);
-
+      const [useCenterAsSegmentIndex, setUseCenterAsSegmentIndex] = useState(false);
       const handlePreviewEditsChange = checked => {
         setPreviewEdits(checked);
         commandsManager.run('toggleSegmentPreviewEdit', { toggle: checked });
@@ -47,6 +47,11 @@ export default function getSegmentationPanelCustomization({ commandsManager, ser
         commandsManager.run('toggleSegmentSelect', { toggle: checked });
       };
 
+      const handleUseCenterAsSegmentIndexChange = checked => {
+        setUseCenterAsSegmentIndex(checked);
+        commandsManager.run('toggleUseCenterSegmentIndex', { toggle: checked });
+      };
+
       return (
         <div className="bg-muted flex flex-col gap-4 border-b border-b-[2px] border-black px-2 py-3">
           <div className="flex items-center gap-2">
@@ -57,12 +62,20 @@ export default function getSegmentationPanelCustomization({ commandsManager, ser
             <span className="text-base text-white">Preview edits before creating</span>
           </div>
 
+          <div className="flex items-center gap-2">
+            <Switch
+              checked={useCenterAsSegmentIndex}
+              onCheckedChange={handleUseCenterAsSegmentIndexChange}
+            />
+            <span className="text-base text-white">Use Center as Segment Index</span>
+          </div>
+
           <div className="flex items-center gap-2">
             <Switch
               checked={toggleSegmentEnabled}
               onCheckedChange={handleToggleSegmentEnabledChange}
             />
-            <span className="text-base text-white">Highlight segments to select</span>
+            <span className="text-base text-white">Hover on segment border to activate</span>
           </div>
         </div>
       );

extensions/cornerstone/src/initCornerstoneTools.js

@@ -38,7 +38,9 @@ import {
   OrientationMarkerTool,
   WindowLevelRegionTool,
   SegmentSelectTool,
+  RegionSegmentPlusTool,
 } from '@cornerstonejs/tools';
+import { LabelmapSlicePropagationTool, MarkerLabelmapTool } from '@cornerstonejs/ai';
 import * as polySeg from '@cornerstonejs/polymorphic-segmentation';
 
 import CalibrationLineTool from './tools/CalibrationLineTool';
@@ -98,6 +100,9 @@ export default function initCornerstoneTools(configuration = {}) {
   addTool(WindowLevelRegionTool);
   addTool(PlanarFreehandContourSegmentationTool);
   addTool(SegmentSelectTool);
+  addTool(LabelmapSlicePropagationTool);
+  addTool(MarkerLabelmapTool);
+  addTool(RegionSegmentPlusTool);
   // Modify annotation tools to use dashed lines on SR
   const annotationStyle = {
     textBoxFontSize: '15px',
@@ -152,6 +157,9 @@ const toolNames = {
   PlanarFreehandContourSegmentation: PlanarFreehandContourSegmentationTool.toolName,
   SegmentBidirectional: SegmentBidirectionalTool.toolName,
   SegmentSelect: SegmentSelectTool.toolName,
+  LabelmapSlicePropagation: LabelmapSlicePropagationTool.toolName,
+  MarkerLabelmap: MarkerLabelmapTool.toolName,
+  RegionSegmentPlus: RegionSegmentPlusTool.toolName,
 };
 
 export { toolNames };

extensions/cornerstone/src/services/SegmentationService/SegmentationService.ts

@@ -360,7 +360,7 @@ class SegmentationService extends PubSubService {
         type: LABELMAP,
         data: {
           imageIds: segImageIds,
-          referencedVolumeId: this._getVolumeIdForDisplaySet(displaySet),
+          // referencedVolumeId: this._getVolumeIdForDisplaySet(displaySet),
           referencedImageIds: referenceImageIds,
         },
       },

extensions/default/src/Toolbar/ToolBoxWrapper.tsx

@@ -42,14 +42,14 @@ export function ToolBoxButtonGroupWrapper({ groupId, buttonSection, ...props })
   );
 }
 
-export function ToolBoxButtonWrapper({ onInteraction, options, ...props }) {
+export function ToolBoxButtonWrapper({ onInteraction, className, options, ...props }) {
   return (
     <div className="bg-popover flex flex-row rounded-md px-0 py-0">
       <ToolButton
         {...props}
         id={props.id}
         size="small"
-        className={classNames(props.disabled && 'text-foreground/70')}
+        className={classNames(props.disabled && 'text-foreground/70', className)}
         onInteraction={event => {
           onInteraction?.({
             event,

extensions/default/src/ViewerLayout/index.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useState, useCallback } from 'react';
+import React, { useEffect, useState, useCallback, useRef } from 'react';
 import PropTypes from 'prop-types';
 
 import { InvestigationalUseDialog } from '@ohif/ui';
@@ -55,6 +55,18 @@ function ViewerLayout({
     setRightPanelClosed
   );
 
+  const leftPanelRef = useRef<HTMLDivElement>(null);
+  const rightPanelRef = useRef<HTMLDivElement>(null);
+
+  const handleMouseEnter = () => {
+    if (
+      leftPanelRef.current?.contains(document.activeElement) ||
+      rightPanelRef.current?.contains(document.activeElement)
+    ) {
+      (document.activeElement as HTMLElement)?.blur();
+    }
+  };
+
   const LoadingIndicatorProgress = customizationService.getCustomization(
     'ui.loadingIndicatorProgress'
   );
@@ -150,16 +162,17 @@ function ViewerLayout({
           {showLoadingIndicator && <LoadingIndicatorProgress className="h-full w-full bg-black" />}
           <ResizablePanelGroup {...resizablePanelGroupProps}>
             {/* LEFT SIDEPANELS */}
-
             {hasLeftPanels ? (
               <>
                 <ResizablePanel {...resizableLeftPanelProps}>
-                  <SidePanelWithServices
+                  <div ref={leftPanelRef}>
-                    side="left"
+                    <SidePanelWithServices
-                    isExpanded={!leftPanelClosedState}
+                      side="left"
-                    servicesManager={servicesManager}
+                      isExpanded={!leftPanelClosedState}
-                    {...leftPanelProps}
+                      servicesManager={servicesManager}
-                  />
+                      {...leftPanelProps}
+                    />
+                  </div>
                 </ResizablePanel>
                 <ResizableHandle
                   onDragging={onHandleDragging}
@@ -171,7 +184,10 @@ function ViewerLayout({
             {/* TOOLBAR + GRID */}
             <ResizablePanel {...resizableViewportGridPanelProps}>
               <div className="flex h-full flex-1 flex-col">
-                <div className="relative flex h-full flex-1 items-center justify-center overflow-hidden bg-black">
+                <div
+                  className="relative flex h-full flex-1 items-center justify-center overflow-hidden bg-black"
+                  onMouseEnter={handleMouseEnter}
+                >
                   <ViewportGridComp
                     servicesManager={servicesManager}
                     viewportComponents={viewportComponents}
@@ -188,12 +204,14 @@ function ViewerLayout({
                   className={resizableHandleClassName}
                 />
                 <ResizablePanel {...resizableRightPanelProps}>
-                  <SidePanelWithServices
+                  <div ref={rightPanelRef}>
-                    side="right"
+                    <SidePanelWithServices
-                    isExpanded={!rightPanelClosedState}
+                      side="right"
-                    servicesManager={servicesManager}
+                      isExpanded={!rightPanelClosedState}
-                    {...rightPanelProps}
+                      servicesManager={servicesManager}
-                  />
+                      {...rightPanelProps}
+                    />
+                  </div>
                 </ResizablePanel>
               </>
             ) : null}

extensions/default/src/utils/Toolbox.tsx

@@ -95,7 +95,7 @@ export function Toolbox({ buttonSectionId, title }: { buttonSectionId: string; t
         )}
       </PanelSection.Header>
 
-      <PanelSection.Content className="flex-shrink-0 border-none">
+      <PanelSection.Content className="bg-muted flex-shrink-0 border-none">
         {showConfig && <CustomConfigComponent />}
         {toolboxSections.map(section => {
           const sectionId = section.componentProps.buttonSection;
@@ -131,7 +131,7 @@ export function Toolbox({ buttonSectionId, title }: { buttonSectionId: string; t
           );
         })}
         {activeToolOptions && (
-          <div className="bg-primary-dark h-auto px-2">
+          <div className="bg-primary-dark mt-1 h-auto px-2">
             <ToolSettings options={activeToolOptions} />
           </div>
         )}

extensions/measurement-tracking/package.json

@@ -32,8 +32,8 @@
     "start": "yarn run dev"
   },
   "peerDependencies": {
-    "@cornerstonejs/core": "^3.7.17",
+    "@cornerstonejs/core": "^3.8.2",
-    "@cornerstonejs/tools": "^3.7.17",
+    "@cornerstonejs/tools": "^3.8.2",
     "@ohif/core": "3.10.0-beta.143",
     "@ohif/extension-cornerstone-dicom-sr": "3.10.0-beta.143",
     "@ohif/extension-default": "3.10.0-beta.143",

modes/basic-test-mode/src/toolbarButtons.ts

@@ -42,8 +42,10 @@ export const setToolActiveToolbar = {
 
 const ReferenceLinesListeners = [
   {
-    commandName: 'setSourceViewportForReferenceLinesTool',
+    commandName: 'setViewportForToolConfiguration',
-    context: 'CORNERSTONE',
+    commandOptions: {
+      toolName: 'ReferenceLines',
+    },
   },
 ];
 

modes/segmentation/src/index.tsx

@@ -92,11 +92,14 @@ function modeFactory({ modeConfiguration }) {
         'SegmentationTools',
       ]);
       toolbarService.createButtonSection('segmentationToolboxUtilitySection', [
+        'LabelmapSlicePropagation',
         'InterpolateLabelmap',
         'SegmentBidirectional',
       ]);
       toolbarService.createButtonSection('segmentationToolboxToolsSection', [
         'BrushTools',
+        'MarkerLabelmap',
+        'RegionSegmentPlus',
         'Shapes',
       ]);
       toolbarService.createButtonSection('brushToolsSection', ['Brush', 'Eraser', 'Threshold']);

modes/segmentation/src/initToolGroups.ts

@@ -27,6 +27,15 @@ function createTools(utilityModule) {
           activeStrategy: 'FILL_INSIDE_CIRCLE',
         },
       },
+      {
+        toolName: toolNames.LabelmapSlicePropagation,
+      },
+      {
+        toolName: toolNames.MarkerLabelmap,
+      },
+      {
+        toolName: toolNames.RegionSegmentPlus,
+      },
       {
         toolName: 'CircularEraser',
         parentTool: 'Brush',

modes/segmentation/src/toolbarButtons.ts

@@ -1,4 +1,5 @@
 import type { Button } from '@ohif/core/types';
+import { ViewportGridService } from '@ohif/core';
 
 const setToolActiveToolbar = {
   commandName: 'setToolActiveToolbar',
@@ -7,6 +8,15 @@ const setToolActiveToolbar = {
   },
 };
 
+const callbacks = (toolName: string) => [
+  {
+    commandName: 'setViewportForToolConfiguration',
+    commandOptions: {
+      toolName,
+    },
+  },
+];
+
 const toolbarButtons: Button[] = [
   // sections
   {
@@ -322,6 +332,94 @@ const toolbarButtons: Button[] = [
       commands: 'runSegmentBidirectional',
     },
   },
+  {
+    id: 'RegionSegmentPlus',
+    uiType: 'ohif.toolBoxButton',
+    props: {
+      icon: 'icon-tool-click-segment',
+      label: 'One Click Segment',
+      tooltip: 'One Click Segment',
+      evaluate: {
+        name: 'evaluate.cornerstone.segmentation',
+        toolNames: ['RegionSegmentPlus'],
+        disabledText: 'Create new segmentation to enable this tool.',
+      },
+      commands: 'setToolActiveToolbar',
+    },
+  },
+  {
+    id: 'LabelmapSlicePropagation',
+    uiType: 'ohif.toolBoxButton',
+    props: {
+      icon: 'icon-labelmap-slice-propagation',
+      label: 'Labelmap Assist',
+      tooltip: 'Labelmap Assist Based on Previous / Next Slice Data',
+      evaluate: [
+        'evaluate.cornerstoneTool.toggle',
+        {
+          name: 'evaluate.cornerstone.hasSegmentation',
+        },
+      ],
+      listeners: {
+        [ViewportGridService.EVENTS.ACTIVE_VIEWPORT_ID_CHANGED]: callbacks(
+          'LabelmapSlicePropagation'
+        ),
+        [ViewportGridService.EVENTS.VIEWPORTS_READY]: callbacks('LabelmapSlicePropagation'),
+      },
+      commands: 'toggleEnabledDisabledToolbar',
+    },
+  },
+  {
+    id: 'MarkerLabelmap',
+    uiType: 'ohif.toolBoxButton',
+    props: {
+      icon: 'icon-marker-labelmap',
+      label: 'Marker Guided Labelmap',
+      tooltip:
+        'Use Positive and Negative Points to Guide Labelmap Creation, hit N to move to next slice',
+      evaluate: [
+        {
+          name: 'evaluate.cornerstone.segmentation',
+          toolNames: ['MarkerLabelmap', 'MarkerInclude', 'MarkerExclude'],
+        },
+      ],
+      commands: 'setToolActiveToolbar',
+      listeners: {
+        [ViewportGridService.EVENTS.ACTIVE_VIEWPORT_ID_CHANGED]: callbacks('MarkerLabelmap'),
+        [ViewportGridService.EVENTS.VIEWPORTS_READY]: callbacks('MarkerLabelmap'),
+      },
+      options: [
+        {
+          name: 'Clear Markers',
+          type: 'button',
+          id: 'clear-markers',
+          commands: 'clearMarkersForMarkerLabelmap',
+        },
+        {
+          name: 'Marker Mode',
+          type: 'radio',
+          id: 'marker-mode',
+          value: 'markerInclude',
+          values: [
+            { value: 'markerInclude', label: 'Include' },
+            { value: 'markerExclude', label: 'Exclude' },
+          ],
+          commands: ({ commandsManager, options }) => {
+            const markerModeOption = options.find(option => option.id === 'marker-mode');
+            if (markerModeOption.value === 'markerInclude') {
+              commandsManager.run('setToolActive', {
+                toolName: 'MarkerInclude',
+              });
+            } else {
+              commandsManager.run('setToolActive', {
+                toolName: 'MarkerExclude',
+              });
+            }
+          },
+        },
+      ],
+    },
+  },
   {
     id: 'Eraser',
     uiType: 'ohif.toolBoxButton',

platform/app/.webpack/webpack.pwa.js

@@ -100,6 +100,10 @@ module.exports = (env, argv) => {
               ignore: ['**/config/**', '**/html-templates/**', '.DS_Store'],
             },
           },
+          {
+            from: '../../../node_modules/onnxruntime-web/dist',
+            to: `${DIST_DIR}/ort`,
+          },
           // Short term solution to make sure GCloud config is available in output
           // for our docker implementation
           {

platform/app/package.json

@@ -54,7 +54,7 @@
     "@cornerstonejs/codec-libjpeg-turbo-8bit": "^1.2.2",
     "@cornerstonejs/codec-openjpeg": "^1.2.4",
     "@cornerstonejs/codec-openjph": "^2.4.5",
-    "@cornerstonejs/dicom-image-loader": "^3.7.17",
+    "@cornerstonejs/dicom-image-loader": "^3.8.2",
     "@emotion/serialize": "^1.1.3",
     "@ohif/core": "3.10.0-beta.143",
     "@ohif/extension-cornerstone": "3.10.0-beta.143",

platform/core/package.json

@@ -37,7 +37,7 @@
     "@cornerstonejs/codec-libjpeg-turbo-8bit": "^1.2.2",
     "@cornerstonejs/codec-openjpeg": "^1.2.4",
     "@cornerstonejs/codec-openjph": "^2.4.5",
-    "@cornerstonejs/dicom-image-loader": "^3.7.17",
+    "@cornerstonejs/dicom-image-loader": "^3.8.2",
     "@ohif/ui": "3.10.0-beta.143",
     "cornerstone-math": "0.1.9",
     "dicom-parser": "^1.8.21"

platform/core/src/defaults/hotkeyBindings.ts

@@ -177,19 +177,24 @@ const bindings = [
     label: 'Reject Preview',
     keys: ['esc'],
   },
-  // after we have the ui for undo/redo, we can add these back in
+  {
-  // {
+    commandName: 'undo',
-  //   commandName: 'undo',
+    label: 'Undo',
-  //   label: 'Undo',
+    keys: ['ctrl+z'],
-  //   keys: ['ctrl+z'],
+    isEditable: true,
-  //   isEditable: true,
+  },
-  // },
+  {
-  // {
+    commandName: 'redo',
-  //   commandName: 'redo',
+    label: 'Redo',
-  //   label: 'Redo',
+    keys: ['ctrl+y'],
-  //   keys: ['ctrl+y'],
+    isEditable: true,
-  //   isEditable: true,
+  },
-  // },
+  {
+    commandName: 'interpolateScrollForMarkerLabelmap',
+    label: 'Interpolate Scroll',
+    keys: ['n'],
+    isEditable: true,
+  },
 ];
 
 export default bindings;

platform/ui-next/src/components/Icons/Icons.tsx

@@ -132,6 +132,7 @@ import {
   ToolSegmentAnything,
   ToolContract,
   ToolExpand,
+  ToolClickSegment,
 } from './Sources/Tools';
 import ActionNewDialog from './Sources/ActionNewDialog';
 import NotificationInfo from './Sources/NotificationInfo';
@@ -597,9 +598,10 @@ export const Icons = {
   'checkbox-active': (props: IconProps) => CheckBoxChecked(props),
   'icon-tool-eraser': (props: IconProps) => ToolEraser(props),
   'icon-tool-brush': (props: IconProps) => ToolBrush(props),
-  'icon-tool-labelmap-assist': (props: IconProps) => ToolLabelmapAssist(props),
+  'icon-labelmap-slice-propagation': (props: IconProps) => ToolLabelmapAssist(props),
-  'icon-tool-segment-anything': (props: IconProps) => ToolSegmentAnything(props),
+  'icon-marker-labelmap': (props: IconProps) => ToolSegmentAnything(props),
   'icon-tool-threshold': (props: IconProps) => ToolThreshold(props),
+  'icon-tool-click-segment': (props: IconProps) => ToolClickSegment(props),
   'icon-tool-pet-segment': (props: IconProps) => ToolPETSegment(props),
   'icon-tool-interpolation': (props: IconProps) => ToolInterpolation(props),
   'icon-tool-bidirectional-segment': (props: IconProps) => ToolBidirectionalSegment(props),

platform/ui-next/src/components/Icons/Sources/Tools.tsx

@@ -97,6 +97,27 @@ export const ToolLayout = (props: IconProps) => (
   </svg>
 );
 
+export const ToolClickSegment = (props: IconProps) => (
+  <svg
+    width="24"
+    height="24"
+    viewBox="0 0 24 24"
+    fill="none"
+    {...props}
+  >
+    <path
+      d="M18.7855 10.7063C19.0677 8.68223 18.754 6.39013 17.3334 4.86666C15.7881 3.20953 13.3171 2.78346 11.098 3.09398C9.51269 3.31581 7.93737 4.03721 6.63198 5.25454C5.24718 6.54592 4.39791 8.17926 4.11016 9.84527C3.74697 11.9481 4.27832 14.1031 5.75643 15.6882C7.30168 17.3453 9.54033 18.0136 11.7594 17.7031"
+      stroke="currentColor"
+      strokeWidth="1.5"
+      strokeLinecap="round"
+    />
+    <path
+      d="M15.0003 18.562L15.0714 12.3038C15.0714 12.1629 15.1293 12.0689 15.2452 12.0219C15.3663 11.9749 15.4769 12.0036 15.577 12.108L19.8981 16.502C20.0034 16.6169 20.0271 16.7318 19.9692 16.8467C19.9165 16.9615 19.8164 17.0216 19.669 17.0268L17.9627 17.0973L19.353 20.2929C19.3899 20.366 19.3951 20.4391 19.3688 20.5123C19.3477 20.5854 19.303 20.6376 19.2345 20.6689L18.4445 20.9822C18.3708 21.0083 18.2997 21.0057 18.2312 20.9744C18.1628 20.943 18.1128 20.8908 18.0812 20.8177L16.754 17.5673L15.5533 18.7656C15.4532 18.8648 15.3347 18.8935 15.1978 18.8518C15.0609 18.81 14.995 18.7134 15.0003 18.562Z"
+      fill="currentColor"
+    />
+  </svg>
+);
+
 export const ToolLength = (props: IconProps) => (
   <svg
     width="28px"

platform/ui-next/src/components/OHIFToolSettings/ToolSettings.tsx

@@ -2,12 +2,14 @@ import React from 'react';
 import RowInputRange from './RowInputRange';
 import RowSegmentedControl from './RowSegmentedControl';
 import RowDoubleRange from './RowDoubleRange';
+import { Button } from '../Button';
 
 const SETTING_TYPES = {
   RANGE: 'range',
   RADIO: 'radio',
   CUSTOM: 'custom',
   DOUBLE_RANGE: 'double-range',
+  BUTTON: 'button',
 };
 
 function ToolSettings({ options }) {
@@ -35,6 +37,8 @@ function ToolSettings({ options }) {
             return renderDoubleRangeSetting(option);
           case SETTING_TYPES.CUSTOM:
             return renderCustomSetting(option);
+          case SETTING_TYPES.BUTTON:
+            return renderButtonSetting(option);
           default:
             return null;
         }
@@ -97,4 +101,16 @@ const renderCustomSetting = option => {
   );
 };
 
+const renderButtonSetting = option => {
+  return (
+    <Button
+      className="bg-primary"
+      key={option.id}
+      onClick={() => option.onChange()}
+    >
+      {option.name}
+    </Button>
+  );
+};
+
 export default ToolSettings;

rsbuild.config.ts

@@ -102,6 +102,11 @@ export default defineConfig({
       // Copy plugin files (handled by writePluginImportsFile)
       ...(writePluginImportsFile(SRC_DIR, DIST_DIR) || []),
       // Copy public directory except config and html-templates
+      {
+        from: path.resolve(__dirname, 'node_modules/onnxruntime-web/dist'),
+        to: DIST_DIR,
+        force: true,
+      },
       {
         from: PUBLIC_DIR,
         to: DIST_DIR,

yarn.lock

@@ -1393,10 +1393,10 @@
   resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"
   integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==
 
-"@cornerstonejs/adapters@^3.7.17":
+"@cornerstonejs/adapters@^3.8.2":
-  version "3.7.17"
+  version "3.8.2"
-  resolved "https://registry.yarnpkg.com/@cornerstonejs/adapters/-/adapters-3.7.17.tgz#7e0a84d4010c4757e58c0d350c5604de865d8c56"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/adapters/-/adapters-3.8.2.tgz#44faeeb1cf3a384254630cf17c015e7427853feb"
-  integrity sha512-K1tz4y5YRgen+z8Yva5mzO3iwn1UZRoqf0wX8VjU1zszx/F4m19DSNcR9H5BJvlGM09DZjpJ42QXK2o9pIvnTg==
+  integrity sha512-6bxIa1BBvWD/X06MbB4QBOScTYmA1eTdLwg9jL+dHw5od8/CQhR22pfa0ZWsa8lkfFAMAdC1cU8BbiPNadG/0Q==
   dependencies:
     "@babel/runtime-corejs2" "^7.17.8"
     buffer "^6.0.3"
@@ -1404,6 +1404,20 @@
     gl-matrix "^3.4.3"
     ndarray "^1.0.19"
 
+"@cornerstonejs/ai@^3.8.2":
+  version "3.8.2"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/ai/-/ai-3.8.2.tgz#889d2b138e0f2d76a5355fc37675f915527c5d54"
+  integrity sha512-B9Mr3BPWOQT+GKhG79v9xsuEfgh8a/YCK/qaseFfUGahxD/7QEcu49MuzIdigbA1rbRA3OELL0ZjXxVx3ON27A==
+  dependencies:
+    "@babel/runtime-corejs2" "^7.17.8"
+    buffer "^6.0.3"
+    dcmjs "^0.29.8"
+    gl-matrix "^3.4.3"
+    lodash.clonedeep "^4.5.0"
+    ndarray "^1.0.19"
+    onnxruntime-common "1.17.1"
+    onnxruntime-web "1.17.1"
+
 "@cornerstonejs/calculate-suv@^1.1.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@cornerstonejs/calculate-suv/-/calculate-suv-1.1.0.tgz#35c8049d3eea03611225a8398d398cd81bf0031a"
@@ -1429,20 +1443,20 @@
   resolved "https://registry.yarnpkg.com/@cornerstonejs/codec-openjph/-/codec-openjph-2.4.5.tgz#8690b61a86fa53ef38a70eee9d665a79229517c0"
   integrity sha512-MZCUy8VG0VG5Nl1l58+g+kH3LujAzLYTfJqkwpWI2gjSrGXnP6lgwyy4GmPRZWVoS40/B1LDNALK905cNWm+sg==
 
-"@cornerstonejs/core@^3.7.17":
+"@cornerstonejs/core@^3.8.2":
-  version "3.7.17"
+  version "3.8.2"
-  resolved "https://registry.yarnpkg.com/@cornerstonejs/core/-/core-3.7.17.tgz#4a7986aa8bb7a68fbff597d3906e96e6b6483fc2"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/core/-/core-3.8.2.tgz#27f7bc10603f3ce6fd41057ec4ef52b09e3a62d5"
-  integrity sha512-2Z3FqrGJept9k1dmlGeMBPon1MDR/P7CkcTTevpKTlYiT3D7bqQ+Q5xdssLzWRD86et0PkRSJGktBM163WCgGw==
+  integrity sha512-zhtCXXXZMGooVYCmsfL7sC/o7wSFxXAfFm4rOp92t5m8n8UFbtmcLAB167/EEp/csHiRuv0PR/dOgd6y9tLb0A==
   dependencies:
     "@kitware/vtk.js" "32.12.1"
     comlink "^4.4.1"
     gl-matrix "^3.4.3"
     loglevel "^1.9.2"
 
-"@cornerstonejs/dicom-image-loader@^3.7.17":
+"@cornerstonejs/dicom-image-loader@^3.8.2":
-  version "3.7.17"
+  version "3.8.2"
-  resolved "https://registry.yarnpkg.com/@cornerstonejs/dicom-image-loader/-/dicom-image-loader-3.7.17.tgz#b14ba0cf64ab8a454ac086cd1ed6f2481359d533"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/dicom-image-loader/-/dicom-image-loader-3.8.2.tgz#dcfefe7fb77b63286e84c5bfbdaecd0647ef4e3f"
-  integrity sha512-LKHQ7x7S47TtF8M4ZmR4S+afbSsAT+yVMkKD81VU3A8oyuX7hT4+cUO4JlGGck3BxG4FyjgNSGT+T3YxAH1SHw==
+  integrity sha512-K0z5N+X6GpuRTfZhyuqpLIl+Y4JePyybQof/ZZ4aqwEIUm9NjXAYEUiYZEQQGtLItkPw+Vl7TbWA0FeI5GAC3Q==
   dependencies:
     "@cornerstonejs/codec-charls" "^1.2.3"
     "@cornerstonejs/codec-libjpeg-turbo-8bit" "^1.2.2"
@@ -1454,25 +1468,25 @@
     pako "^2.0.4"
     uuid "^9.0.0"
 
-"@cornerstonejs/labelmap-interpolation@^3.7.17":
+"@cornerstonejs/labelmap-interpolation@^3.8.2":
-  version "3.7.17"
+  version "3.8.2"
-  resolved "https://registry.yarnpkg.com/@cornerstonejs/labelmap-interpolation/-/labelmap-interpolation-3.7.17.tgz#fb57706c19c77cbc8192d81d9f2d6e086be637ae"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/labelmap-interpolation/-/labelmap-interpolation-3.8.2.tgz#2fe31e711d16366a7d591e414612205d0eeb7a38"
-  integrity sha512-K8R11wqv4TFljMArbx+4JbMtsINeMandp0mD8zyrj1rWXA+RY49QcNa7ob6KBnV26xJqXMT0VyJ0k/Ow53TiLQ==
+  integrity sha512-nvMHY4X6crsTL8EXKsKxXfVP/T04cCxCU90tsg7ceDi5mZ9qh6qnE0r8XOTclx/SsQ/YrwEaOIaflrfLKft19g==
   dependencies:
     "@itk-wasm/morphological-contour-interpolation" "1.1.0"
     itk-wasm "1.0.0-b.165"
 
-"@cornerstonejs/polymorphic-segmentation@^3.7.17":
+"@cornerstonejs/polymorphic-segmentation@^3.8.2":
-  version "3.7.17"
+  version "3.8.2"
-  resolved "https://registry.yarnpkg.com/@cornerstonejs/polymorphic-segmentation/-/polymorphic-segmentation-3.7.17.tgz#f6b4f4530996d974d59fff97ec29bd34a7a8cb16"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/polymorphic-segmentation/-/polymorphic-segmentation-3.8.2.tgz#faae193c34e7624d8fc55dabe3bd75b6d4b4e1db"
-  integrity sha512-ZI0643arqYbkrjadDSOsGTO2JS1OUtJkF8qEnjjG8MlxqQ7g6sPnTM/atrYRc9ZFoJxFL4kz7D0gOjcBrarYeQ==
+  integrity sha512-svlT/loFoB6t0r9dt31bMOkin82LftTF1J8ZWjJTOrbky2HxSyM4lnMbjQZbdhZDkhdQq0imQEOVIYNyhUAWBg==
   dependencies:
     "@icr/polyseg-wasm" "0.4.0"
 
-"@cornerstonejs/tools@^3.7.17":
+"@cornerstonejs/tools@^3.8.2":
-  version "3.7.17"
+  version "3.8.2"
-  resolved "https://registry.yarnpkg.com/@cornerstonejs/tools/-/tools-3.7.17.tgz#dc758ec14e0016bf76d04f54d3e91bd824ebdc7a"
+  resolved "https://registry.yarnpkg.com/@cornerstonejs/tools/-/tools-3.8.2.tgz#661cd986a19e5e3f49fffedea365aa290c0646fb"
-  integrity sha512-ZDMboShGzMDkqnzEDE7Y9UTJgHqGkVQt8l6AL5dCyA/IIlWEXXdxVe1weotJxgjMqaPETiwJ5EtN/3D4+tf1Rg==
+  integrity sha512-da8pJGv7b+sjAGIomH5CyM8+xHOvjhLipY4i/gp7Z6f5Wr5b/gYWc/l0CTD9G0aVFkyP06scGaBjwv0Ei1mqMQ==
   dependencies:
     "@types/offscreencanvas" "2019.7.3"
     comlink "^4.4.1"
@@ -10947,6 +10961,11 @@ flat@^5.0.2:
   resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241"
   integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==
 
+flatbuffers@^1.12.0:
+  version "1.12.0"
+  resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa"
+  integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==
+
 flatted@^3.2.9:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a"
@@ -11629,6 +11648,11 @@ graphemer@^1.4.0:
   resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6"
   integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==
 
+guid-typescript@^1.0.9:
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc"
+  integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==
+
 gunzip-maybe@^1.4.2:
   version "1.4.2"
   resolved "https://registry.yarnpkg.com/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz#b913564ae3be0eda6f3de36464837a9cd94b98ac"
@@ -14276,6 +14300,11 @@ long@^5.0.0:
   resolved "https://registry.yarnpkg.com/long/-/long-5.2.4.tgz#ee651d5c7c25901cfca5e67220ae9911695e99b2"
   integrity sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==
 
+long@^5.2.3:
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/long/-/long-5.3.1.tgz#9d4222d3213f38a5ec809674834e0f0ab21abe96"
+  integrity sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng==
+
 longest-streak@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-3.1.0.tgz#62fa67cd958742a1574af9f39866364102d90cd4"
@@ -16005,6 +16034,23 @@ onetime@^6.0.0:
   dependencies:
     mimic-fn "^4.0.0"
 
+onnxruntime-common@1.17.1:
+  version "1.17.1"
+  resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.17.1.tgz#1d3cfcfe56f9a5fc2ef5530c8e8723c86708a290"
+  integrity sha512-6wLNhpn+1hnsKN+jq6ulqUEJ61TdRmyFkGCvtRNnZkAupH8Yfr805UeNxjl9jtiX9B1q48pq6Q/67fEFpxT7Dw==
+
+onnxruntime-web@1.17.1:
+  version "1.17.1"
+  resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.17.1.tgz#ff0c5ecdcc4063e653f3cac4ff8e049a57301f60"
+  integrity sha512-EotY9uJU4xFY/ZVZ2Zrl2OZmBcbTVTWn/2OOh4cCWODPwtsYN2xeJYgoz8LfCgZSrhenGg0q4ceYUWATXqEsYQ==
+  dependencies:
+    flatbuffers "^1.12.0"
+    guid-typescript "^1.0.9"
+    long "^5.2.3"
+    onnxruntime-common "1.17.1"
+    platform "^1.3.6"
+    protobufjs "^7.2.4"
+
 open@^7.4.2:
   version "7.4.2"
   resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321"
@@ -16665,6 +16711,11 @@ pkg-up@^3.1.0:
   dependencies:
     find-up "^3.0.0"
 
+platform@^1.3.6:
+  version "1.3.6"
+  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7"
+  integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==
+
 playwright-core@1.48.0:
   version "1.48.0"
   resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.48.0.tgz#34d209dd4aba8fccd4a96116f1c4f7630f868722"
@@ -17436,7 +17487,7 @@ proto-list@~1.2.1:
   resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849"
   integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==
 
-protobufjs@^7.1.2:
+protobufjs@^7.1.2, protobufjs@^7.2.4:
   version "7.4.0"
   resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.4.0.tgz#7efe324ce9b3b61c82aae5de810d287bc08a248a"
   integrity sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==