* feat: 🎸 Progressive volume loading for vtk viewport (#1055)

* feat: 🎸 Progressive volume loading for vtk viewport

Adds progressive volume loading for vtkjs viewport whilst the frames are
streamed from PACs and rebuilt in the volume.

Closes: closes #1051

* Update to react-vtkjs-viewport 0.3.0

extensions/vtk/package.json

@@ -48,7 +48,7 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.5.5",
-    "react-vtkjs-viewport": "^0.1.6"
+    "react-vtkjs-viewport": "^0.3.0"
   },
   "devDependencies": {
     "@ohif/core": "^1.4.0",

extensions/vtk/src/ConnectedVTKViewport.js

@@ -1,6 +1,6 @@
-import OHIF from "@ohif/core";
-import { View2D } from "react-vtkjs-viewport";
-import { connect } from "react-redux";
+import OHIF from '@ohif/core';
+import { View2D } from 'react-vtkjs-viewport';
+import { connect } from 'react-redux';
 
 const { setViewportActive, setViewportSpecificData } = OHIF.redux.actions;
 
@@ -24,7 +24,7 @@ const mapStateToProps = (state, ownProps) => {
     // Hopefully this doesn't break anything under the hood for this one
     // activeTool: activeButton && activeButton.command,
     ...dataFromStore,
-    enableStackPrefetch: isActive
+    enableStackPrefetch: isActive,
   };
 };
 
@@ -38,7 +38,7 @@ const mapDispatchToProps = (dispatch, ownProps) => {
 
     setViewportSpecificData: data => {
       dispatch(setViewportSpecificData(viewportIndex, data));
-    }
+    },
   };
 };
 
@@ -63,10 +63,10 @@ const mergeProps = (propsFromState, propsFromDispatch, ownProps) => {
       // Store the API details for later
       //setViewportSpecificData({ vtkApi: api });
 
-      if (afterCreation && typeof afterCreation === "function") {
+      if (afterCreation && typeof afterCreation === 'function') {
         afterCreation(api);
       }
-    }
+    },
   };
   return props;
 };

extensions/vtk/src/LoadingIndicator.css

@@ -3,15 +3,15 @@
 }
 
 .loadingIndicator {
-  background-color: rgba(0, 0, 0, 0.75);
-  font-size: 18px;
-  height: 100%;
+  background-color: rgba(0, 0, 0, 0);
+  font-size: 8px;
+  height: 20%;
   overflow: hidden;
   position: absolute;
   top: 0;
   width: 100%;
   z-index: 1;
 
-  /* Necessary for click-through to cornerstone element below */
+  /* Necessary for click-through to vtkjs element below */
   pointer-events: none;
 }

extensions/vtk/src/LoadingIndicator.js

@@ -1,7 +1,7 @@
 import './LoadingIndicator.css';
 
 import React, { PureComponent } from 'react';
-
+import { withTranslation } from 'react-i18next';
 import PropTypes from 'prop-types';
 
 class LoadingIndicator extends PureComponent {
@@ -35,7 +35,8 @@ class LoadingIndicator extends PureComponent {
           <div className="imageViewerLoadingIndicator loadingIndicator">
             <div className="indicatorContents">
               <p>
-                Loading... <i className="fa fa-spin fa-circle-o-notch fa-fw" />{' '}
+                {this.props.t('Reformatting')}...
+                <i className="fa fa-spin fa-circle-o-notch fa-fw" />
                 {percComplete}
               </p>
             </div>
@@ -46,4 +47,4 @@ class LoadingIndicator extends PureComponent {
   }
 }
 
-export default LoadingIndicator;
+export default withTranslation('Common')(LoadingIndicator);

extensions/vtk/src/OHIFVTKViewport.js

@@ -172,47 +172,51 @@ class OHIFVTKViewport extends Component {
       default:
         imageDataObject = getImageData(stack.imageIds, displaySetInstanceUid);
 
-        const loadImageDataPromise = loadImageData(imageDataObject);
-
-        return loadImageDataPromise.then(() => {
-          return {
-            data: imageDataObject.vtkImageData,
-          };
-        });
+        return imageDataObject;
     }
   };
 
-  getOrCreateVolume(data, displaySetInstanceUid) {
+  getOrCreateVolume(imageDataObject, displaySetInstanceUid) {
     if (volumeCache[displaySetInstanceUid]) {
       return volumeCache[displaySetInstanceUid];
     }
 
+    const { vtkImageData, imageMetaData0 } = imageDataObject;
+
+    const { windowWidth, windowCenter, modality } = imageMetaData0;
+
+    let lower;
+    let upper;
+
+    if (modality === 'PT') {
+      // For PET just set the range to 0-5 SUV
+      lower = 0;
+      upper = 5;
+    } else {
+      lower = windowCenter - windowWidth / 2.0;
+      upper = windowCenter + windowWidth / 2.0;
+    }
+
     const volumeActor = vtkVolume.newInstance();
     const volumeMapper = vtkVolumeMapper.newInstance();
 
     volumeActor.setMapper(volumeMapper);
-    volumeMapper.setInputData(data);
+    volumeMapper.setInputData(vtkImageData);
 
-    const range = data
-      .getPointData()
-      .getScalars()
-      .getRange();
-
-    // TODO: For PET we might want to just set this to 0-5 SUV
     volumeActor
       .getProperty()
       .getRGBTransferFunction(0)
-      .setRange(range[0], range[1]);
+      .setRange(lower, upper);
 
     // TODO: Should look into implementing autoAdjustSampleDistance in vtk
-    const sampleDistance =
-      1.2 *
-      Math.sqrt(
-        data
-          .getSpacing()
-          .map(v => v * v)
-          .reduce((a, b) => a + b, 0)
-      );
+    // Manually changing the sample distance here by a couple orders of
+    // Magnitude in either direction makes no difference.
+    // Also setting volumeMapper.setMaximumSamplesPerRay(10000)
+    // Prevents the warning of too many samples, but the resolution isn't improved.
+
+    const spacing = vtkImageData.getSpacing();
+    // https://github.com/Kitware/VTK/blob/6b559c65bb90614fb02eb6d1b9e3f0fca3fe4b0b/Rendering/VolumeOpenGL2/vtkSmartVolumeMapper.cxx#L344
+    const sampleDistance = (spacing[0] + spacing[1] + spacing[2]) / 6.0;
 
     volumeMapper.setSampleDistance(sampleDistance);
 
@@ -221,7 +225,7 @@ class OHIFVTKViewport extends Component {
     return volumeActor;
   }
 
-  async setStateFromProps() {
+  setStateFromProps() {
     const { studies, displaySet } = this.props.viewportData;
     const {
       studyInstanceUid,
@@ -238,8 +242,7 @@ class OHIFVTKViewport extends Component {
     }
 
     const sopClassUid = sopClassUids[0];
-
-    let { data, labelmap } = await this.getViewportData(
+    const imageDataObject = this.getViewportData(
       studies,
       studyInstanceUid,
       displaySetInstanceUid,
@@ -254,13 +257,37 @@ class OHIFVTKViewport extends Component {
       labelmap = createLabelMapImageData(data);
     }*/
 
-    const volumeActor = this.getOrCreateVolume(data, displaySetInstanceUid);
+    const volumeActor = this.getOrCreateVolume(
+      imageDataObject,
+      displaySetInstanceUid
+    );
 
     this.setState({
-      volumes: [volumeActor],
-      paintFilterBackgroundImageData: data,
-      paintFilterLabelMapImageData: labelmap,
+      paintFilterBackgroundImageData: imageDataObject.vtkImageData,
+      paintFilterLabelMapImageData: null, // TODO
+      percentComplete: 0,
     });
+
+    this.setState(
+      {
+        paintFilterBackgroundImageData: imageDataObject.vtkImageData,
+        paintFilterLabelMapImageData: null, // TODO
+        percentComplete: 0,
+      },
+      () => {
+        this.loadProgressively(imageDataObject);
+
+        // TODO: There must be a better way to do this.
+        // We do this so that if all the data is available the react-vtkjs-viewport
+        // Will render _something_ before the volumes are set and the volume
+        // Construction that happens in react-vtkjs-viewport locks up the CPU.
+        setTimeout(() => {
+          this.setState({
+            volumes: [volumeActor],
+          });
+        }, 200);
+      }
+    );
   }
 
   componentDidMount() {
@@ -285,6 +312,37 @@ class OHIFVTKViewport extends Component {
     }
   }
 
+  loadProgressively(imageDataObject) {
+    loadImageData(imageDataObject);
+
+    const { isLoading, insertPixelDataPromises } = imageDataObject;
+
+    const numberOfFrames = insertPixelDataPromises.length;
+
+    if (!isLoading) {
+      this.setState({ isLoaded: true });
+      return;
+    }
+
+    insertPixelDataPromises.forEach(promise => {
+      promise.then(numberProcessed => {
+        const percentComplete = Math.floor(
+          (numberProcessed * 100) / numberOfFrames
+        );
+
+        if (percentComplete !== this.state.percentComplete) {
+          this.setState({
+            percentComplete,
+          });
+        }
+      });
+    });
+
+    Promise.all(insertPixelDataPromises).then(() => {
+      this.setState({ isLoaded: true });
+    });
+  }
+
   render() {
     let childrenWithProps = null;
 
@@ -302,21 +360,23 @@ class OHIFVTKViewport extends Component {
 
     return (
       <>
-        {this.state.volumes ? (
-          <ConnectedVTKViewport
-            volumes={this.state.volumes}
-            paintFilterLabelMapImageData={
-              this.state.paintFilterLabelMapImageData
-            }
-            paintFilterBackgroundImageData={
-              this.state.paintFilterBackgroundImageData
-            }
-            viewportIndex={this.props.viewportIndex}
-          />
-        ) : (
-          <div style={style}>
-            <LoadingIndicator />
-          </div>
+        <div style={style}>
+          {!this.state.isLoaded && (
+            <LoadingIndicator percentComplete={this.state.percentComplete} />
+          )}
+          {this.state.volumes && (
+            <ConnectedVTKViewport
+              volumes={this.state.volumes}
+              paintFilterLabelMapImageData={
+                this.state.paintFilterLabelMapImageData
+              }
+              paintFilterBackgroundImageData={
+                this.state.paintFilterBackgroundImageData
+              }
+              viewportIndex={this.props.viewportIndex}
+            />
+          )}
+        </div>
         )}
         {childrenWithProps}
       </>

extensions/vtk/src/commandsModule.js

@@ -116,8 +116,6 @@ function _setView(api, sliceNormal, viewUp) {
 
 function switchMPRInteractors(api, istyle) {
   const renderWindow = api.genericRenderWindow.getRenderWindow();
-  const renderer = api.genericRenderWindow.getRenderer();
-  const camera = renderer.getActiveCamera();
   const currentIStyle = renderWindow.getInteractor().getInteractorStyle();
 
   let currentViewport;
@@ -131,10 +129,12 @@ function switchMPRInteractors(api, istyle) {
     defaultSlabThickness = currentSlabThickness;
   }
 
-  renderWindow.getInteractor().setInteractorStyle(istyle);
+  const interactor = renderWindow.getInteractor();
+
+  interactor.setInteractorStyle(istyle);
 
   // TODO: Not sure why this is required the second time this function is called
-  istyle.setInteractor(renderWindow.getInteractor());
+  istyle.setInteractor(interactor);
 
   if (currentViewport) {
     istyle.setViewport(currentViewport);
@@ -260,12 +260,61 @@ const actions = {
     });
   },
   mpr2d: async ({ viewports }) => {
+    // TODO push a lot of this backdoor logic lower down to the library level.
     const displaySet =
       viewports.viewportSpecificData[viewports.activeViewportIndex];
 
+    // TODO -> Clean this logic up a bit.
+    const cornerstoneElement = cornerstone.getEnabledElement(displaySet.dom);
+
+    let cornerstoneVOI;
+
+    if (cornerstoneElement) {
+      const imageId = cornerstoneElement.image.imageId;
+
+      const { modality } = cornerstone.metaData.get(
+        'generalSeriesModule',
+        imageId
+      );
+
+      if (modality !== 'PT') {
+        const { windowWidth, windowCenter } = cornerstoneElement.viewport.voi;
+
+        cornerstoneVOI = {
+          windowWidth,
+          windowCenter,
+        };
+      }
+    }
+
+    const viewportProps = [
+      {
+        //Axial
+        orientation: {
+          sliceNormal: [0, 0, 1],
+          viewUp: [0, -1, 0],
+        },
+      },
+
+      {
+        // Sagital
+        orientation: {
+          sliceNormal: [1, 0, 0],
+          viewUp: [0, 0, 1],
+        },
+      },
+      {
+        // Coronal
+        orientation: {
+          sliceNormal: [0, 1, 0],
+          viewUp: [0, 0, 1],
+        },
+      },
+    ];
+
     let apiByViewport;
     try {
-      apiByViewport = await setMPRLayout(displaySet);
+      apiByViewport = await setMPRLayout(displaySet, viewportProps);
     } catch (error) {
       throw new Error(error);
     }
@@ -276,11 +325,29 @@ const actions = {
       .getProperty()
       .getRGBTransferFunction(0);
 
+    if (cornerstoneVOI) {
+      const { windowWidth, windowCenter } = cornerstoneVOI;
+      const lower = windowCenter - windowWidth / 2.0;
+      const upper = windowCenter + windowWidth / 2.0;
+
+      rgbTransferFunction.setRange(lower, upper);
+
+      apiByViewport.forEach(api => {
+        api.updateVOI(windowWidth, windowCenter);
+      });
+    }
+
     const onModifiedSubscription = rgbTransferFunction.onModified(() => {
-      apiByViewport.forEach(a => {
-        const renderWindow = a.genericRenderWindow.getRenderWindow();
+      const range = rgbTransferFunction.getMappingRange();
+      const windowWidth = Math.abs(range[1] - range[0]);
+      const windowCenter = range[0] + windowWidth / 2;
+
+      apiByViewport.forEach(api => {
+        const renderWindow = api.genericRenderWindow.getRenderWindow();
 
         renderWindow.render();
+
+        api.updateVOI(windowWidth, windowCenter);
       });
     });
 
@@ -289,7 +356,6 @@ const actions = {
     apiByViewport.forEach((api, index) => {
       const renderWindow = api.genericRenderWindow.getRenderWindow();
       const renderer = api.genericRenderWindow.getRenderer();
-      const camera = renderer.getActiveCamera();
 
       const istyle = vtkInteractorStyleMPRCrosshairs.newInstance();
       renderWindow.getInteractor().setInteractorStyle(istyle);
@@ -311,25 +377,10 @@ const actions = {
         crosshairsWidget,
       };
 
-      switch (index) {
-        default:
-        case 0:
-          //Axial
-          istyle.setSliceNormal(0, 0, 1);
-          istyle.setViewUp(0, -1, 0);
+      const orientation = api.getOrientation();
 
-          break;
-        case 1:
-          // sagittal
-          istyle.setSliceNormal(1, 0, 0);
-          istyle.setViewUp(0, 0, 1);
-          break;
-        case 2:
-          // Coronal
-          istyle.setSliceNormal(0, 1, 0);
-          istyle.setViewUp(0, 0, 1);
-          break;
-      }
+      istyle.setViewUp(...orientation.viewUp);
+      istyle.setSliceNormal(...orientation.sliceNormal);
 
       renderWindow.render();
     });

extensions/vtk/src/utils/setMPRLayout.js

@@ -1,11 +1,23 @@
 import setLayoutAndViewportData from './setLayoutAndViewportData.js';
 
-export default function setMPRLayout(displaySet) {
+export default function setMPRLayout(
+  displaySet,
+  viewportPropsArray,
+  numRows = 1,
+  numColumns = 3
+) {
   return new Promise((resolve, reject) => {
     const viewports = [];
-    const numRows = 1;
-    const numColumns = 3;
     const numViewports = numRows * numColumns;
+
+    if (viewportPropsArray && viewportPropsArray.length !== numViewports) {
+      reject(
+        new Error(
+          'viewportProps is supplied but its length is not equal to numViewports'
+        )
+      );
+    }
+
     const viewportSpecificData = {};
 
     for (let i = 0; i < numViewports; i++) {
@@ -17,8 +29,8 @@ export default function setMPRLayout(displaySet) {
     const apis = [];
     viewports.forEach((viewport, index) => {
       apis[index] = null;
+      const viewportProps = viewportPropsArray[index];
       viewports[index] = Object.assign({}, viewports[index], {
-        // plugin: 'vtk',
         vtk: {
           mode: 'mpr', // TODO: not used
           afterCreation: api => {
@@ -28,6 +40,7 @@ export default function setMPRLayout(displaySet) {
               resolve(apis);
             }
           },
+          ...viewportProps,
         },
       });
     });

platform/viewer/package.json

@@ -82,7 +82,7 @@
     "redux-oidc": "3.1.x",
     "redux-thunk": "^2.3.0",
     "reselect": "^4.0.0",
-    "vtk.js": "^11.0.1"
+    "vtk.js": "^11.6.0"
   },
   "devDependencies": {
     "cypress": "^3.3.1",

yarn.lock

@@ -15478,10 +15478,10 @@ react-transition-group@^4.1.1:
     loose-envify "^1.4.0"
     prop-types "^15.6.2"
 
-react-vtkjs-viewport@^0.1.6:
-  version "0.1.6"
-  resolved "https://registry.yarnpkg.com/react-vtkjs-viewport/-/react-vtkjs-viewport-0.1.6.tgz#d9bfb2c27c850c21c07c587405d0d3edcbd31d45"
-  integrity sha512-NO5Dx7MXnS6NTG+crIwcGlm1uBFYMTR/tXtLKfTfPwWNakazkqlUuhqpsw2dEcMqEB+v/RmAX6bzrArcjv4M4A==
+react-vtkjs-viewport@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/react-vtkjs-viewport/-/react-vtkjs-viewport-0.3.0.tgz#6aa09ce72d9e6f8396388a19222e1e7af230e270"
+  integrity sha512-LzaeUZR4h6BuePMOHZLPadfagnPM6SflSBRqALlBDe2Eii57uxp3lg6+9Qc74i36+H6u3/GwWLY2ftuVOkSDQw==
   dependencies:
     date-fns "^2.2.1"
     gl-matrix "^3.1.0"
@@ -19009,10 +19009,10 @@ void-elements@^2.0.1:
   resolved "https://registry.yarnpkg.com/void-elements/-/void-elements-2.0.1.tgz#c066afb582bb1cb4128d60ea92392e94d5e9dbec"
   integrity sha1-wGavtYK7HLQSjWDqkjkulNXp2+w=
 
-vtk.js@^11.0.1:
-  version "11.4.2"
-  resolved "https://registry.yarnpkg.com/vtk.js/-/vtk.js-11.4.2.tgz#6a967a4522a88704b6982a0d1356f24f77b09989"
-  integrity sha512-lr0GFZM5fGVUwM008KhxaSCB9dIeqKeD1ZZJQGcP+4rfcrPgC7Zp2L16kgQkj6rupPTseI+Nsk6UV5Dc8lJN4w==
+vtk.js@^11.6.0:
+  version "11.6.0"
+  resolved "https://registry.yarnpkg.com/vtk.js/-/vtk.js-11.6.0.tgz#a30f95cfb138453890bd85f5bca88c53ed7c2775"
+  integrity sha512-emkpLZVsuaSPLrVsbaosdD5RQQmal4PPfVotDIWb58aBbswXYt4pliaY3EDFNW3nTsKPs9ttadM4f4n8fVWVbQ==
   dependencies:
     blueimp-md5 "2.10.0"
     commander "2.11.0"