1545 upgrade base image to pytorch 20.12 (#1613)

* test new image

Signed-off-by: Wenqi Li <[email protected]>

* temp no cron

Signed-off-by: Wenqi Li <[email protected]>

* fixes unit tests

Signed-off-by: Wenqi Li <[email protected]>

* adds base docker test

Signed-off-by: Wenqi Li <[email protected]>

* undo temp tests

Signed-off-by: Wenqi Li <[email protected]>

Author: wyli

.github/workflows/cron.yml

@@ -53,6 +53,38 @@ jobs:
         fail_ci_if_error: false
         file: ./coverage.xml
 
+  cron-pt-image:
+    if: github.repository == 'Project-MONAI/MONAI'
+    container:
+      image: nvcr.io/nvidia/pytorch:20.12-py3  # testing with the latest pytorch base image
+      options: "--gpus all"
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install the dependencies
+      run: |
+        which python
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r requirements-dev.txt
+        python -m pip list
+    - name: Run tests report coverage
+      run: |
+        export LAUNCH_DELAY=$[ $RANDOM % 16 * 60 ]
+        echo "Sleep $LAUNCH_DELAY"
+        sleep $LAUNCH_DELAY
+        nvidia-smi
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
+        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage
+        coverage xml
+    - name: Upload coverage
+      uses: codecov/codecov-action@v1
+      with:
+        fail_ci_if_error: false
+        file: ./coverage.xml
+
   cron-docker:
     if: github.repository == 'Project-MONAI/MONAI'
     container:
@@ -61,7 +93,8 @@ jobs:
     runs-on: [self-hosted, linux, x64, common]
     steps:
     - name: Run tests report coverage
-      # The docker image process has done the compilation.  BUILD_MONAI=1 may not be necessary.
+      # The docker image process has done the compilation.
+      # BUILD_MONAI=1 is necessary for triggering the USE_COMPILED flag.
       run: |
         cd /opt/monai
         nvidia-smi

Dockerfile

@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:20.10-py3
+ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:20.12-py3
 
 FROM ${PYTORCH_IMAGE}
 

tests/test_ensemble_evaluator.py

@@ -55,7 +55,7 @@ def forward(self, x):
         def run_post_transform(engine):
             for i in range(5):
                 expected_value = engine.state.iteration + i
-                torch.testing.assert_allclose(engine.state.output[f"pred{i}"], expected_value)
+                torch.testing.assert_allclose(engine.state.output[f"pred{i}"], torch.tensor([[expected_value]]))
 
         val_engine.run()
 

tests/test_handler_checkpoint_loader.py

@@ -40,7 +40,7 @@ def test_one_save_one_load(self):
             engine = Engine(lambda e, b: None)
             CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
             engine.run([0] * 8, max_epochs=1)
-            torch.testing.assert_allclose(net2.state_dict()["weight"], 0.1)
+            torch.testing.assert_allclose(net2.state_dict()["weight"], torch.tensor([0.1]))
 
     def test_two_save_one_load(self):
         logging.basicConfig(stream=sys.stdout, level=logging.INFO)
@@ -62,7 +62,7 @@ def test_two_save_one_load(self):
             engine = Engine(lambda e, b: None)
             CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
             engine.run([0] * 8, max_epochs=1)
-            torch.testing.assert_allclose(net2.state_dict()["weight"], 0.1)
+            torch.testing.assert_allclose(net2.state_dict()["weight"], torch.tensor([0.1]))
 
     def test_save_single_device_load_multi_devices(self):
         logging.basicConfig(stream=sys.stdout, level=logging.INFO)
@@ -83,7 +83,7 @@ def test_save_single_device_load_multi_devices(self):
             engine = Engine(lambda e, b: None)
             CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
             engine.run([0] * 8, max_epochs=1)
-            torch.testing.assert_allclose(net2.state_dict()["module.weight"].cpu(), 0.1)
+            torch.testing.assert_allclose(net2.state_dict()["module.weight"].cpu(), torch.tensor([0.1]))
 
 
 if __name__ == "__main__":