Skip to content
Snippets Groups Projects
Unverified Commit a6dfde5e authored by Tong Gao's avatar Tong Gao Committed by GitHub
Browse files

[CI] Full tests (#402) 


* [CI] Full tests

* Add github tests

* fix

* fix typo

Co-authored-by: default avatarzhouzaida <zhouzaida@163.com>
parent 2f093426
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 609 additions and 153 deletions
.circleci/config.yml
+ 26
− 142
View file @ a6dfde5e
version: 2.1 version: 2.1
jobs: # this allows you to use CircleCI's dynamic configuration feature
lint: setup: true
docker:
- image: cimg/python:3.7.4
steps:
- checkout
- run:
name: Linting
command: |
pip install pre-commit
pre-commit run --all-files
- run:
name: Check docstring coverage
command: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 80 mmengine
build_without_torch: # the path-filtering orb is required to continue a pipeline based on
parameters: # the path of an updated fileset
# The python version must match available image tags in orbs:
# https://circleci.com/developer/images/image/cimg/python path-filtering: circleci/path-filtering@0.1.2
python:
type: string
default: "3.7.4"
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
steps:
- checkout
- run:
name: Upgrade pip
command: |
python -V
python -m pip install pip --upgrade
python -m pip --version
- run:
name: Install mmengine dependencies
command: python -m pip install -r requirements.txt
- run:
name: Build and install
command: python -m pip install -e .
- run:
name: Run unit tests
command: python -m pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils
build_cpu:
parameters:
# The python version must match available image tags in
# https://circleci.com/developer/images/image/cimg/python
python:
type: string
default: "3.7.4"
torch:
type: string
torchvision:
type: string
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
steps:
- checkout
- run:
name: Upgrade pip
command: |
python -V
python -m pip install pip --upgrade
python -m pip --version
- run:
name: Install PyTorch
command: python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
- run:
name: Install mmcv-full
command: python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8.0/index.html
- run:
name: Install mmengine dependencies
command: python -m pip install -r requirements.txt
- run:
name: Build and install
command: python -m pip install -e .
- run:
name: Run unit tests
command: python -m pytest tests/
build_cu102:
machine:
image: ubuntu-1604-cuda-10.1:201909-23 # the actual version of cuda is 10.2
resource_class: gpu.nvidia.small
steps:
- checkout
- run:
# https://github.com/pytorch/vision/issues/2921
name: Install dependency of torchvision when using pyenv
command: sudo apt-get install -y liblzma-dev
- run:
# python3.7 should be re-installed due to the issue https://github.com/pytorch/vision/issues/2921
name: Select python3.7
command: |
pyenv uninstall -f 3.7.0
pyenv install 3.7.0
pyenv global 3.7.0
- run:
name: Upgrade pip
command: |
python -V
python -m pip install pip --upgrade
python -m pip --version
- run:
name: Install PyTorch
command: python -m pip install torch==1.8.1+cu102 torchvision==0.9.1+cu102 -f https://download.pytorch.org/whl/torch_stable.html
- run:
name: Install mmcv-full
command: python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.8.0/index.html
- run:
name: Install mmengine dependencies
command: python -m pip install -r requirements.txt
- run:
name: Build and install
command: python -m pip install -e .
- run:
name: Run unit tests
command: |
python -m coverage run --branch --source mmengine -m pytest tests/
python -m coverage xml
python -m coverage report -m
workflows: workflows:
unit_tests: # the always-run workflow is always triggered, regardless of the pipeline parameters.
always-run:
jobs: jobs:
- lint # the path-filtering/filter job determines which pipeline
- build_without_torch: # parameters to update.
requires: - path-filtering/filter:
- lint name: check-updated-files
- build_cpu: # 3-column, whitespace-delimited mapping. One mapping per
name: build_cpu_th1.8_py3.7 # line:
torch: 1.8.0 # <regex path-to-test> <parameter-to-set> <value-of-pipeline-parameter>
torchvision: 0.9.0 mapping: |
requires: mmengine/.* lint_only false
- lint requirements/.* lint_only false
- build_without_torch tests/.* lint_only false
- hold: .circleci/.* lint_only false
type: approval # <<< This key-value pair will set your workflow to a status of "On Hold" base-revision: main
requires: # this is the path of the configuration we should trigger once
- build_cpu_th1.8_py3.7 # path filtering and pipeline parameter value updates are
- build_cu102: # complete. In this case, we are using the parent dynamic
requires: # configuration itself.
- hold config-path: .circleci/test.yml
.circleci/docker/Dockerfile 0 → 100644
+ 11
− 0
View file @ a6dfde5e
ARG PYTORCH="1.8.1"
ARG CUDA="10.2"
ARG CUDNN="7"
FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel
# To fix GPG key error when running apt-get update
RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx
.circleci/scripts/get_mmcv_var.sh 0 → 100755
+ 19
− 0
View file @ a6dfde5e
#!/bin/bash
TORCH=$1
CUDA=$2
# 10.2 -> cu102
MMCV_CUDA="cu`echo ${CUDA} | tr -d '.'`"
# MMCV only provides pre-compiled packages for torch 1.x.0
# which works for any subversions of torch 1.x.
# We force the torch version to be 1.x.0 to ease package searching
# and avoid unnecessary rebuild during MMCV's installation.
TORCH_VER_ARR=(${TORCH//./ })
TORCH_VER_ARR[2]=0
printf -v MMCV_TORCH "%s." "${TORCH_VER_ARR[@]}"
MMCV_TORCH=${MMCV_TORCH%?} # Remove the last dot
echo "export MMCV_CUDA=${MMCV_CUDA}" >> $BASH_ENV
echo "export MMCV_TORCH=${MMCV_TORCH}" >> $BASH_ENV
.circleci/test.yml 0 → 100644
+ 214
− 0
View file @ a6dfde5e
version: 2.1
# the default pipeline parameters, which will be updated according to
# the results of the path-filtering orb
parameters:
lint_only:
type: boolean
default: true
jobs:
lint:
docker:
- image: cimg/python:3.7.4
steps:
- checkout
- run:
name: Install pre-commit hook
command: |
pip install pre-commit
pre-commit install
- run:
name: Linting
command: pre-commit run --all-files
- run:
name: Check docstring coverage
command: |
pip install interrogate
interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 80 mmengine
build_without_torch:
parameters:
# The python version must match available image tags in
# https://circleci.com/developer/images/image/cimg/python
python:
type: string
default: "3.7.4"
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
steps:
- checkout
- run:
name: Upgrade pip
command: |
python -V
python -m pip install pip --upgrade
python -m pip --version
- run:
name: Install mmengine dependencies
command: python -m pip install -r requirements.txt
- run:
name: Build and install
command: python -m pip install -e .
- run:
name: Run unit tests
command: python -m pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils
build_cpu:
parameters:
# The python version must match available image tags in
# https://circleci.com/developer/images/image/cimg/python
python:
type: string
torch:
type: string
torchvision:
type: string
docker:
- image: cimg/python:<< parameters.python >>
resource_class: large
steps:
- checkout
- run:
name: Get MMCV_TORCH as environment variables
command: |
. .circleci/scripts/get_mmcv_var.sh << parameters.torch >>
source $BASH_ENV
- run:
name: Install Libraries
command: |
sudo apt-get update
sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5
- run:
name: Configure Python & pip
command: |
python -m pip install --upgrade pip
python -m pip install wheel
- run:
name: Install PyTorch
command: |
python -V
python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
- run:
name: Install MMEngine dependencies
# python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${MMCV_TORCH}/index.html
command: |
python -m pip install -r requirements.txt
- run:
name: Build and install
command: |
python -m pip install -e .
- run:
name: Run unittests
command: |
python -m coverage run --branch --source mmengine -m pytest tests/
python -m coverage xml
python -m coverage report -m
build_cuda:
parameters:
torch:
type: string
cuda:
type: enum
enum: ["10.1", "10.2", "11.1"]
cudnn:
type: integer
default: 7
machine:
image: ubuntu-2004-cuda-11.4:202110-01
docker_layer_caching: true
resource_class: gpu.nvidia.small
steps:
- checkout
- run:
name: Get MMCV_TORCH and MMCV_CUDA as environment variables
command: |
. .circleci/scripts/get_mmcv_var.sh << parameters.torch >> << parameters.cuda >>
source $BASH_ENV
- run:
name: Build Docker image
command: |
docker build .circleci/docker -t mmengine:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >>
docker run --gpus all -t -d -v /home/circleci/project:/mmengine -w /mmengine --name mmengine mmengine:gpu
- run:
name: Install mmengine dependencies
# docker exec mmengine pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/${MMCV_CUDA}/torch${MMCV_TORCH}/index.html
command: |
docker exec mmengine pip install -r requirements.txt
- run:
name: Build and install
command: |
docker exec mmengine pip install -e .
- run:
name: Run unittests
command: |
docker exec mmengine python -m pytest tests/
workflows:
pr_stage_lint:
when: << pipeline.parameters.lint_only >>
jobs:
- lint:
name: lint
filters:
branches:
ignore:
- main
pr_stage_test:
when:
not:
<< pipeline.parameters.lint_only >>
jobs:
- lint:
name: lint
filters:
branches:
ignore:
- main
- build_without_torch:
name: build without torch
requires:
- lint
- build_cpu:
name: minimum_version_cpu
torch: 1.6.0
torchvision: 0.7.0
python: 3.6.9 # The lowest python 3.6.x version available on CircleCI images
requires:
- lint
- build_cpu:
name: maximum_version_cpu
torch: 1.9.0
torchvision: 0.10.0
python: 3.9.0
requires:
- minimum_version_cpu
- hold:
type: approval
requires:
- maximum_version_cpu
- build_cuda:
name: mainstream_version_gpu
torch: 1.8.1
# Use double quotation mark to explicitly specify its type
# as string instead of number
cuda: "10.2"
requires:
- hold
merge_stage_test:
when:
not:
<< pipeline.parameters.lint_only >>
jobs:
- build_cuda:
name: minimum_version_gpu
torch: 1.6.0
# Use double quotation mark to explicitly specify its type
# as string instead of number
cuda: "10.1"
filters:
branches:
only:
- main
.github/workflows/merge_stage_test.yml 0 → 100644
+ 179
− 0
View file @ a6dfde5e
name: merge_stage_test
on:
push:
paths-ignore:
- ".github/**.md"
- "docker/**"
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build_cpu_py:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.6, 3.8, 3.9]
torch: [1.8.1]
include:
- torch: 1.8.1
torchvision: 0.9.1
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Get MMCV_TORCH as the environment variable
run: . .github/workflows/scripts/get_mmcv_var.sh ${{matrix.torch}}
shell: bash
- name: Upgrade pip
run: pip install pip --upgrade
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
# - name: Install MMCV
# run: pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${MMCV_TORCH}/index.html
- name: Install other dependencies
run: |
pip install -r requirements/tests.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmengine -m pytest tests/
coverage xml
coverage report -m
build_cpu_pt:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
torch: [1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.1, 1.11.0, 1.12.0]
include:
- torch: 1.6.0
torchvision: 0.7.0
- torch: 1.7.1
torchvision: 0.8.2
- torch: 1.8.1
torchvision: 0.9.1
- torch: 1.9.1
torchvision: 0.10.1
- torch: 1.10.1
torchvision: 0.11.2
- torch: 1.11.0
torchvision: 0.12.0
- torch: 1.12.0
torchvision: 0.13.0
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
- name: Get MMCV_TORCH as the environment variable
run: . .github/workflows/scripts/get_mmcv_var.sh ${{matrix.torch}}
shell: bash
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
# - name: Install MMCV
# run: pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${MMCV_TORCH}/index.html
- name: Install other dependencies
run: |
pip install -r requirements/tests.txt -r requirements/optional.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmengine -m pytest tests/
coverage xml
coverage report -m
# Only upload coverage report for python3.7 && pytorch1.8.1 cpu
- name: Upload coverage to Codecov
if: ${{matrix.torch == '1.8.1' && matrix.python-version == '3.7'}}
uses: codecov/codecov-action@v1.0.14
with:
file: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
fail_ci_if_error: false
build_cu102:
runs-on: ubuntu-18.04
container:
image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel
strategy:
matrix:
python-version: [3.7]
include:
- torch: 1.8.1
cuda: 10.2
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: python -m pip install pip --upgrade
- name: Fetch GPG keys
run: |
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
- name: Get MMCV_TORCH and MMCV_CUDA as environment variables
run: . .github/workflows/scripts/get_mmcv_var.sh ${{matrix.torch}} ${{matrix.cuda}}
shell: bash
- name: Install Python-dev
run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
if: ${{matrix.python-version != 3.9}}
- name: Install system dependencies
run: |
apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6
- name: Install dependencies for compiling onnx when python=3.9
run: python -m pip install protobuf && apt-get update && apt-get -y install libprotobuf-dev protobuf-compiler cmake
if: ${{matrix.python-version == '3.9'}}
- name: Install mmengine dependencies
# python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/${MMCV_CUDA}/torch${MMCV_TORCH}/index.html
run: |
python -m pip install -r requirements/tests.txt
- name: Build and install
run: |
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 python -m pip install -e .
build_windows:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-2022]
python: [3.7]
platform: [cpu, cu102]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: python -m pip install pip --upgrade
- name: Install lmdb
run: python -m pip install lmdb
- name: Install PyTorch
run: python -m pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
- name: Install mmengine dependencies
# python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8/index.html --only-binary mmcv-full
run: |
python -m pip install -r requirements/tests.txt
- name: Build and install
run: |
python -m pip install -e .
- name: Run unittests and generate coverage report
run: |
pytest tests/
.github/workflows/pr_stage_test.yml 0 → 100644
+ 130
− 0
View file @ a6dfde5e
name: pr_stage_test
on:
pull_request:
paths-ignore:
- ".github/**.md"
- "docker/**"
- "docs/**"
- "docs_zh-CN/**"
- "README.md"
- "README_zh-CN.md"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build_cpu:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
include:
- torch: 1.8.1
torchvision: 0.9.1
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: pip install pip --upgrade
# - name: Get MMCV_TORCH as the environment variable
# run: . .github/workflows/scripts/get_mmcv_var.sh ${{matrix.torch}}
shell: bash
- name: Install PyTorch
run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
# - name: Install MMCV
# run: pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${MMCV_TORCH}/index.html
- name: Install other dependencies
run: |
pip install -r requirements/tests.txt
- name: Build and install
run: rm -rf .eggs && pip install -e .
- name: Run unittests and generate coverage report
run: |
coverage run --branch --source mmengine -m pytest tests/
coverage xml
coverage report -m
# Upload coverage report for python3.7 && pytorch1.8.1 cpu
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1.0.14
with:
file: ./coverage.xml
flags: unittests
env_vars: OS,PYTHON
name: codecov-umbrella
fail_ci_if_error: false
build_cu102:
runs-on: ubuntu-18.04
container:
image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel
strategy:
matrix:
python-version: [3.7]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: python -m pip install pip --upgrade
- name: Fetch GPG keys
run: |
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
# - name: Get MMCV_TORCH and MMCV_CUDA as environment variables
# run: . .github/workflows/scripts/get_mmcv_var.sh ${{matrix.torch}} ${{matrix.cuda}}
shell: bash
- name: Install Python-dev
run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
if: ${{matrix.python-version != 3.9}}
- name: Install system dependencies
run: |
apt-get update
apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libxrender-dev
- name: Install dependencies for compiling onnx when python=3.9
run: python -m pip install protobuf && apt-get update && apt-get -y install libprotobuf-dev protobuf-compiler cmake
if: ${{matrix.python-version == '3.9'}}
- name: Install mmengine dependencies
# python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/${MMCV_CUDA}/torch${MMCV_TORCH}/index.html
run: |
python -m pip install -r requirements/tests.txt
- name: Build and install
run: |
python setup.py check -m -s
TORCH_CUDA_ARCH_LIST=7.0 python -m pip install -e .
build_windows:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-2022]
python: [3.7]
platform: [cpu, cu102]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: python -m pip install pip --upgrade
- name: Install lmdb
run: python -m pip install lmdb
- name: Install PyTorch
run: python -m pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
- name: Install mmengine dependencies
# python -m pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8/index.html --only-binary mmcv-full
run: |
python -m pip install -r requirements/tests.txt
- name: Build and install
run: |
python -m pip install -e .
- name: Run unittests and generate coverage report
run: |
pytest tests/
.github/workflows/scripts/get_mmcv_var.sh 0 → 100755
+ 19
− 0
View file @ a6dfde5e
#!/bin/bash
TORCH=$1
CUDA=$2
# 10.2 -> cu102
MMCV_CUDA="cu`echo ${CUDA} | tr -d '.'`"
# MMCV only provides pre-compiled packages for torch 1.x.0
# which works for any subversions of torch 1.x.
# We force the torch version to be 1.x.0 to ease package searching
# and avoid unnecessary rebuild during MMCV's installation.
TORCH_VER_ARR=(${TORCH//./ })
TORCH_VER_ARR[2]=0
printf -v MMCV_TORCH "%s." "${TORCH_VER_ARR[@]}"
MMCV_TORCH=${MMCV_TORCH%?} # Remove the last dot
echo "MMCV_CUDA=${MMCV_CUDA}" >> $GITHUB_ENV
echo "MMCV_TORCH=${MMCV_TORCH}" >> $GITHUB_ENV
mmengine/dist/dist.py
+ 11
− 11
View file @ a6dfde5e
# Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) OpenMMLab. All rights reserved.
from typing import Any, List, Optional, Tuple, Dict, Generator, Union import os.path as osp
from collections import OrderedDict
import shutil
import pickle import pickle
import numpy as np import shutil
import tempfile import tempfile
from collections import OrderedDict
from typing import Any, Dict, Generator, List, Optional, Tuple, Union
import numpy as np
import torch import torch
import os.path as osp
from torch import Tensor from torch import Tensor
from torch import distributed as torch_dist
from torch._utils import (_flatten_dense_tensors, _take_tensors, from torch._utils import (_flatten_dense_tensors, _take_tensors,
_unflatten_dense_tensors) _unflatten_dense_tensors)
from torch import distributed as torch_dist
from torch.distributed import ProcessGroup from torch.distributed import ProcessGroup
import mmengine import mmengine
from .utils import (get_world_size, get_rank, get_backend, get_dist_info, from .utils import (get_world_size, get_rank, get_backend, get_dist_info,
get_default_group, barrier, get_data_device, get_default_group, barrier, get_data_device,
get_comm_device, cast_data_device) get_comm_device, cast_data_device)
from mmengine.utils.version_utils import digit_version from mmengine.utils import digit_version
from mmengine.utils.dl_utils import TORCH_VERSION from mmengine.utils.dl_utils import TORCH_VERSION
...@@ -95,10 +96,9 @@ def all_reduce(data: Tensor, ...@@ -95,10 +96,9 @@ def all_reduce(data: Tensor,
if op.lower() == 'mean': if op.lower() == 'mean':
torch_dist.all_reduce(data_on_device, _get_reduce_op('sum'), group) torch_dist.all_reduce(data_on_device, _get_reduce_op('sum'), group)
# When the type of `data_on_device` is int64, # use true_divide to handle torch1.6.0 throws an RuntimeError when
# `data_on_device.div_(world_size)` will appear RuntimeError: # the type of `data_on_device` is int64
# result type Float can't be cast to the desired output type Long. data_on_device = torch.true_divide(data_on_device, world_size)
data_on_device = data_on_device / world_size # type: ignore
else: else:
torch_dist.all_reduce(data_on_device, _get_reduce_op(op), group) torch_dist.all_reduce(data_on_device, _get_reduce_op(op), group)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Datenschutz und Impressum