diff --git a/.gitignore b/.gitignore index 8f6fd08e34a8e26e7911c29d5f863548c3d35325..003794778f439e7771af85fbce5e47035390f4a8 100644 --- a/.gitignore +++ b/.gitignore @@ -105,8 +105,6 @@ venv.bak/ # mypy .mypy_cache/ -data/ -data .vscode .idea .DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb7df7adaeb0d8c27fb8196bd14f6842336effa6..98d142c3f9282a91eb63e84d23e2c5b5c426c6a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +exclude: ^tests/data/ repos: - repo: https://gitlab.com/pycqa/flake8.git rev: 3.8.3 diff --git a/tests/data/config/json_config/base3.json b/tests/data/config/json_config/base3.json new file mode 100644 index 0000000000000000000000000000000000000000..3251c5d6395974fa788eb27a368b03150eabd72c --- /dev/null +++ b/tests/data/config/json_config/base3.json @@ -0,0 +1,3 @@ +{ + "item3": true +} diff --git a/tests/data/config/json_config/simple.config.json b/tests/data/config/json_config/simple.config.json new file mode 100644 index 0000000000000000000000000000000000000000..cc317436ec91dc176eb825af2472d26a5b5cda82 --- /dev/null +++ b/tests/data/config/json_config/simple.config.json @@ -0,0 +1,8 @@ +{ + "item1": [1, 2], + "item2": { + "a": 0 + }, + "item3": true, + "item4": "test" +} diff --git a/tests/data/config/json_config/simple_config.json b/tests/data/config/json_config/simple_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cc317436ec91dc176eb825af2472d26a5b5cda82 --- /dev/null +++ b/tests/data/config/json_config/simple_config.json @@ -0,0 +1,8 @@ +{ + "item1": [1, 2], + "item2": { + "a": 0 + }, + "item3": true, + "item4": "test" +} diff --git a/tests/data/config/json_config/test_base.json b/tests/data/config/json_config/test_base.json new file mode 100644 index 0000000000000000000000000000000000000000..dd2917b946e91179f07c737eb8ab4b50d423862f --- /dev/null +++ b/tests/data/config/json_config/test_base.json @@ -0,0 +1,13 @@ +{ + "_base_": [ + "../py_config/base1.py", + "../yaml_config/base2.yaml", + "./base3.json", + "../py_config/base4.py" + ], + "item3": false, + "item4": "test", + "item8": "{{fileBasename}}", + "item9": {{ _base_.item2 }}, + "item10": {{ _base_.item7.b.c }} +} diff --git a/tests/data/config/json_config/test_base_variables_nested.json b/tests/data/config/json_config/test_base_variables_nested.json new file mode 100644 index 0000000000000000000000000000000000000000..af13a7eb997a55856231066df3e59b079ab212d0 --- /dev/null +++ b/tests/data/config/json_config/test_base_variables_nested.json @@ -0,0 +1,26 @@ +{ + "_base_": [ + "../py_config/test_base_variables.py" + ], + "base": "_base_.item8", + "item11": {{ _base_.item8 }}, + "item12": {{ _base_.item9 }}, + "item13": {{ _base_.item10 }}, + "item14": {{ _base_.item1 }}, + "item15": { + "a": { + "b": {{ _base_.item2 }} + }, + "b": [ + {{ _base_.item3 }} + ], + "c": [{{ _base_.item4 }}], + "d": [[ + { + "e": {{ _base_.item5.a }} + } + ], + {{ _base_.item6 }}], + "e": {{ _base_.item1 }} + } +} diff --git a/tests/data/config/json_config/test_predefined_var.json b/tests/data/config/json_config/test_predefined_var.json new file mode 100644 index 0000000000000000000000000000000000000000..84c5e3ed33ffb4365385c4af9b83196f9d28008d --- /dev/null +++ b/tests/data/config/json_config/test_predefined_var.json @@ -0,0 +1,3 @@ +{ + "item1": "{{ fileDirname }}" +} diff --git a/tests/data/config/json_config/test_reserved_key.json b/tests/data/config/json_config/test_reserved_key.json new file mode 100644 index 0000000000000000000000000000000000000000..599d20a989666ecdfb3dbfe445209066b360fcd2 --- /dev/null +++ b/tests/data/config/json_config/test_reserved_key.json @@ -0,0 +1,3 @@ +{ + "filename": "reserved.py" +} diff --git a/tests/data/config/py_config/base.py b/tests/data/config/py_config/base.py new file mode 100644 index 0000000000000000000000000000000000000000..2364e1d10b054e99c2e1e5780cf8d0e007d659c2 --- /dev/null +++ b/tests/data/config/py_config/base.py @@ -0,0 +1,5 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item1 = [1, 2] +item2 = {'a': 0} +item3 = True +item4 = 'test' diff --git a/tests/data/config/py_config/base1.py b/tests/data/config/py_config/base1.py new file mode 100644 index 0000000000000000000000000000000000000000..13db1375e71095d4295bde140bceaad9db9e1c31 --- /dev/null +++ b/tests/data/config/py_config/base1.py @@ -0,0 +1,2 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item1 = [1, 2] diff --git a/tests/data/config/py_config/base4.py b/tests/data/config/py_config/base4.py new file mode 100644 index 0000000000000000000000000000000000000000..cb7b4365ec3674339d3de106bee06c451d4d09ee --- /dev/null +++ b/tests/data/config/py_config/base4.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item5 = dict(a=0, b=1) +item6 = [dict(a=0), dict(b=1)] +item7 = dict(a=[0, 1, 2], b=dict(c=[3.1, 4.2, 5.3])) diff --git a/tests/data/config/py_config/config.py b/tests/data/config/py_config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..65c03bf884f9548e4cb5a2e04e51d127c30a272a --- /dev/null +++ b/tests/data/config/py_config/config.py @@ -0,0 +1,5 @@ +# Copyright (c) OpenMMLab. All rights reserved. +test_int = 1 +test_list = [1, 2, 3] +# include type, optimizer can be initiated by build_from_cfg +optimizer = dict(type='SGD', lr=0.1, momentum=0.9, weight_decay=0.0001) diff --git a/tests/data/config/py_config/simple.config.py b/tests/data/config/py_config/simple.config.py new file mode 100644 index 0000000000000000000000000000000000000000..2364e1d10b054e99c2e1e5780cf8d0e007d659c2 --- /dev/null +++ b/tests/data/config/py_config/simple.config.py @@ -0,0 +1,5 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item1 = [1, 2] +item2 = {'a': 0} +item3 = True +item4 = 'test' diff --git a/tests/data/config/py_config/simple_config.py b/tests/data/config/py_config/simple_config.py new file mode 100644 index 0000000000000000000000000000000000000000..2364e1d10b054e99c2e1e5780cf8d0e007d659c2 --- /dev/null +++ b/tests/data/config/py_config/simple_config.py @@ -0,0 +1,5 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item1 = [1, 2] +item2 = {'a': 0} +item3 = True +item4 = 'test' diff --git a/tests/data/config/py_config/test_base_variables.py b/tests/data/config/py_config/test_base_variables.py new file mode 100644 index 0000000000000000000000000000000000000000..4d20d7f0250565aa0af5e1f9742ec5a6579a2639 --- /dev/null +++ b/tests/data/config/py_config/test_base_variables.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = [ + './base1.py', '../yaml_config/base2.yaml', '../json_config/base3.json', + './base4.py' +] + +item3 = False +item4 = 'test' +item8 = '{{fileBasename}}' +item9 = {{_base_.item2}} +item10 = {{_base_.item7.b.c}} diff --git a/tests/data/config/py_config/test_base_variables_nested.py b/tests/data/config/py_config/test_base_variables_nested.py new file mode 100644 index 0000000000000000000000000000000000000000..ea9a6004a86c15a68c9e43743376411cd07e2faf --- /dev/null +++ b/tests/data/config/py_config/test_base_variables_nested.py @@ -0,0 +1,13 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = ['./test_base_variables.py'] +base = '_base_.item8' +item11 = {{_base_.item8}} +item12 = {{_base_.item9}} +item13 = {{_base_.item10}} +item14 = {{_base_.item1}} +item15 = dict( + a=dict(b={{_base_.item2}}), + b=[{{_base_.item3}}], + c=[{{_base_.item4}}], + d=[[dict(e={{_base_.item5.a}})], {{_base_.item6}}], + e={{_base_.item1}}) diff --git a/tests/data/config/py_config/test_code_in_config.py b/tests/data/config/py_config/test_code_in_config.py new file mode 100644 index 0000000000000000000000000000000000000000..cd59844b7d5e670b216d7b5eb77ab50a839adc7f --- /dev/null +++ b/tests/data/config/py_config/test_code_in_config.py @@ -0,0 +1,5 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmcv import Config # isort:skip + +cfg = Config.fromfile('tests/data/config/py_config/simple_config.py') +item5 = cfg.item1[0] + cfg.item2.a diff --git a/tests/data/config/py_config/test_custom_import.py b/tests/data/config/py_config/test_custom_import.py new file mode 100644 index 0000000000000000000000000000000000000000..d485a19005c6185d400e78f3fc588c95d8dcfd91 --- /dev/null +++ b/tests/data/config/py_config/test_custom_import.py @@ -0,0 +1,3 @@ +# Copyright (c) OpenMMLab. All rights reserved. +custom_imports = dict( + imports=['test_custom_import_module'], allow_failed_imports=False) diff --git a/tests/data/config/py_config/test_custom_import_module.py b/tests/data/config/py_config/test_custom_import_module.py new file mode 100644 index 0000000000000000000000000000000000000000..853b31ae6bd8b2dbd239ea59d303e1d6bd61d183 --- /dev/null +++ b/tests/data/config/py_config/test_custom_import_module.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os + +os.environ['TEST_VALUE'] = 'test' diff --git a/tests/data/config/py_config/test_dump_pickle_support.py b/tests/data/config/py_config/test_dump_pickle_support.py new file mode 100644 index 0000000000000000000000000000000000000000..fa7aae266e9068f9c85ffdf4d268ddfa01923ebb --- /dev/null +++ b/tests/data/config/py_config/test_dump_pickle_support.py @@ -0,0 +1,17 @@ +# Copyright (c) OpenMMLab. All rights reserved. +test_item1 = [1, 2] +bool_item2 = True +str_item3 = 'test' +dict_item4 = dict( + a={ + 'c/d': 'path/d', + 'f': 's3//f', + 6: '2333', + '2333': 'number' + }, + b={'8': 543}, + c={9: 678}, + d={'a': 0}, + f=dict(a='69')) +dict_item5 = {'x/x': {'a.0': 233}} +dict_list_item6 = {'x/x': [{'a.0': 1., 'b.0': 2.}, {'c/3': 3.}]} diff --git a/tests/data/config/py_config/test_merge_delete.py b/tests/data/config/py_config/test_merge_delete.py new file mode 100644 index 0000000000000000000000000000000000000000..f8a1eaf64c46d301f47a90d4ac907d1a0362e84e --- /dev/null +++ b/tests/data/config/py_config/test_merge_delete.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = './base.py' +item1 = {'a': 0, '_delete_': True} +item2 = {'b': 0} diff --git a/tests/data/config/py_config/test_merge_from_base_error.py b/tests/data/config/py_config/test_merge_from_base_error.py new file mode 100644 index 0000000000000000000000000000000000000000..1340e4bd27198e3d3ef82dbf516f22d8daf236f2 --- /dev/null +++ b/tests/data/config/py_config/test_merge_from_base_error.py @@ -0,0 +1,3 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = './base.py' +item3 = {'a': 1} diff --git a/tests/data/config/py_config/test_merge_from_base_single.py b/tests/data/config/py_config/test_merge_from_base_single.py new file mode 100644 index 0000000000000000000000000000000000000000..19edcf82d0c9a40c007ba6a1eca03153f7056ce0 --- /dev/null +++ b/tests/data/config/py_config/test_merge_from_base_single.py @@ -0,0 +1,6 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = './base.py' +item1 = [2, 3] +item2 = {'a': 1} +item3 = False +item4 = 'test_base' diff --git a/tests/data/config/py_config/test_merge_from_dict.py b/tests/data/config/py_config/test_merge_from_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..cca07539c8942c7d0424d685d7a3c5e829f27d3a --- /dev/null +++ b/tests/data/config/py_config/test_merge_from_dict.py @@ -0,0 +1,2 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item = [{'a': 0}, {'b': 0, 'c': 0}] diff --git a/tests/data/config/py_config/test_merge_from_multiple_bases.py b/tests/data/config/py_config/test_merge_from_multiple_bases.py new file mode 100644 index 0000000000000000000000000000000000000000..da575c39bc507914f7a233ad2bf7aedb995b646a --- /dev/null +++ b/tests/data/config/py_config/test_merge_from_multiple_bases.py @@ -0,0 +1,9 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = [ + './base1.py', '../yaml_config/base2.yaml', '../json_config/base3.json', + './base4.py' +] +item3 = False +item4 = 'test' +item_bool = True +item_float = 1.0 diff --git a/tests/data/config/py_config/test_merge_from_multiple_error.py b/tests/data/config/py_config/test_merge_from_multiple_error.py new file mode 100644 index 0000000000000000000000000000000000000000..b38596d95393ddc0673288d86c610c26d0f6ecf5 --- /dev/null +++ b/tests/data/config/py_config/test_merge_from_multiple_error.py @@ -0,0 +1,7 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = [ + './base1.py', '../yaml_config/base2.yaml', '../json_config/base3.json', + 'simple_config.py' +] +item3 = False +item4 = 'test' diff --git a/tests/data/config/py_config/test_merge_intermediate_variable_base.py b/tests/data/config/py_config/test_merge_intermediate_variable_base.py new file mode 100644 index 0000000000000000000000000000000000000000..f31a46a15de9d84191e25e8117d84a50fc967474 --- /dev/null +++ b/tests/data/config/py_config/test_merge_intermediate_variable_base.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item1 = [1, 2] +item2 = {'a': 0} +item3 = True +item4 = 'test' +item_cfg = {'b': 1} +item5 = {'cfg': item_cfg} +item6 = {'cfg': item_cfg} diff --git a/tests/data/config/py_config/test_merge_intermediate_variable_child.py b/tests/data/config/py_config/test_merge_intermediate_variable_child.py new file mode 100644 index 0000000000000000000000000000000000000000..17325b13bc515f55c044453d3d27b4f4110d1001 --- /dev/null +++ b/tests/data/config/py_config/test_merge_intermediate_variable_child.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = './test_merge_intermediate_variable_base.py' +item_cfg = {'b': 2} +item6 = {'cfg': item_cfg} diff --git a/tests/data/config/py_config/test_merge_recursive_bases.py b/tests/data/config/py_config/test_merge_recursive_bases.py new file mode 100644 index 0000000000000000000000000000000000000000..6d2218bab5ca7eb8c529e11e9cb0c159923b276e --- /dev/null +++ b/tests/data/config/py_config/test_merge_recursive_bases.py @@ -0,0 +1,3 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = './test_merge_from_base_single.py' +item4 = 'test_recursive_bases' diff --git a/tests/data/config/py_config/test_pre_substitute_base_vars.py b/tests/data/config/py_config/test_pre_substitute_base_vars.py new file mode 100644 index 0000000000000000000000000000000000000000..72d67ab404d153dea759cd0e71ad8fc28faac308 --- /dev/null +++ b/tests/data/config/py_config/test_pre_substitute_base_vars.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +_base_ = ['./test_base_variables_nested.py'] +item21 = {{_base_.item11}} +item22 = item21 +item23 = {{_base_.item10}} +item24 = item23 +item25 = dict( + a=dict(b=item24), + b=[item24], + c=[[dict(e=item22)], {{_base_.item6}}], + e=item21) diff --git a/tests/data/config/py_config/test_predefined_var.py b/tests/data/config/py_config/test_predefined_var.py new file mode 100644 index 0000000000000000000000000000000000000000..82594590cf4a73bed123e92ad8c392f3d4723148 --- /dev/null +++ b/tests/data/config/py_config/test_predefined_var.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +item1 = '{{fileBasename}}' +item2 = '{{ fileDirname}}' +item3 = 'abc_{{ fileBasenameNoExtension }}' diff --git a/tests/data/config/py_config/test_reserved_key.py b/tests/data/config/py_config/test_reserved_key.py new file mode 100644 index 0000000000000000000000000000000000000000..34d4ebe2f898a01ee8aa11a51f0383040213dc7f --- /dev/null +++ b/tests/data/config/py_config/test_reserved_key.py @@ -0,0 +1,2 @@ +# Copyright (c) OpenMMLab. All rights reserved. +filename = 'reserved.py' diff --git a/tests/data/config/yaml_config/base2.yaml b/tests/data/config/yaml_config/base2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b73902b39a3cf65231aaa667a530c54cfa03bde2 --- /dev/null +++ b/tests/data/config/yaml_config/base2.yaml @@ -0,0 +1 @@ +item2: {'a': 0} diff --git a/tests/data/config/yaml_config/simple.config.yaml b/tests/data/config/yaml_config/simple.config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5365b7142fa06524678f3fd2502a97f4080c1d6c --- /dev/null +++ b/tests/data/config/yaml_config/simple.config.yaml @@ -0,0 +1,4 @@ +item1: [1, 2] +item2: {'a': 0} +item3: True +item4: 'test' diff --git a/tests/data/config/yaml_config/simple_config.yaml b/tests/data/config/yaml_config/simple_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5365b7142fa06524678f3fd2502a97f4080c1d6c --- /dev/null +++ b/tests/data/config/yaml_config/simple_config.yaml @@ -0,0 +1,4 @@ +item1: [1, 2] +item2: {'a': 0} +item3: True +item4: 'test' diff --git a/tests/data/config/yaml_config/test_base.yaml b/tests/data/config/yaml_config/test_base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db16138e31cd8e9d5b681fd9191bc5ac5f04814d --- /dev/null +++ b/tests/data/config/yaml_config/test_base.yaml @@ -0,0 +1,6 @@ +_base_ : ['../py_config/base1.py', './base2.yaml', '../json_config/base3.json', '../py_config/base4.py'] +item3 : False +item4 : 'test' +item8 : '{{fileBasename}}' +item9 : {{ _base_.item2 }} +item10 : {{ _base_.item7.b.c }} diff --git a/tests/data/config/yaml_config/test_base_variables_nested.yaml b/tests/data/config/yaml_config/test_base_variables_nested.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b3431de1026b2e45de06141fe954bb42703a26c --- /dev/null +++ b/tests/data/config/yaml_config/test_base_variables_nested.yaml @@ -0,0 +1,15 @@ +_base_: ["../py_config/test_base_variables.py"] +base: "_base_.item8" +item11: {{ _base_.item8 }} +item12: {{ _base_.item9 }} +item13: {{ _base_.item10 }} +item14: {{ _base_.item1 }} +item15: + a: + b: {{ _base_.item2 }} + b: [{{ _base_.item3 }}] + c: [{{ _base_.item4 }}] + d: + - [e: {{ _base_.item5.a }}] + - {{ _base_.item6 }} + e: {{ _base_.item1 }} diff --git a/tests/data/config/yaml_config/test_predefined_var.yaml b/tests/data/config/yaml_config/test_predefined_var.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b3e46e81a0b44a8c029e034d7008fa68fdf1c7f --- /dev/null +++ b/tests/data/config/yaml_config/test_predefined_var.yaml @@ -0,0 +1 @@ +item1: '{{ fileDirname }}' diff --git a/tests/data/config/yaml_config/test_reserved_key.yaml b/tests/data/config/yaml_config/test_reserved_key.yaml new file mode 100644 index 0000000000000000000000000000000000000000..456aceb160d881c6a9b8d5b0eeeb07c31bc1e625 --- /dev/null +++ b/tests/data/config/yaml_config/test_reserved_key.yaml @@ -0,0 +1 @@ +filename: reserved.py diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..9f3d121c99924b5d9141946f9734d2038db24715 --- /dev/null +++ b/tests/test_config/test_config.py @@ -0,0 +1,627 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp +import platform +import sys +from importlib import import_module +from pathlib import Path + +import pytest + +from mmengine import Config, ConfigDict, DictAction +from mmengine.fileio import dump, load + + +class TestConfig: + data_path = osp.join(osp.dirname(osp.dirname(__file__)), 'data/') + + @pytest.mark.parametrize('file_format', ['py', 'json', 'yaml']) + def test_init(self, file_format): + # test init Config by __init__ + cfg = Config() + assert cfg.filename is None + assert cfg.text == '' + assert len(cfg) == 0 + assert cfg._cfg_dict == {} + + # test `cfg_dict` parameter + # `cfg_dict` is either dict or None + with pytest.raises(TypeError, match='cfg_dict must be a dict'): + Config([0, 1]) + + # test `filename` parameter + cfg_dict = dict( + item1=[1, 2], item2=dict(a=0), item3=True, item4='test') + cfg_file = osp.join( + self.data_path, + f'config/{file_format}_config/simple_config.{file_format}') + cfg = Config(cfg_dict, filename=cfg_file) + assert isinstance(cfg, Config) + assert cfg.filename == cfg_file + assert cfg.text == open(cfg_file, 'r').read() + + cfg_file = osp.join( + self.data_path, + f'config/{file_format}_config/test_reserved_key.{file_format}') + # reserved keys cannot be set in config + with pytest.raises( + KeyError, match='filename is reserved for config ' + 'file'): + Config.fromfile(cfg_file) + + def test_fromfile(self): + # test whether import `custom_imports` from cfg_file. + cfg_file = osp.join(self.data_path, 'config', + 'py_config/test_custom_import.py') + sys.path.append(osp.join(self.data_path, 'config/py_config')) + cfg = Config.fromfile(cfg_file, import_custom_modules=True) + assert isinstance(cfg, Config) + # If import successfully, os.environ[''TEST_VALUE''] will be + # set to 'test' + assert os.environ.pop('TEST_VALUE') == 'test' + Config.fromfile(cfg_file, import_custom_modules=False) + assert 'TEST_VALUE' not in os.environ + + @pytest.mark.parametrize('file_format', ['py', 'json', 'yaml']) + def test_fromstring(self, file_format): + filename = f'{file_format}_config/simple_config.{file_format}' + cfg_file = osp.join(self.data_path, 'config', filename) + file_format = osp.splitext(filename)[-1] + in_cfg = Config.fromfile(cfg_file) + + cfg_str = open(cfg_file, 'r').read() + out_cfg = Config.fromstring(cfg_str, file_format) + assert in_cfg._cfg_dict == out_cfg._cfg_dict + + # test pretty_text only supports py file format + # in_cfg.pretty_text is .py format, cannot be parsed to .json + if file_format != '.py': + with pytest.raises(Exception): + Config.fromstring(in_cfg.pretty_text, file_format) + + # error format + with pytest.raises(IOError): + Config.fromstring(cfg_str, '.xml') + + def test_magic_methods(self): + cfg_dict = dict( + item1=[1, 2], item2=dict(a=0), item3=True, item4='test') + filename = 'py_config/simple_config.py' + cfg_file = osp.join(self.data_path, 'config', filename) + cfg = Config.fromfile(cfg_file) + # len(cfg) + assert len(cfg) == 4 + # cfg.keys() + assert set(cfg.keys()) == set(cfg_dict.keys()) + assert set(cfg._cfg_dict.keys()) == set(cfg_dict.keys()) + # cfg.values() + for value in cfg.values(): + assert value in cfg_dict.values() + # cfg.items() + for name, value in cfg.items(): + assert name in cfg_dict + assert value in cfg_dict.values() + # cfg.field + assert cfg.item1 == cfg_dict['item1'] + assert cfg.item2 == cfg_dict['item2'] + assert cfg.item2.a == 0 + assert cfg.item3 == cfg_dict['item3'] + assert cfg.item4 == cfg_dict['item4'] + # accessing keys that do not exist will cause error + with pytest.raises(AttributeError): + cfg.not_exist + # field in cfg, cfg[field], cfg.get() + for name in ['item1', 'item2', 'item3', 'item4']: + assert name in cfg + assert cfg[name] == cfg_dict[name] + assert cfg.get(name) == cfg_dict[name] + assert cfg.get('not_exist') is None + assert cfg.get('not_exist', 0) == 0 + # accessing keys that do not exist will cause error + with pytest.raises(KeyError): + cfg['not_exist'] + assert 'item1' in cfg + assert 'not_exist' not in cfg + # cfg.update() + cfg.update(dict(item1=0)) + assert cfg.item1 == 0 + cfg.update(dict(item2=dict(a=1))) + assert cfg.item2.a == 1 + # test __setattr__ + cfg = Config() + cfg.item1 = [1, 2] + cfg.item2 = {'a': 0} + cfg['item5'] = {'a': {'b': None}} + assert cfg._cfg_dict['item1'] == [1, 2] + assert cfg.item1 == [1, 2] + assert cfg._cfg_dict['item2'] == {'a': 0} + assert cfg.item2.a == 0 + assert cfg._cfg_dict['item5'] == {'a': {'b': None}} + assert cfg.item5.a.b is None + + def test_merge_from_dict(self): + cfg_file = osp.join(self.data_path, + 'config/py_config/simple_config.py') + cfg = Config.fromfile(cfg_file) + input_options = {'item2.a': 1, 'item2.b': 0.1, 'item3': False} + cfg.merge_from_dict(input_options) + assert cfg.item2 == dict(a=1, b=0.1) + assert cfg.item3 is False + + cfg_file = osp.join(self.data_path, + 'config/py_config/test_merge_from_dict.py') + cfg = Config.fromfile(cfg_file) + + # Allow list keys + input_options = {'item.0.a': 1, 'item.1.b': 1} + cfg.merge_from_dict(input_options, allow_list_keys=True) + assert cfg.item == [{'a': 1}, {'b': 1, 'c': 0}] + + # allow_list_keys is False + input_options = {'item.0.a': 1, 'item.1.b': 1} + with pytest.raises(TypeError): + cfg.merge_from_dict(input_options, allow_list_keys=False) + + # Overflowed index number + input_options = {'item.2.a': 1} + with pytest.raises(KeyError): + cfg.merge_from_dict(input_options, allow_list_keys=True) + + def test_auto_argparser(self): + tmp = sys.argv[1] + sys.argv[1] = osp.join( + self.data_path, + 'config/py_config/test_merge_from_multiple_bases.py') + parser, cfg = Config.auto_argparser() + args = parser.parse_args() + assert args.config == sys.argv[1] + for key in cfg._cfg_dict.keys(): + if not isinstance(cfg[key], ConfigDict): + assert getattr(args, key) is None + # TODO currently do not support nested keys, bool args will be + # overwritten by int + sys.argv[1] = tmp + + @pytest.mark.parametrize('file_path', [ + 'config/py_config/simple_config.py', + 'config/py_config/test_merge_from_multiple_bases.py' + ]) + def test_dump(self, file_path, tmp_path): + cfg_file = osp.join(self.data_path, file_path) + cfg = Config.fromfile(cfg_file) + dump_py = tmp_path / 'simple_config.py' + dump_json = tmp_path / 'simple_config.json' + dump_yaml = tmp_path / 'simple_config.yaml' + + cfg.dump(dump_py) + cfg.dump(dump_json) + cfg.dump(dump_yaml) + + assert cfg.dump() == cfg.pretty_text + + assert open(dump_py, 'r').read() == cfg.pretty_text + assert open(dump_json, 'r').read() == cfg.pretty_text + assert open(dump_yaml, 'r').read() == cfg.pretty_text + + # test pickle + cfg_file = osp.join(self.data_path, + 'config/py_config/test_dump_pickle_support.py') + cfg = Config.fromfile(cfg_file) + + text_cfg_filename = tmp_path / '_text_config.py' + cfg.dump(text_cfg_filename) + text_cfg = Config.fromfile(text_cfg_filename) + + assert text_cfg._cfg_dict == cfg._cfg_dict + + cfg_file = osp.join(self.data_path, + 'config/py_config/test_dump_pickle_support.py') + cfg = Config.fromfile(cfg_file) + + pkl_cfg_filename = tmp_path / '_pickle.pkl' + dump(cfg, pkl_cfg_filename) + pkl_cfg = load(pkl_cfg_filename) + + assert pkl_cfg._cfg_dict == cfg._cfg_dict + + def test_pretty_text(self, tmp_path): + cfg_file = osp.join( + self.data_path, + 'config/py_config/test_merge_from_multiple_bases.py') + cfg = Config.fromfile(cfg_file) + text_cfg_filename = tmp_path / '_text_config.py' + with open(text_cfg_filename, 'w') as f: + f.write(cfg.pretty_text) + text_cfg = Config.fromfile(text_cfg_filename) + assert text_cfg._cfg_dict == cfg._cfg_dict + + def test_repr(self, tmp_path): + cfg_file = osp.join(self.data_path, + 'config/py_config/simple_config.py') + cfg = Config.fromfile(cfg_file) + tmp_txt = tmp_path / 'tmp.txt' + with open(tmp_txt, 'w') as f: + print(cfg, file=f) + with open(tmp_txt, 'r') as f: + assert f.read().strip() == f'Config (path: {cfg.filename}): ' \ + f'{cfg._cfg_dict.__repr__()}' + + def test_dict_action(self): + parser = argparse.ArgumentParser(description='Train a detector') + parser.add_argument( + '--options', nargs='+', action=DictAction, help='custom options') + # Nested brackets + args = parser.parse_args( + ['--options', 'item2.a=a,b', 'item2.b=[(a,b), [1,2], false]']) + out_dict = { + 'item2.a': ['a', 'b'], + 'item2.b': [('a', 'b'), [1, 2], False] + } + assert args.options == out_dict + # Single Nested brackets + args = parser.parse_args(['--options', 'item2.a=[[1]]']) + out_dict = {'item2.a': [[1]]} + assert args.options == out_dict + # Imbalance bracket will cause error + with pytest.raises(AssertionError): + parser.parse_args(['--options', 'item2.a=[(a,b), [1,2], false']) + # Normal values + args = parser.parse_args([ + '--options', 'item2.a=1', 'item2.b=0.1', 'item2.c=x', 'item3=false' + ]) + out_dict = { + 'item2.a': 1, + 'item2.b': 0.1, + 'item2.c': 'x', + 'item3': False + } + assert args.options == out_dict + cfg_file = osp.join(self.data_path, + 'config/py_config/simple_config.py') + cfg = Config.fromfile(cfg_file) + cfg.merge_from_dict(args.options) + assert cfg.item2 == dict(a=1, b=0.1, c='x') + assert cfg.item3 is False + + def test_validate_py_syntax(self, tmp_path): + tmp_cfg = tmp_path / 'tmp_config.py' + with open(tmp_cfg, 'w') as f: + f.write('dict(a=1,b=2.c=3)') + # Incorrect point in dict will cause error + with pytest.raises(SyntaxError): + Config._validate_py_syntax(tmp_cfg) + with open(tmp_cfg, 'w') as f: + f.write('[dict(a=1, b=2, c=(1, 2)]') + # Imbalance bracket will cause error + with pytest.raises(SyntaxError): + Config._validate_py_syntax(tmp_cfg) + with open(tmp_cfg, 'w') as f: + f.write('dict(a=1,b=2\nc=3)') + # Incorrect feed line in dict will cause error + with pytest.raises(SyntaxError): + Config._validate_py_syntax(tmp_cfg) + + def test_substitute_predefined_vars(self, tmp_path): + cfg_text = 'a={{fileDirname}}\n' \ + 'b={{fileBasename}}\n' \ + 'c={{fileBasenameNoExtension}}\n' \ + 'd={{fileExtname}}\n' + + cfg = tmp_path / 'tmp_cfg1.py' + substituted_cfg = tmp_path / 'tmp_cfg2.py' + + file_dirname = osp.dirname(cfg) + file_basename = osp.basename(cfg) + file_basename_no_extension = osp.splitext(file_basename)[0] + file_extname = osp.splitext(cfg)[1] + + expected_text = f'a={file_dirname}\n' \ + f'b={file_basename}\n' \ + f'c={file_basename_no_extension}\n' \ + f'd={file_extname}\n' + expected_text = expected_text.replace('\\', '/') + with open(cfg, 'w') as f: + f.write(cfg_text) + Config._substitute_predefined_vars(cfg, substituted_cfg) + + with open(substituted_cfg, 'r') as f: + assert f.read() == expected_text + + def test_pre_substitute_base_vars(self, tmp_path): + cfg_path = osp.join(self.data_path, 'config', + 'py_config/test_pre_substitute_base_vars.py') + tmp_cfg = tmp_path / 'tmp_cfg.py' + base_var_dict = Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + assert 'item6' in base_var_dict.values() + assert 'item10' in base_var_dict.values() + assert 'item11' in base_var_dict.values() + sys.path.append(str(tmp_path)) + cfg_module_dict = import_module(tmp_cfg.name.strip('.py')).__dict__ + assert cfg_module_dict['item22'].startswith('_item11') + assert cfg_module_dict['item23'].startswith('_item10') + assert cfg_module_dict['item25']['c'][1].startswith('_item6') + sys.path.pop() + + cfg_path = osp.join(self.data_path, 'config', + 'json_config/test_base.json') + tmp_cfg = tmp_path / 'tmp_cfg.json' + Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + cfg_module_dict = load(tmp_cfg) + assert cfg_module_dict['item9'].startswith('_item2') + assert cfg_module_dict['item10'].startswith('_item7') + + cfg_path = osp.join(self.data_path, 'config', + 'yaml_config/test_base.yaml') + tmp_cfg = tmp_path / 'tmp_cfg.yaml' + Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + cfg_module_dict = load(tmp_cfg) + assert cfg_module_dict['item9'].startswith('_item2') + assert cfg_module_dict['item10'].startswith('_item7') + + def test_substitute_base_vars(self): + cfg = dict( + item4='_item1.12345', + item5=dict(item3='1', item2='_item2_.fswf'), + item0=('_item0_.12ed21wq', 1)) + cfg_base = dict(item1=0, item2=[1, 2, 3], item0=(1, 2, 3)) + base_var_dict = { + '_item1.12345': 'item1', + '_item2_.fswf': 'item2', + '_item0_.12ed21wq': 'item0' + } + cfg = Config._substitute_base_vars(cfg, base_var_dict, cfg_base) + assert cfg['item4'] == cfg_base['item1'] + assert cfg['item5']['item2'] == cfg_base['item2'] + + def test_file2dict(self, tmp_path): + + # test error format config + tmp_cfg = tmp_path / 'tmp_cfg.xml' + tmp_cfg.write_text('exist') + # invalid config format + with pytest.raises(IOError): + Config.fromfile(tmp_cfg) + # invalid config file path + with pytest.raises(FileNotFoundError): + Config.fromfile('no_such_file.py') + + self._simple_load() + self._predefined_vars() + self._base_variables() + self._merge_from_base() + self._code_in_config() + self._merge_from_multiple_bases() + self._merge_delete() + self._merge_intermediate_variable() + self._merge_recursive_bases() + + def _simple_load(self): + # test load simple config + for file_format in ['py', 'json', 'yaml']: + for name in ['simple.config', 'simple_config']: + filename = f'{file_format}_config/{name}.{file_format}' + + cfg_file = osp.join(self.data_path, 'config', filename) + cfg_dict, cfg_text = Config._file2dict(cfg_file) + assert isinstance(cfg_text, str) + assert isinstance(cfg_dict, dict) + + def _get_file_path(self, file_path): + if platform.system() == 'Windows': + return file_path.replace('\\', '/') + else: + return file_path + + def _predefined_vars(self): + # test parse predefined_var in config + cfg_file = osp.join(self.data_path, + 'config/py_config/test_predefined_var.py') + path = osp.join(self.data_path, 'config/py_config') + + path = Path(path).as_posix() + cfg_dict_dst = dict( + item1='test_predefined_var.py', + item2=path, + item3='abc_test_predefined_var') + + assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + + # test `use_predefined_variable=False` + cfg_dict_ori = dict( + item1='{{fileBasename}}', + item2='{{ fileDirname}}', + item3='abc_{{ fileBasenameNoExtension }}') + + assert Config._file2dict(cfg_file, + False)[0]['item1'] == cfg_dict_ori['item1'] + assert Config._file2dict(cfg_file, + False)[0]['item2'] == cfg_dict_ori['item2'] + assert Config._file2dict(cfg_file, + False)[0]['item3'] == cfg_dict_ori['item3'] + + # test test_predefined_var.yaml + cfg_file = osp.join(self.data_path, + 'config/yaml_config/test_predefined_var.yaml') + + # test `use_predefined_variable=False` + assert Config._file2dict(cfg_file, + False)[0]['item1'] == '{{ fileDirname }}' + assert Config._file2dict(cfg_file)[0]['item1'] == self._get_file_path( + osp.dirname(cfg_file)) + + # test test_predefined_var.json + cfg_file = osp.join(self.data_path, + 'config/json_config/test_predefined_var.json') + + assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}' + assert Config.fromfile(cfg_file)['item1'] == self._get_file_path( + osp.dirname(cfg_file)) + + def _merge_from_base(self): + cfg_file = osp.join(self.data_path, + 'config/py_config/test_merge_from_base_single.py') + cfg_dict = Config._file2dict(cfg_file)[0] + + assert cfg_dict['item1'] == [2, 3] + assert cfg_dict['item2']['a'] == 1 + assert cfg_dict['item3'] is False + assert cfg_dict['item4'] == 'test_base' + # item3 is a dict in the child config but a boolean in base config + with pytest.raises(TypeError): + Config.fromfile( + osp.join(self.data_path, + 'config/py_config/test_merge_from_base_error.py')) + + def _merge_from_multiple_bases(self): + cfg_file = osp.join( + self.data_path, + 'config/py_config/test_merge_from_multiple_bases.py') + cfg_dict = Config._file2dict(cfg_file)[0] + + # cfg.fcfg_dictd + assert cfg_dict['item1'] == [1, 2] + assert cfg_dict['item2']['a'] == 0 + assert cfg_dict['item3'] is False + assert cfg_dict['item4'] == 'test' + assert cfg_dict['item5'] == dict(a=0, b=1) + assert cfg_dict['item6'] == [dict(a=0), dict(b=1)] + assert cfg_dict['item7'] == dict( + a=[0, 1, 2], b=dict(c=[3.1, 4.2, 5.3])) + # Redefine key + with pytest.raises(KeyError): + Config.fromfile( + osp.join(self.data_path, + 'config/py_config/test_merge_from_multiple_error.py')) + + def _base_variables(self): + for file in [ + 'py_config/test_base_variables.py', + 'json_config/test_base.json', 'yaml_config/test_base.yaml' + ]: + cfg_file = osp.join(self.data_path, 'config', file) + cfg_dict = Config._file2dict(cfg_file)[0] + + assert cfg_dict['item1'] == [1, 2] + assert cfg_dict['item2']['a'] == 0 + assert cfg_dict['item3'] is False + assert cfg_dict['item4'] == 'test' + assert cfg_dict['item5'] == dict(a=0, b=1) + assert cfg_dict['item6'] == [dict(a=0), dict(b=1)] + assert cfg_dict['item7'] == dict( + a=[0, 1, 2], b=dict(c=[3.1, 4.2, 5.3])) + assert cfg_dict['item8'] == file.split('/')[-1] + assert cfg_dict['item9'] == dict(a=0) + assert cfg_dict['item10'] == [3.1, 4.2, 5.3] + + # test nested base + for file in [ + 'py_config/test_base_variables_nested.py', + 'json_config/test_base_variables_nested.json', + 'yaml_config/test_base_variables_nested.yaml' + ]: + cfg_file = osp.join(self.data_path, 'config', file) + cfg_dict = Config._file2dict(cfg_file)[0] + + assert cfg_dict['base'] == '_base_.item8' + assert cfg_dict['item1'] == [1, 2] + assert cfg_dict['item2']['a'] == 0 + assert cfg_dict['item3'] is False + assert cfg_dict['item4'] == 'test' + assert cfg_dict['item5'] == dict(a=0, b=1) + assert cfg_dict['item6'] == [dict(a=0), dict(b=1)] + assert cfg_dict['item7'] == dict( + a=[0, 1, 2], b=dict(c=[3.1, 4.2, 5.3])) + assert cfg_dict['item8'] == 'test_base_variables.py' + assert cfg_dict['item9'] == dict(a=0) + assert cfg_dict['item10'] == [3.1, 4.2, 5.3] + assert cfg_dict['item11'] == 'test_base_variables.py' + assert cfg_dict['item12'] == dict(a=0) + assert cfg_dict['item13'] == [3.1, 4.2, 5.3] + assert cfg_dict['item14'] == [1, 2] + assert cfg_dict['item15'] == dict( + a=dict(b=dict(a=0)), + b=[False], + c=['test'], + d=[[{ + 'e': 0 + }], [{ + 'a': 0 + }, { + 'b': 1 + }]], + e=[1, 2]) + + # test reference assignment for py + cfg_file = osp.join( + self.data_path, + 'config/py_config/test_pre_substitute_base_vars.py') + cfg_dict = Config._file2dict(cfg_file)[0] + + assert cfg_dict['item21'] == 'test_base_variables.py' + assert cfg_dict['item22'] == 'test_base_variables.py' + assert cfg_dict['item23'] == [3.1, 4.2, 5.3] + assert cfg_dict['item24'] == [3.1, 4.2, 5.3] + assert cfg_dict['item25'] == dict( + a=dict(b=[3.1, 4.2, 5.3]), + b=[[3.1, 4.2, 5.3]], + c=[[{ + 'e': 'test_base_variables.py' + }], [{ + 'a': 0 + }, { + 'b': 1 + }]], + e='test_base_variables.py') + + def _merge_recursive_bases(self): + cfg_file = osp.join(self.data_path, + 'config/py_config/test_merge_recursive_bases.py') + cfg_dict = Config._file2dict(cfg_file)[0] + + assert cfg_dict['item1'] == [2, 3] + assert cfg_dict['item2']['a'] == 1 + assert cfg_dict['item3'] is False + assert cfg_dict['item4'] == 'test_recursive_bases' + + def _merge_delete(self): + cfg_file = osp.join(self.data_path, + 'config/py_config/test_merge_delete.py') + cfg_dict = Config._file2dict(cfg_file)[0] + # cfg.field + assert cfg_dict['item1'] == dict(a=0) + assert cfg_dict['item2'] == dict(a=0, b=0) + assert cfg_dict['item3'] is True + assert cfg_dict['item4'] == 'test' + assert '_delete_' not in cfg_dict['item2'] + + assert type(cfg_dict['item1']) == ConfigDict + assert type(cfg_dict['item2']) == dict + + def _merge_intermediate_variable(self): + + cfg_file = osp.join( + self.data_path, + 'config/py_config/test_merge_intermediate_variable_child.py') + cfg_dict = Config._file2dict(cfg_file)[0] + # cfg.field + assert cfg_dict['item1'] == [1, 2] + assert cfg_dict['item2'] == dict(a=0) + assert cfg_dict['item3'] is True + assert cfg_dict['item4'] == 'test' + assert cfg_dict['item_cfg'] == dict(b=2) + assert cfg_dict['item5'] == dict(cfg=dict(b=1)) + assert cfg_dict['item6'] == dict(cfg=dict(b=2)) + + def _code_in_config(self): + cfg_file = osp.join(self.data_path, + 'config/py_config/test_code_in_config.py') + cfg = Config.fromfile(cfg_file) + # cfg.field + assert cfg.cfg.item1 == [1, 2] + assert cfg.cfg.item2 == dict(a=0) + assert cfg.cfg.item3 is True + assert cfg.cfg.item4 == 'test' + assert cfg.item5 == 1