From 579f4fd15d7ecd1f9bb1a222f3c898e03e729a3d Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 10:07:59 +0100 Subject: [PATCH 01/13] Adding data value --- force_bdss/api.py | 1 + force_bdss/core/__init__.py | 0 force_bdss/core/data_value.py | 20 ++++++++++++++ force_bdss/core_evaluation_driver.py | 4 +-- .../csv_extractor_data_source.py | 17 +++++------- .../dummy/dummy_dakota/dakota_communicator.py | 14 +++++----- .../dummy_data_source/dummy_data_source.py | 10 ++----- .../dummy_kpi_calculator.py | 11 ++------ .../dummy/kpi_adder/kpi_adder_calculator.py | 26 ++++++++----------- force_bdss/mco/base_mco_communicator.py | 9 +++---- 10 files changed, 56 insertions(+), 56 deletions(-) create mode 100644 force_bdss/core/__init__.py create mode 100644 force_bdss/core/data_value.py diff --git a/force_bdss/api.py b/force_bdss/api.py index 5c71f3d..4f54ab3 100644 --- a/force_bdss/api.py +++ b/force_bdss/api.py @@ -1,5 +1,6 @@ from .base_extension_plugin import BaseExtensionPlugin # noqa from .ids import bundle_id, plugin_id # noqa +from .core.data_value import DataValue # noqa from .data_sources.base_data_source_model import BaseDataSourceModel # noqa from .data_sources.data_source_result import DataSourceResult # noqa diff --git a/force_bdss/core/__init__.py b/force_bdss/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/force_bdss/core/data_value.py b/force_bdss/core/data_value.py new file mode 100644 index 0000000..f03aca3 --- /dev/null +++ b/force_bdss/core/data_value.py @@ -0,0 +1,20 @@ +from traits.api import HasStrictTraits, Any, String + + +class DataValue(HasStrictTraits): + """Contains the parameters as passed from the MCO.""" + #: The CUBA types associated to the values + type = String() + + #: The user-defined names associated to the values. + name = String() + + #: The values as a single array. + value = Any() + + def __str__(self): + return """ + {} {} : {} + """.format(str(self.type), + str(self.name), + str(self.value)) diff --git a/force_bdss/core_evaluation_driver.py b/force_bdss/core_evaluation_driver.py index ee8de2b..8cfe761 100644 --- a/force_bdss/core_evaluation_driver.py +++ b/force_bdss/core_evaluation_driver.py @@ -31,13 +31,13 @@ class CoreEvaluationDriver(BaseCoreDriver): mco_bundle = mco_model.bundle mco_communicator = mco_bundle.create_communicator() - parameters = mco_communicator.receive_from_mco(mco_model) + mco_data_values = mco_communicator.receive_from_mco(mco_model) ds_results = [] for ds_model in workflow.data_sources: ds_bundle = ds_model.bundle data_source = ds_bundle.create_data_source() - ds_results.append(data_source.run(ds_model, parameters)) + ds_results.append(data_source.run(ds_model, mco_data_values)) kpi_results = [] for kpic_model in workflow.kpi_calculators: diff --git a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py index b42cf1b..d159906 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py @@ -1,7 +1,5 @@ import csv -import numpy -from force_bdss.api import BaseDataSource -from force_bdss.api import DataSourceResult +from force_bdss.api import BaseDataSource, DataValue class CSVExtractorDataSource(BaseDataSource): @@ -13,13 +11,12 @@ class CSVExtractorDataSource(BaseDataSource): continue if rowindex == model.row: - return DataSourceResult( - originator=self, - value_types=[model.cuba_type], - values=numpy.array( - parameters.values[0]+float( - row[model.column])).reshape(1, 1) - ) + return [ + DataValue( + type=model.cuba_type, + value=float(row[model.column]) + ) + ] return None return None diff --git a/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py b/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py index d5b53d9..958d5c2 100644 --- a/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py +++ b/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py @@ -1,7 +1,8 @@ import sys -import numpy -from force_bdss.api import DataSourceParameters, BaseMCOCommunicator +from force_bdss.api import ( + BaseMCOCommunicator, + DataValue) class DummyDakotaCommunicator(BaseMCOCommunicator): @@ -11,11 +12,10 @@ class DummyDakotaCommunicator(BaseMCOCommunicator): value_names = [p.value_name for p in model.parameters] value_types = [p.value_type for p in model.parameters] - return DataSourceParameters( - value_names=value_names, - value_types=value_types, - values=numpy.array(values) - ) + return [ + DataValue(type=type_, name=name, value=value) + for type_, name, value in zip( + value_types, value_names, values)] def send_to_mco(self, model, kpi_results): data = " ".join( diff --git a/force_bdss/core_plugins/dummy/dummy_data_source/dummy_data_source.py b/force_bdss/core_plugins/dummy/dummy_data_source/dummy_data_source.py index cd0446b..03c36de 100644 --- a/force_bdss/core_plugins/dummy/dummy_data_source/dummy_data_source.py +++ b/force_bdss/core_plugins/dummy/dummy_data_source/dummy_data_source.py @@ -1,12 +1,6 @@ -from force_bdss.api import BaseDataSource, DataSourceResult +from force_bdss.api import BaseDataSource class DummyDataSource(BaseDataSource): def run(self, model, parameters): - print(parameters) - return DataSourceResult( - originator=self, - value_names=parameters.value_names, - value_types=parameters.value_types, - values=parameters.values.reshape( - parameters.values.shape + (1,))) + return parameters diff --git a/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py b/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py index 0fe0077..f1a8a16 100644 --- a/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py +++ b/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py @@ -1,15 +1,8 @@ -from force_bdss.api import BaseKPICalculator, KPICalculatorResult, bundle_id +from force_bdss.api import BaseKPICalculator, bundle_id class DummyKPICalculator(BaseKPICalculator): id = bundle_id("enthought", "dummy_kpi_calculator") def run(self, model, data_source_results): - res = KPICalculatorResult( - originator=self, - value_names=data_source_results[0].value_names, - value_types=data_source_results[0].value_types, - values=data_source_results[0].values.reshape( - data_source_results[0].values.shape[:-1])) - - return res + return data_source_results diff --git a/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py b/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py index 4611683..2c24d62 100644 --- a/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py +++ b/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py @@ -1,22 +1,18 @@ -import numpy - -from force_bdss.api import BaseKPICalculator -from force_bdss.api import KPICalculatorResult +from force_bdss.api import BaseKPICalculator, DataValue class KPIAdderCalculator(BaseKPICalculator): def run(self, model, data_source_results): sum = 0.0 - for res in data_source_results: - try: - value_idx = res.value_types.index(model.cuba_type_in) - except ValueError: - continue + for ds_res in data_source_results: + for res in ds_res: + if res.type != model.cuba_type_in: + continue - sum += res.values[value_idx].sum() + sum += res.value - return KPICalculatorResult( - originator=self, - value_types=[model.cuba_type_out], - values=numpy.array([sum]) - ) + return [ + DataValue( + type=model.cuba_type_out, + value=sum + )] diff --git a/force_bdss/mco/base_mco_communicator.py b/force_bdss/mco/base_mco_communicator.py index 4cb0f3b..073b16c 100644 --- a/force_bdss/mco/base_mco_communicator.py +++ b/force_bdss/mco/base_mco_communicator.py @@ -29,8 +29,8 @@ class BaseMCOCommunicator(ABCHasStrictTraits): The conversion is specific to the format of the communication between the MCO and its evaluator program. - Must return a single DataSourceParameters object, containing - the parameters as passed by the MCO. + Must return a list of DataValue objects, containing the data passed + by the MCO. Parameters ---------- @@ -39,9 +39,8 @@ class BaseMCOCommunicator(ABCHasStrictTraits): Returns ------- - DataSourceParameters - An instance of the DataSourceParameters with the appropriate - information filled in. + List(DataValue) + A list of the DataValues with the appropriate information filled in """ @abc.abstractmethod -- GitLab From 5b32a0566aa8b92edd534bbc53db4d6de421f6e7 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 11:10:04 +0100 Subject: [PATCH 02/13] Restructuring base classes against the DataValue concept --- force_bdss/core/data_value.py | 18 +++-- force_bdss/data_sources/base_data_source.py | 14 ++-- .../data_sources/data_source_parameters.py | 26 -------- force_bdss/data_sources/data_source_result.py | 66 ------------------- force_bdss/kpi/base_kpi_calculator.py | 13 ++-- force_bdss/kpi/kpi_calculator_result.py | 54 --------------- force_bdss/mco/base_mco_communicator.py | 2 +- 7 files changed, 27 insertions(+), 166 deletions(-) delete mode 100644 force_bdss/data_sources/data_source_parameters.py delete mode 100644 force_bdss/data_sources/data_source_result.py delete mode 100644 force_bdss/kpi/kpi_calculator_result.py diff --git a/force_bdss/core/data_value.py b/force_bdss/core/data_value.py index f03aca3..fc20a49 100644 --- a/force_bdss/core/data_value.py +++ b/force_bdss/core/data_value.py @@ -1,17 +1,25 @@ -from traits.api import HasStrictTraits, Any, String +from traits.api import HasStrictTraits, Any, String, Int class DataValue(HasStrictTraits): - """Contains the parameters as passed from the MCO.""" - #: The CUBA types associated to the values + """Contains in-transit data between the various components (MCO/DS/KPI). + Each DataValue instance holds information about the CUBA type it + contains, the name as assigned by the user, and the value (which can be + anything. + """ + #: The CUBA type associated to the value. type = String() - #: The user-defined names associated to the values. + #: The user-defined name associated to the value. name = String() - #: The values as a single array. + #: The value. value = Any() + accuracy = Any() + + quality = Int() + def __str__(self): return """ {} {} : {} diff --git a/force_bdss/data_sources/base_data_source.py b/force_bdss/data_sources/base_data_source.py index 3a9fedb..6c6f5da 100644 --- a/force_bdss/data_sources/base_data_source.py +++ b/force_bdss/data_sources/base_data_source.py @@ -20,20 +20,20 @@ class BaseDataSource(ABCHasStrictTraits): @abc.abstractmethod def run(self, model, parameters): """ - Executes the KPI evaluation and returns the results it computes. - Reimplement this method in your specific KPI calculator. + Executes the Data Source evaluation and returns the results it + computes. Reimplement this method in your specific DataSource. Parameters ---------- model: BaseDataSourceModel The model of the DataSource, instantiated through create_model() - parameters: DataSourceParameters - a DataResultParameters instance containing the information coming - from the MCO + parameters: List(DataValue) + a list of DataValue objects containing the information needed + for the execution of the DataSource. Returns ------- - DataSourceResult - Instance that holds the results computed by this DataSource. + List(DataValue) + A list containing the computed Data Values. """ diff --git a/force_bdss/data_sources/data_source_parameters.py b/force_bdss/data_sources/data_source_parameters.py deleted file mode 100644 index 378ad26..0000000 --- a/force_bdss/data_sources/data_source_parameters.py +++ /dev/null @@ -1,26 +0,0 @@ -from traits.api import HasStrictTraits, Array, List, String - - -class DataSourceParameters(HasStrictTraits): - """Contains the parameters as passed from the MCO.""" - #: The user-defined names associated to the values. - value_names = List(String) - - #: The CUBA types associated to the values - value_types = List(String) - - #: The values as a single array. - values = Array(shape=(None,)) - - def __str__(self): - return """ - DataSourceParameters - value_names: - {} - value_types: - {} - values: - {} - """.format(str(self.value_names), - str(self.value_types), - str(self.values)) diff --git a/force_bdss/data_sources/data_source_result.py b/force_bdss/data_sources/data_source_result.py deleted file mode 100644 index d19c16b..0000000 --- a/force_bdss/data_sources/data_source_result.py +++ /dev/null @@ -1,66 +0,0 @@ -from traits.api import HasTraits, Array, ArrayOrNone, List, String, Instance - -from .base_data_source import BaseDataSource - - -class DataSourceResult(HasTraits): - """Represents the result of a DataSource evaluation. - - Note - ---- - Difference between accuracy and quality: - - uncertainty is a numerical quantity defining the accuracy of the value. - For example, a pressure can be 10.4 +/- 0.1, with 0.1 being the - accuracy - - quality is the level of importance and reliability of that value. - It should be considered as a weight of how much trust one should hold - on this information. - """ - - #: A reference to the DataSource that computed this result. - originator = Instance(BaseDataSource) - - #: The user-defined names associated to each result. - value_names = List(String) - - #: The CUBA types of each value. - value_types = List(String) - - #: The values for each entry. Note that this is a NxM array, allowing - #: to propagate more than single scalar values associated to a given value. - values = Array(shape=(None, None)) - - #: If present, the numerical accuracy of the above values. - accuracy = ArrayOrNone(shape=(None, None)) - - #: If present, the assessed quality of the above values. - quality = ArrayOrNone(shape=(None, None)) - - def __str__(self): - return """ - DataSourceResults - - originator: - {} - - value_names: - {} - - value_types: - {} - - values: - {} - - Accuracy: - {} - - Quality: - {} - """.format( - self.originator, - self.value_names, - self.value_types, - self.values, - self.accuracy, - self.quality) diff --git a/force_bdss/kpi/base_kpi_calculator.py b/force_bdss/kpi/base_kpi_calculator.py index 0d86647..1bb1bf4 100644 --- a/force_bdss/kpi/base_kpi_calculator.py +++ b/force_bdss/kpi/base_kpi_calculator.py @@ -18,7 +18,7 @@ class BaseKPICalculator(ABCHasStrictTraits): super(BaseKPICalculator, self).__init__(*args, **kwargs) @abc.abstractmethod - def run(self, model, data_source_results): + def run(self, model, data_values): """ Executes the KPI evaluation and returns the results it computes. Reimplement this method in your specific KPI calculator. @@ -29,13 +29,12 @@ class BaseKPICalculator(ABCHasStrictTraits): The model of the KPI Calculator, instantiated through create_model() - data_source_results: - a list of DataSourceResult instances containing the results of the - evaluation. Each DataSourceResult contains the results from one - specific DataSource. + data_values: + a list of DataValue instances containing data from the + MCO and DataSources. Returns ------- - KPICalculatorResult - Instance that holds the results computed by this KPICalculator. + List[DataValue]: + The result of this KPI evaluation, as a list of DataValues. """ diff --git a/force_bdss/kpi/kpi_calculator_result.py b/force_bdss/kpi/kpi_calculator_result.py deleted file mode 100644 index dce0df9..0000000 --- a/force_bdss/kpi/kpi_calculator_result.py +++ /dev/null @@ -1,54 +0,0 @@ -from traits.api import HasTraits, List, Array, ArrayOrNone, String, Instance - -from .base_kpi_calculator import BaseKPICalculator - - -class KPICalculatorResult(HasTraits): - """Contains the results from a single KPICalculator evaluation""" - - #: The originating KPI calculator - originator = Instance(BaseKPICalculator) - - #: The user-attributed names of each computed value - value_names = List(String) - - #: The CUBA types of each of the computed values - value_types = List(String) - - #: The values, as a single array of values - values = Array(shape=(None, )) - - #: If present, the numerical accuracy of the above values. - accuracy = ArrayOrNone(shape=(None, )) - - #: If present, the quality of the above values. - quality = ArrayOrNone(shape=(None, )) - - def __str__(self): - return """ - DataSourceResults - - originator: - {} - - value_names: - {} - - value_types: - {} - - values: - {} - - Accuracy: - {} - - Quality: - {} - """.format( - self.originator, - self.value_names, - self.value_types, - self.values, - self.accuracy, - self.quality) diff --git a/force_bdss/mco/base_mco_communicator.py b/force_bdss/mco/base_mco_communicator.py index 073b16c..cad5ef3 100644 --- a/force_bdss/mco/base_mco_communicator.py +++ b/force_bdss/mco/base_mco_communicator.py @@ -55,6 +55,6 @@ class BaseMCOCommunicator(ABCHasStrictTraits): model: BaseMCOModel The model of the optimizer, instantiated through create_model() - kpi_results: List(KPICalculatorResult) + kpi_results: List(DataValue) A list of KPI calculator results, one per each KPI calculator. """ -- GitLab From 35bae43e9751fb82f471375d27aa61925a06e7c7 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 13:11:33 +0100 Subject: [PATCH 03/13] Fixed tests --- force_bdss/api.py | 3 --- force_bdss/core_evaluation_driver.py | 4 ++-- .../dummy/csv_extractor/csv_extractor_data_source.py | 4 ++-- .../tests/test_csv_extractor_data_source.py | 12 ++++++++---- .../dummy/dummy_dakota/dakota_communicator.py | 6 ++---- .../dummy_dakota/tests/test_dakota_communicator.py | 9 ++++----- .../dummy_kpi_calculator/dummy_kpi_calculator.py | 4 +--- .../dummy/kpi_adder/kpi_adder_calculator.py | 10 +++++----- force_bdss/tests/test_core_evaluation_driver.py | 6 +++--- 9 files changed, 27 insertions(+), 31 deletions(-) diff --git a/force_bdss/api.py b/force_bdss/api.py index 4f54ab3..7480b1b 100644 --- a/force_bdss/api.py +++ b/force_bdss/api.py @@ -3,14 +3,11 @@ from .ids import bundle_id, plugin_id # noqa from .core.data_value import DataValue # noqa from .data_sources.base_data_source_model import BaseDataSourceModel # noqa -from .data_sources.data_source_result import DataSourceResult # noqa -from .data_sources.data_source_parameters import DataSourceParameters # noqa from .data_sources.base_data_source import BaseDataSource # noqa from .data_sources.base_data_source_bundle import BaseDataSourceBundle # noqa from .data_sources.i_data_source_bundle import IDataSourceBundle # noqa from .kpi.base_kpi_calculator import BaseKPICalculator # noqa -from .kpi.kpi_calculator_result import KPICalculatorResult # noqa from .kpi.base_kpi_calculator_model import BaseKPICalculatorModel # noqa from .kpi.base_kpi_calculator_bundle import BaseKPICalculatorBundle # noqa from .kpi.i_kpi_calculator_bundle import IKPICalculatorBundle # noqa diff --git a/force_bdss/core_evaluation_driver.py b/force_bdss/core_evaluation_driver.py index 8cfe761..d6f7181 100644 --- a/force_bdss/core_evaluation_driver.py +++ b/force_bdss/core_evaluation_driver.py @@ -37,12 +37,12 @@ class CoreEvaluationDriver(BaseCoreDriver): for ds_model in workflow.data_sources: ds_bundle = ds_model.bundle data_source = ds_bundle.create_data_source() - ds_results.append(data_source.run(ds_model, mco_data_values)) + ds_results.extend(data_source.run(ds_model, mco_data_values)) kpi_results = [] for kpic_model in workflow.kpi_calculators: kpic_bundle = kpic_model.bundle kpi_calculator = kpic_bundle.create_kpi_calculator() - kpi_results.append(kpi_calculator.run(kpic_model, ds_results)) + kpi_results.extend(kpi_calculator.run(kpic_model, ds_results)) mco_communicator.send_to_mco(mco_model, kpi_results) diff --git a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py index d159906..3b48584 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py @@ -18,5 +18,5 @@ class CSVExtractorDataSource(BaseDataSource): ) ] - return None - return None + return [] + return [] diff --git a/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py index 1be1beb..0df1be2 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py @@ -1,12 +1,12 @@ import unittest +from force_bdss.core.data_value import DataValue from force_bdss.core_plugins.dummy.csv_extractor.csv_extractor_data_source \ import CSVExtractorDataSource from force_bdss.core_plugins.dummy.csv_extractor.csv_extractor_model import \ CSVExtractorModel from force_bdss.data_sources.base_data_source_bundle import \ BaseDataSourceBundle -from force_bdss.data_sources.data_source_result import DataSourceResult from force_bdss.tests import fixtures try: @@ -27,7 +27,11 @@ class TestCSVExtractorDataSource(unittest.TestCase): ds = CSVExtractorDataSource(self.bundle) model = CSVExtractorModel(self.bundle) model.filename = fixtures.get("foo.csv") - mock_params = mock.Mock() - mock_params.values = [1.0] + model.row = 3 + model.column = 5 + mock_params = [] result = ds.run(model, mock_params) - self.assertIsInstance(result, DataSourceResult) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], DataValue) + self.assertEqual(result[0].value, 42) diff --git a/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py b/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py index 958d5c2..e1d10fd 100644 --- a/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py +++ b/force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py @@ -17,8 +17,6 @@ class DummyDakotaCommunicator(BaseMCOCommunicator): for type_, name, value in zip( value_types, value_names, values)] - def send_to_mco(self, model, kpi_results): - data = " ".join( - [" ".join(list(map(str, r.values.tolist()))) for r in kpi_results] - ) + def send_to_mco(self, model, data_values): + data = " ".join([str(dv.value) for dv in data_values]) sys.stdout.write(data) diff --git a/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py b/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py index 6e74d74..7954507 100644 --- a/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py +++ b/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py @@ -9,7 +9,6 @@ from envisage.plugin import Plugin from force_bdss.core_plugins.dummy.dummy_dakota.dakota_bundle import ( DummyDakotaBundle) -from force_bdss.data_sources.data_source_parameters import DataSourceParameters from force_bdss.mco.parameters.base_mco_parameter_factory import \ BaseMCOParameterFactory @@ -31,7 +30,7 @@ class TestDakotaCommunicator(unittest.TestCase): stdin.read.return_value = "1" data = comm.receive_from_mco(model) - self.assertIsInstance(data, DataSourceParameters) - self.assertEqual(len(data.value_names), 1) - self.assertEqual(len(data.value_types), 1) - self.assertEqual(len(data.values), 1) + self.assertIsInstance(data, list) + self.assertEqual(len(data), 1) + self.assertEqual(data[0].value, 1) + self.assertEqual(data[0].type, "") diff --git a/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py b/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py index f1a8a16..bd10473 100644 --- a/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py +++ b/force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py @@ -1,8 +1,6 @@ -from force_bdss.api import BaseKPICalculator, bundle_id +from force_bdss.api import BaseKPICalculator class DummyKPICalculator(BaseKPICalculator): - id = bundle_id("enthought", "dummy_kpi_calculator") - def run(self, model, data_source_results): return data_source_results diff --git a/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py b/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py index 2c24d62..94128b3 100644 --- a/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py +++ b/force_bdss/core_plugins/dummy/kpi_adder/kpi_adder_calculator.py @@ -4,12 +4,12 @@ from force_bdss.api import BaseKPICalculator, DataValue class KPIAdderCalculator(BaseKPICalculator): def run(self, model, data_source_results): sum = 0.0 - for ds_res in data_source_results: - for res in ds_res: - if res.type != model.cuba_type_in: - continue - sum += res.value + for res in data_source_results: + if res.type != model.cuba_type_in: + continue + + sum += res.value return [ DataValue( diff --git a/force_bdss/tests/test_core_evaluation_driver.py b/force_bdss/tests/test_core_evaluation_driver.py index 5be1849..3ce8480 100644 --- a/force_bdss/tests/test_core_evaluation_driver.py +++ b/force_bdss/tests/test_core_evaluation_driver.py @@ -53,7 +53,7 @@ class NullMCOCommunicator(BaseMCOCommunicator): pass def receive_from_mco(self, model): - pass + return [] class NullMCOBundle(BaseMCOBundle): @@ -78,7 +78,7 @@ class NullKPICalculatorModel(BaseKPICalculatorModel): class NullKPICalculator(BaseKPICalculator): def run(self, model, data_source_results): - pass + return [] class NullKPICalculatorBundle(BaseKPICalculatorBundle): @@ -95,7 +95,7 @@ class NullDataSourceModel(BaseDataSourceModel): class NullDataSource(BaseDataSource): def run(self, model, parameters): - pass + return [] class NullDataSourceBundle(BaseDataSourceBundle): -- GitLab From b8ccf5fba0872f4bc4fd11b9e960b25350afe533 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 13:18:43 +0100 Subject: [PATCH 04/13] Added test for data value --- force_bdss/core/data_value.py | 6 ++++-- force_bdss/core/tests/__init__.py | 0 force_bdss/core/tests/test_data_value.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 force_bdss/core/tests/__init__.py create mode 100644 force_bdss/core/tests/test_data_value.py diff --git a/force_bdss/core/data_value.py b/force_bdss/core/data_value.py index fc20a49..f360446 100644 --- a/force_bdss/core/data_value.py +++ b/force_bdss/core/data_value.py @@ -1,4 +1,4 @@ -from traits.api import HasStrictTraits, Any, String, Int +from traits.api import HasStrictTraits, Any, String, Int, Enum class DataValue(HasStrictTraits): @@ -16,9 +16,11 @@ class DataValue(HasStrictTraits): #: The value. value = Any() + # The numerical accuracy of the value. accuracy = Any() - quality = Int() + #: A flag for the quality of the data. + quality = Enum("AVERAGE", "POOR", "GOOD") def __str__(self): return """ diff --git a/force_bdss/core/tests/__init__.py b/force_bdss/core/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/force_bdss/core/tests/test_data_value.py b/force_bdss/core/tests/test_data_value.py new file mode 100644 index 0000000..c4aa549 --- /dev/null +++ b/force_bdss/core/tests/test_data_value.py @@ -0,0 +1,13 @@ +import unittest + +from force_bdss.core.data_value import DataValue + + +class TestDataValue(unittest.TestCase): + def test_initialization(self): + dv = DataValue() + self.assertEqual(dv.type, "") + self.assertEqual(dv.value, None) + self.assertEqual(dv.accuracy, None) + self.assertEqual(dv.quality, "AVERAGE") + -- GitLab From 018ca1791d52f36c9a4629c8e88d8934a447c268 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 13:20:29 +0100 Subject: [PATCH 05/13] Documentation --- doc/source/api/force_bdss.core.rst | 29 ++++++++++++++++++++++ doc/source/api/force_bdss.core.tests.rst | 22 ++++++++++++++++ doc/source/api/force_bdss.data_sources.rst | 16 ------------ doc/source/api/force_bdss.kpi.rst | 8 ------ doc/source/api/force_bdss.rst | 1 + 5 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 doc/source/api/force_bdss.core.rst create mode 100644 doc/source/api/force_bdss.core.tests.rst diff --git a/doc/source/api/force_bdss.core.rst b/doc/source/api/force_bdss.core.rst new file mode 100644 index 0000000..cea914b --- /dev/null +++ b/doc/source/api/force_bdss.core.rst @@ -0,0 +1,29 @@ +force_bdss.core package +======================= + +Subpackages +----------- + +.. toctree:: + + force_bdss.core.tests + +Submodules +---------- + +force_bdss.core.data_value module +--------------------------------- + +.. automodule:: force_bdss.core.data_value + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: force_bdss.core + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/force_bdss.core.tests.rst b/doc/source/api/force_bdss.core.tests.rst new file mode 100644 index 0000000..f5a4f03 --- /dev/null +++ b/doc/source/api/force_bdss.core.tests.rst @@ -0,0 +1,22 @@ +force_bdss.core.tests package +============================= + +Submodules +---------- + +force_bdss.core.tests.test_data_value module +-------------------------------------------- + +.. automodule:: force_bdss.core.tests.test_data_value + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: force_bdss.core.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/api/force_bdss.data_sources.rst b/doc/source/api/force_bdss.data_sources.rst index 7df2d7d..0d7330f 100644 --- a/doc/source/api/force_bdss.data_sources.rst +++ b/doc/source/api/force_bdss.data_sources.rst @@ -35,22 +35,6 @@ force_bdss.data_sources.base_data_source_model module :undoc-members: :show-inheritance: -force_bdss.data_sources.data_source_parameters module ------------------------------------------------------ - -.. automodule:: force_bdss.data_sources.data_source_parameters - :members: - :undoc-members: - :show-inheritance: - -force_bdss.data_sources.data_source_result module -------------------------------------------------- - -.. automodule:: force_bdss.data_sources.data_source_result - :members: - :undoc-members: - :show-inheritance: - force_bdss.data_sources.i_data_source_bundle module --------------------------------------------------- diff --git a/doc/source/api/force_bdss.kpi.rst b/doc/source/api/force_bdss.kpi.rst index b90b1b5..55b7515 100644 --- a/doc/source/api/force_bdss.kpi.rst +++ b/doc/source/api/force_bdss.kpi.rst @@ -43,14 +43,6 @@ force_bdss.kpi.i_kpi_calculator_bundle module :undoc-members: :show-inheritance: -force_bdss.kpi.kpi_calculator_result module -------------------------------------------- - -.. automodule:: force_bdss.kpi.kpi_calculator_result - :members: - :undoc-members: - :show-inheritance: - Module contents --------------- diff --git a/doc/source/api/force_bdss.rst b/doc/source/api/force_bdss.rst index e558a1c..701918c 100644 --- a/doc/source/api/force_bdss.rst +++ b/doc/source/api/force_bdss.rst @@ -7,6 +7,7 @@ Subpackages .. toctree:: force_bdss.cli + force_bdss.core force_bdss.core_plugins force_bdss.data_sources force_bdss.io -- GitLab From 3e756a2ad3a8e5c1334a51bff40dd05c4cec93d9 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 13:25:00 +0100 Subject: [PATCH 06/13] Raise error if the csv data is not found --- .../dummy/csv_extractor/csv_extractor_data_source.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py index 3b48584..72bcdec 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py @@ -18,5 +18,7 @@ class CSVExtractorDataSource(BaseDataSource): ) ] - return [] - return [] + raise IndexError("Could not find specified data. " + "Unexistent column.") + + raise IndexError("Could not find specified data. Unexistent row.") -- GitLab From e9353feea58299448ff64c45475085a1b093f47b Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 13:30:44 +0100 Subject: [PATCH 07/13] Tested CSV extractor for exceptions --- .../csv_extractor/csv_extractor_data_source.py | 10 ++++------ .../tests/test_csv_extractor_data_source.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py index 72bcdec..471f3ac 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/csv_extractor_data_source.py @@ -9,16 +9,14 @@ class CSVExtractorDataSource(BaseDataSource): for rowindex, row in enumerate(reader): if rowindex < model.row: continue - - if rowindex == model.row: + elif rowindex == model.row: return [ DataValue( type=model.cuba_type, value=float(row[model.column]) ) ] + else: + break - raise IndexError("Could not find specified data. " - "Unexistent column.") - - raise IndexError("Could not find specified data. Unexistent row.") + raise IndexError("Could not find specified data.") diff --git a/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py index 0df1be2..f619f45 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py @@ -35,3 +35,20 @@ class TestCSVExtractorDataSource(unittest.TestCase): self.assertEqual(len(result), 1) self.assertIsInstance(result[0], DataValue) self.assertEqual(result[0].value, 42) + + def test_run_with_exception(self): + ds = CSVExtractorDataSource(self.bundle) + model = CSVExtractorModel(self.bundle) + model.filename = fixtures.get("foo.csv") + mock_params = [] + model.row = 30 + model.column = 5 + with self.assertRaises(IndexError): + ds.run(model, mock_params) + + model.row = 3 + model.column = 50 + with self.assertRaises(IndexError): + ds.run(model, mock_params) + + -- GitLab From 6aeefad0e8d4e951bee42aeedb55a9186c3ac3ef Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 13:38:22 +0100 Subject: [PATCH 08/13] flake --- force_bdss/core/data_value.py | 2 +- force_bdss/core/tests/test_data_value.py | 1 - .../dummy/csv_extractor/tests/test_csv_extractor_data_source.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/force_bdss/core/data_value.py b/force_bdss/core/data_value.py index f360446..7aa84b4 100644 --- a/force_bdss/core/data_value.py +++ b/force_bdss/core/data_value.py @@ -1,4 +1,4 @@ -from traits.api import HasStrictTraits, Any, String, Int, Enum +from traits.api import HasStrictTraits, Any, String, Enum class DataValue(HasStrictTraits): diff --git a/force_bdss/core/tests/test_data_value.py b/force_bdss/core/tests/test_data_value.py index c4aa549..5978706 100644 --- a/force_bdss/core/tests/test_data_value.py +++ b/force_bdss/core/tests/test_data_value.py @@ -10,4 +10,3 @@ class TestDataValue(unittest.TestCase): self.assertEqual(dv.value, None) self.assertEqual(dv.accuracy, None) self.assertEqual(dv.quality, "AVERAGE") - diff --git a/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py b/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py index f619f45..a498b9e 100644 --- a/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py +++ b/force_bdss/core_plugins/dummy/csv_extractor/tests/test_csv_extractor_data_source.py @@ -50,5 +50,3 @@ class TestCSVExtractorDataSource(unittest.TestCase): model.column = 50 with self.assertRaises(IndexError): ds.run(model, mock_params) - - -- GitLab From 0123e2af8d564f9299ed832df834c78c86fb3b2c Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 14:08:08 +0100 Subject: [PATCH 09/13] Coverage --- .../tests/test_kpi_adder_calculator.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py diff --git a/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py b/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py new file mode 100644 index 0000000..6f325ef --- /dev/null +++ b/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py @@ -0,0 +1,27 @@ +import unittest + +from force_bdss.core.data_value import DataValue +from force_bdss.core_plugins.dummy.kpi_adder.kpi_adder_model import \ + KPIAdderModel +from force_bdss.kpi.base_kpi_calculator_bundle import BaseKPICalculatorBundle + +try: + import mock +except ImportError: + from unittest import mock + +from force_bdss.core_plugins.dummy.kpi_adder.kpi_adder_calculator import \ + KPIAdderCalculator + + +class TestKPIAdderCalculator(unittest.TestCase): + def test_basic_functionality(self): + kpic = KPIAdderCalculator(mock.Mock(spec=BaseKPICalculatorBundle)) + model = KPIAdderModel(mock.Mock(spec=BaseKPICalculatorBundle)) + model.cuba_type_in = "PRESSURE" + model.cuba_type_out = "TOTAL_PRESSURE" + dv1 = DataValue(type="PRESSURE", value=10) + dv2 = DataValue(type="PRESSURE", value=30) + res = kpic.run(model, [dv1, dv2]) + self.assertEqual(res[0].type, "TOTAL_PRESSURE") + self.assertEqual(res[0].value, 40) -- GitLab From 4988002d875fcce013cbf1e433cef6405bc03bbd Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 14:12:59 +0100 Subject: [PATCH 10/13] Testing __str__ method --- force_bdss/core/data_value.py | 14 +++++++++----- force_bdss/core/tests/test_data_value.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/force_bdss/core/data_value.py b/force_bdss/core/data_value.py index 7aa84b4..c124053 100644 --- a/force_bdss/core/data_value.py +++ b/force_bdss/core/data_value.py @@ -23,8 +23,12 @@ class DataValue(HasStrictTraits): quality = Enum("AVERAGE", "POOR", "GOOD") def __str__(self): - return """ - {} {} : {} - """.format(str(self.type), - str(self.name), - str(self.value)) + + s = "{} {} = {}".format( + str(self.type), str(self.name), str(self.value)) + + if self.accuracy is not None: + s += " +/- {}".format(str(self.accuracy)) + + s += " ({})".format(str(self.quality)) + return s diff --git a/force_bdss/core/tests/test_data_value.py b/force_bdss/core/tests/test_data_value.py index 5978706..1ee866c 100644 --- a/force_bdss/core/tests/test_data_value.py +++ b/force_bdss/core/tests/test_data_value.py @@ -10,3 +10,13 @@ class TestDataValue(unittest.TestCase): self.assertEqual(dv.value, None) self.assertEqual(dv.accuracy, None) self.assertEqual(dv.quality, "AVERAGE") + + def test_string(self): + dv = DataValue(type="PRESSURE", name="p1", value=10) + self.assertEqual(str(dv), "PRESSURE p1 = 10 (AVERAGE)") + dv = DataValue(type="PRESSURE", + name="p1", + value=10, + accuracy=0.1, + quality="POOR") + self.assertEqual(str(dv), "PRESSURE p1 = 10 +/- 0.1 (POOR)") -- GitLab From db678c7ec8a38bec4b85cc01c8590d45ae6f4d53 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 14:27:56 +0100 Subject: [PATCH 11/13] Added test for send_to_mco --- .../tests/test_dakota_communicator.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py b/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py index 7954507..03e8e85 100644 --- a/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py +++ b/force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py @@ -1,5 +1,7 @@ import unittest +from force_bdss.core.data_value import DataValue + try: import mock except ImportError: @@ -34,3 +36,16 @@ class TestDakotaCommunicator(unittest.TestCase): self.assertEqual(len(data), 1) self.assertEqual(data[0].value, 1) self.assertEqual(data[0].type, "") + + def test_send_to_mco(self): + bundle = DummyDakotaBundle(mock.Mock(spec=Plugin)) + model = bundle.create_model() + comm = bundle.create_communicator() + + with mock.patch("sys.stdout") as stdout: + dv = DataValue(value=100) + comm.send_to_mco(model, [dv, dv]) + self.assertEqual(stdout.write.call_args[0][0], '100 100') + + comm.send_to_mco(model, []) + self.assertEqual(stdout.write.call_args[0][0], '') -- GitLab From 0a27c45e54f9e70d3cc067f1a4b61a3250d0ee27 Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 14:40:52 +0100 Subject: [PATCH 12/13] coverage --- .../tests/test_dummy_kpi_calculator.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 force_bdss/core_plugins/dummy/dummy_kpi_calculator/tests/test_dummy_kpi_calculator.py diff --git a/force_bdss/core_plugins/dummy/dummy_kpi_calculator/tests/test_dummy_kpi_calculator.py b/force_bdss/core_plugins/dummy/dummy_kpi_calculator/tests/test_dummy_kpi_calculator.py new file mode 100644 index 0000000..7226b3d --- /dev/null +++ b/force_bdss/core_plugins/dummy/dummy_kpi_calculator/tests/test_dummy_kpi_calculator.py @@ -0,0 +1,23 @@ +import unittest + +try: + import mock +except ImportError: + from unittest import mock + +from force_bdss.core_plugins.dummy.dummy_kpi_calculator.dummy_kpi_calculator \ + import DummyKPICalculator +from force_bdss.core_plugins.dummy.dummy_kpi_calculator \ + .dummy_kpi_calculator_bundle import DummyKPICalculatorBundle +from force_bdss.core_plugins.dummy.dummy_kpi_calculator \ + .dummy_kpi_calculator_model import DummyKPICalculatorModel + + +class TestDummyKPICalculator(unittest.TestCase): + def test_run(self): + bundle = mock.Mock(spec=DummyKPICalculatorBundle) + kpic = DummyKPICalculator(bundle) + model = DummyKPICalculatorModel(bundle) + input_ = [] + output = kpic.run(model, []) + self.assertEqual(input_, output) -- GitLab From 72da5a0769145b072e64061b6f76aaeaff2706bd Mon Sep 17 00:00:00 2001 From: Stefano Borini <sborini@enthought.com> Date: Tue, 1 Aug 2017 14:41:36 +0100 Subject: [PATCH 13/13] coverage --- .../dummy/kpi_adder/tests/test_kpi_adder_calculator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py b/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py index 6f325ef..93fc276 100644 --- a/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py +++ b/force_bdss/core_plugins/dummy/kpi_adder/tests/test_kpi_adder_calculator.py @@ -22,6 +22,7 @@ class TestKPIAdderCalculator(unittest.TestCase): model.cuba_type_out = "TOTAL_PRESSURE" dv1 = DataValue(type="PRESSURE", value=10) dv2 = DataValue(type="PRESSURE", value=30) - res = kpic.run(model, [dv1, dv2]) + dv3 = DataValue(type="VOLUME", value=100) + res = kpic.run(model, [dv1, dv2, dv3]) self.assertEqual(res[0].type, "TOTAL_PRESSURE") self.assertEqual(res[0].value, 40) -- GitLab