Merge pull request #45 from force-h2020/mco_parameters

Added support for MCO named parameters.

force_bdss/base_core_driver.py

@@ -7,6 +7,9 @@ from .bundle_registry_plugin import (
 )
 from .io.workflow_reader import WorkflowReader
 from .workspecs.workflow import Workflow
+from .mco.parameters.mco_parameter_factory_registry import (
+    MCOParameterFactoryRegistry)
+from .mco.parameters.core_mco_parameters import all_core_factories
 
 
 class BaseCoreDriver(Plugin):
@@ -14,15 +17,27 @@ class BaseCoreDriver(Plugin):
     or the evaluation.
     """
 
+    #: The registry of the bundles.
     bundle_registry = Instance(BundleRegistryPlugin)
 
+    #: The registry of the MCO parameters
+    parameter_factory_registry = Instance(MCOParameterFactoryRegistry)
+
     #: Deserialized content of the workflow file.
     workflow = Instance(Workflow)
 
     def _bundle_registry_default(self):
         return self.application.get_plugin(BUNDLE_REGISTRY_PLUGIN_ID)
 
+    def _parameter_factory_registry_default(self):
+        registry = MCOParameterFactoryRegistry()
+        for f in all_core_factories():
+            registry.register(f)
+
+        return registry
+
     def _workflow_default(self):
-        reader = WorkflowReader(self.bundle_registry)
+        reader = WorkflowReader(self.bundle_registry,
+                                self.parameter_factory_registry)
         with open(self.application.workflow_filepath) as f:
             return reader.read(f)

force_bdss/cli/tests/test_execution.py

@@ -28,6 +28,16 @@ class TestExecution(unittest.TestCase):
             out = subprocess.check_call(["force_bdss", "test_csv.json"])
             self.assertEqual(out, 0)
 
+    def test_plain_invocation_evaluate(self):
+        with cd(fixtures.dirpath()):
+            proc = subprocess.Popen([
+                "force_bdss", "--evaluate", "test_csv.json"],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE)
+            proc.communicate(b"1")
+            retcode = proc.wait()
+            self.assertEqual(retcode, 0)
+
     def test_unsupported_file_input(self):
         with cd(fixtures.dirpath()):
             with self.assertRaises(subprocess.CalledProcessError):

force_bdss/core_plugins/dummy/dummy_dakota/dakota_communicator.py

@@ -8,8 +8,12 @@ class DummyDakotaCommunicator(BaseMCOCommunicator):
     def receive_from_mco(self):
         data = sys.stdin.read()
         values = list(map(float, data.split()))
+        value_names = [p.value_name for p in self.model.parameters]
+        value_types = [p.value_type for p in self.model.parameters]
+
         return DataSourceParameters(
-            value_types=["DUMMY"]*len(values),
+            value_names=value_names,
+            value_types=value_types,
             values=numpy.array(values)
         )
 

force_bdss/core_plugins/dummy/dummy_dakota/dakota_optimizer.py

 import subprocess
-
 import sys
+import itertools
+import collections
 
 from force_bdss.api import BaseMultiCriteriaOptimizer
 
 
+def rotated_range(start, stop, starting_value):
+    r = list(range(start, stop))
+    start_idx = r.index(starting_value)
+    d = collections.deque(r)
+    d.rotate(-start_idx)
+    return list(d)
+
+
 class DummyDakotaOptimizer(BaseMultiCriteriaOptimizer):
     def run(self):
-        print("Running dakota optimizer")
-        for initial_value in range(10):
+        parameters = self.model.parameters
+
+        values = []
+        for p in parameters:
+            values.append(
+                rotated_range(int(p.lower_bound),
+                              int(p.upper_bound),
+                              int(p.initial_value))
+            )
+
+        value_iterator = itertools.product(*values)
+
+        for value in value_iterator:
             ps = subprocess.Popen(
                 [sys.argv[0],
                  "--evaluate",
@@ -16,5 +36,7 @@ class DummyDakotaOptimizer(BaseMultiCriteriaOptimizer):
                 stdout=subprocess.PIPE,
                 stdin=subprocess.PIPE)
 
-            out = ps.communicate("{}".format(initial_value).encode("utf-8"))
-            print("{}: {}".format(initial_value, out[0].decode("utf-8")))
+            out = ps.communicate(
+                " ".join([str(v) for v in value]).encode("utf-8"))
+            print("{}: {}".format(" ".join([str(v) for v in value]),
+                                  out[0].decode("utf-8")))

force_bdss/core_plugins/dummy/dummy_dakota/tests/test_dakota_communicator.py

+import unittest
+
+from force_bdss.bdss_application import BDSSApplication
+
+from force_bdss.core_plugins.dummy.dummy_dakota.dakota_bundle import (
+    DummyDakotaBundle)
+from force_bdss.data_sources.data_source_parameters import DataSourceParameters
+
+from force_bdss.mco.parameters.base_mco_parameter_factory import \
+    BaseMCOParameterFactory
+from force_bdss.mco.parameters.core_mco_parameters import RangedMCOParameter
+
+try:
+    import mock
+except ImportError:
+    from unittest import mock
+
+
+class TestDakotaCommunicator(unittest.TestCase):
+    def test_receive_from_mco(self):
+        bundle = DummyDakotaBundle()
+        mock_application = mock.Mock(spec=BDSSApplication)
+        mock_parameter_factory = mock.Mock(spec=BaseMCOParameterFactory)
+        model = bundle.create_model()
+        model.parameters = [
+            RangedMCOParameter(mock_parameter_factory)
+        ]
+        comm = bundle.create_communicator(mock_application, model)
+
+        with mock.patch("sys.stdin") as stdin:
+            stdin.read.return_value = "1"
+
+            data = comm.receive_from_mco()
+            self.assertIsInstance(data, DataSourceParameters)
+            self.assertEqual(len(data.value_names), 1)
+            self.assertEqual(len(data.value_types), 1)
+            self.assertEqual(len(data.values), 1)

force_bdss/core_plugins/dummy/dummy_data_source/dummy_data_source.py

@@ -6,6 +6,7 @@ class DummyDataSource(BaseDataSource):
         print(parameters)
         return DataSourceResult(
                 originator=self,
+                value_names=parameters.value_names,
                 value_types=parameters.value_types,
                 values=parameters.values.reshape(
                     parameters.values.shape + (1,)))

force_bdss/core_plugins/dummy/dummy_kpi_calculator/dummy_kpi_calculator.py

@@ -5,8 +5,11 @@ class DummyKPICalculator(BaseKPICalculator):
     id = bundle_id("enthought", "dummy_kpi_calculator")
 
     def run(self, data_source_results):
-        return KPICalculatorResult(
+        res = KPICalculatorResult(
             originator=self,
+            value_names=data_source_results[0].value_names,
             value_types=data_source_results[0].value_types,
             values=data_source_results[0].values.reshape(
                 data_source_results[0].values.shape[:-1]))
+
+        return res

force_bdss/core_plugins/dummy_mco/dakota/dakota_communicator.py

@@ -8,13 +8,21 @@ class DakotaCommunicator(BaseMCOCommunicator):
     def receive_from_mco(self):
         data = sys.stdin.read()
         values = list(map(float, data.split()))
-        value_types = self.model.value_types
-        if len(values) != len(value_types):
-            raise ValueError("Length of provided data differs from the number "
-                             "of expected types. {} {}".format(values,
-                                                               value_types))
+
+        parameters = self.model.parameters
+
+        if len(values) != len(parameters):
+            raise ValueError(
+                "The passed information length is {}, "
+                "but the model specifies {} values.".format(
+                    len(values), len(parameters)
+                ))
+
+        value_types = [p.value_type for p in parameters]
+        value_names = [p.value_name for p in parameters]
 
         return DataSourceParameters(
+            value_names=value_names,
             value_types=value_types,
             values=numpy.array(values)
         )

force_bdss/data_sources/data_source_parameters.py

@@ -2,14 +2,19 @@ from traits.api import HasStrictTraits, Array, List, String
 
 
 class DataSourceParameters(HasStrictTraits):
+    value_names = List(String)
     value_types = List(String)
     values = Array(shape=(None,))
 
     def __str__(self):
         return """
         DataSourceParameters
+        value_names:
+        {}
         value_types:
         {}
         values:
         {}
-        """.format(str(self.value_types), str(self.values))
+        """.format(str(self.value_names),
+                   str(self.value_types),
+                   str(self.values))

force_bdss/data_sources/data_source_result.py

@@ -13,6 +13,7 @@ class DataSourceResult(HasTraits):
     the importance and reliability of that value. It should be an enumeration
     value such as HIGH, MEDIUM, POOR"""
     originator = Instance(BaseDataSource)
+    value_names = List(String)
     value_types = List(String)
     values = Array(shape=(None, None))
     accuracy = ArrayOrNone(shape=(None, None))
@@ -25,6 +26,9 @@ class DataSourceResult(HasTraits):
         originator:
         {}
 
+        value_names:
+        {}
+
         value_types:
         {}
 
@@ -38,6 +42,7 @@ class DataSourceResult(HasTraits):
         {}
         """.format(
             self.originator,
+            self.value_names,
             self.value_types,
             self.values,
             self.accuracy,

force_bdss/ids.py

@@ -25,6 +25,30 @@ def bundle_id(producer, identifier):
         A unique identifier for the bundle. The producer has authority and
         control over the uniqueness of this identifier.
 
+    Returns
+    -------
+    str: an identifier to be used in the bundle.
+    """
+    return _string_id("bundle", producer, identifier)
+
+
+def mco_parameter_id(producer, identifier):
+    return _string_id("mco_parameter", producer, identifier)
+
+
+def _string_id(entity_namespace, producer, identifier):
+    """Creates an id for a generic entity.
+
+    Parameters
+    ----------
+    entity_namespace: str
+        A namespace for the entity we want to address (e.g. "bundle")
+    producer: str
+        the company or research institute unique identifier (e.g. "enthought")
+    identifier: str
+        A unique identifier for the bundle. The producer has authority and
+        control over the uniqueness of this identifier.
+
     Returns
     -------
     str: an identifier to be used in the bundle.
@@ -35,7 +59,7 @@ def bundle_id(producer, identifier):
             " " not in entry and
             len(entry) != 0)
 
-    if not all(map(is_valid, [producer, identifier])):
+    if not all(map(is_valid, [entity_namespace, producer, identifier])):
         raise ValueError("Invalid parameters specified.")
 
-    return "force.bdss.bundles.{}.{}".format(producer, identifier)
+    return "force.bdss.{}.{}.{}".format(entity_namespace, producer, identifier)

force_bdss/io/tests/test_workflow_reader.py

@@ -6,6 +6,8 @@ from force_bdss.bundle_registry_plugin import BundleRegistryPlugin
 from force_bdss.io.workflow_reader import (
     WorkflowReader,
     InvalidVersionException, InvalidFileException)
+from force_bdss.mco.parameters.mco_parameter_factory_registry import \
+    MCOParameterFactoryRegistry
 
 try:
     import mock
@@ -16,7 +18,12 @@ except ImportError:
 class TestWorkflowReader(unittest.TestCase):
     def setUp(self):
         self.mock_bundle_registry = mock.Mock(spec=BundleRegistryPlugin)
-        self.wfreader = WorkflowReader(self.mock_bundle_registry)
+        self.mock_mco_parameter_registry = mock.Mock(
+            spec=MCOParameterFactoryRegistry)
+
+        self.wfreader = WorkflowReader(
+            self.mock_bundle_registry,
+            self.mock_mco_parameter_registry)
 
     def test_initialization(self):
         self.assertEqual(self.wfreader.bundle_registry,

force_bdss/io/tests/test_workflow_writer.py

@@ -4,13 +4,18 @@ from six import StringIO
 
 from force_bdss.bundle_registry_plugin import BundleRegistryPlugin
 from force_bdss.io.workflow_reader import WorkflowReader
+from force_bdss.mco.parameters.base_mco_parameter import BaseMCOParameter
+from force_bdss.mco.parameters.base_mco_parameter_factory import \
+    BaseMCOParameterFactory
+from force_bdss.mco.parameters.mco_parameter_factory_registry import \
+    MCOParameterFactoryRegistry
 
 try:
     import mock
 except ImportError:
     from unittest import mock
 
-from force_bdss.ids import bundle_id
+from force_bdss.ids import bundle_id, mco_parameter_id
 from force_bdss.io.workflow_writer import WorkflowWriter
 from force_bdss.mco.base_mco_model import BaseMCOModel
 from force_bdss.mco.i_multi_criteria_optimizer_bundle import \
@@ -33,6 +38,9 @@ class TestWorkflowWriter(unittest.TestCase):
         self.mock_registry.mco_bundle_by_id = mock.Mock(
             return_value=mock_mco_bundle)
 
+        self.mock_mco_parameter_registry = mock.Mock(
+            spec=MCOParameterFactoryRegistry)
+
     def test_write(self):
         wfwriter = WorkflowWriter()
         fp = StringIO()
@@ -51,7 +59,8 @@ class TestWorkflowWriter(unittest.TestCase):
         wf = self._create_mock_workflow()
         wfwriter.write(wf, fp)
         fp.seek(0)
-        wfreader = WorkflowReader(self.mock_registry)
+        wfreader = WorkflowReader(self.mock_registry,
+                                  self.mock_mco_parameter_registry)
         wf_result = wfreader.read(fp)
         self.assertEqual(wf_result.multi_criteria_optimizer.bundle.id,
                          wf.multi_criteria_optimizer.bundle.id)
@@ -62,4 +71,12 @@ class TestWorkflowWriter(unittest.TestCase):
             mock.Mock(
                 spec=IMultiCriteriaOptimizerBundle,
                 id=bundle_id("enthought", "mock")))
+        wf.multi_criteria_optimizer.parameters = [
+            BaseMCOParameter(
+                factory=mock.Mock(
+                    spec=BaseMCOParameterFactory,
+                    id=mco_parameter_id("enthought", "mock")
+                )
+            )
+        ]
         return wf

force_bdss/io/workflow_reader.py

@@ -3,8 +3,10 @@ import logging
 
 from traits.api import HasStrictTraits, Instance
 
-from ..workspecs.workflow import Workflow
+from ..mco.parameters.mco_parameter_factory_registry import (
+    MCOParameterFactoryRegistry)
 from ..bundle_registry_plugin import BundleRegistryPlugin
+from ..workspecs.workflow import Workflow
 
 SUPPORTED_FILE_VERSIONS = ["1"]
 
@@ -26,7 +28,15 @@ class WorkflowReader(HasStrictTraits):
     #: bundle-specific model objects.
     bundle_registry = Instance(BundleRegistryPlugin)
 
-    def __init__(self, bundle_registry, *args, **kwargs):
+    #: The registry for the MCO parameters. At the moment this
+    #: is not extensible via plugins as the one above.
+    mco_parameter_registry = Instance(MCOParameterFactoryRegistry)
+
+    def __init__(self,
+                 bundle_registry,
+                 mco_parameter_registry,
+                 *args,
+                 **kwargs):
         """Initializes the reader.
 
         Parameters
@@ -36,6 +46,7 @@ class WorkflowReader(HasStrictTraits):
             for a bundle identified by a given id.
         """
         self.bundle_registry = bundle_registry
+        self.mco_parameter_registry = mco_parameter_registry
 
         super(WorkflowReader, self).__init__(*args, **kwargs)
 
@@ -115,8 +126,12 @@ class WorkflowReader(HasStrictTraits):
 
         mco_id = mco_data["id"]
         mco_bundle = registry.mco_bundle_by_id(mco_id)
-        return mco_bundle.create_model(
+        model_data = wf_data["multi_criteria_optimizer"]["model_data"]
+        model_data["parameters"] = self._extract_mco_parameters(
+            model_data["parameters"])
+        model = mco_bundle.create_model(
             wf_data["multi_criteria_optimizer"]["model_data"])
+        return model
 
     def _extract_data_sources(self, wf_data):
         """Extracts the data sources from the workflow dictionary data.
@@ -166,3 +181,27 @@ class WorkflowReader(HasStrictTraits):
                 kpic_bundle.create_model(kpic_entry["model_data"]))
 
         return kpi_calculators
+
+    def _extract_mco_parameters(self, parameters_data):
+        """Extracts the MCO parameters from the data as dictionary.
+
+        Parameters
+        ----------
+        parameters_data: dict
+            The content of the parameter data key in the MCO model data.
+
+        Returns
+        -------
+        List of instances of a subclass of BaseMCOParameter
+        """
+        registry = self.mco_parameter_registry
+
+        parameters = []
+
+        for p in parameters_data:
+            id = p["id"]
+            factory = registry.get_factory_by_id(id)
+            model = factory.create_model(p["model_data"])
+            parameters.append(model)
+
+        return parameters

force_bdss/io/workflow_writer.py

@@ -27,6 +27,18 @@ class WorkflowWriter(HasStrictTraits):
             "id": workflow.multi_criteria_optimizer.bundle.id,
             "model_data": workflow.multi_criteria_optimizer.__getstate__()
         }
+
+        parameters_data = []
+        for param in wf_data["multi_criteria_optimizer"]["model_data"]["parameters"]:  # noqa
+            parameters_data.append(
+                {
+                    "id": param.factory.id,
+                    "model_data": param.__getstate__()
+                }
+            )
+
+        wf_data["multi_criteria_optimizer"]["model_data"]["parameters"] = parameters_data  # noqa
+
         kpic_data = []
         for kpic in workflow.kpi_calculators:
             kpic_data.append({

force_bdss/kpi/kpi_calculator_result.py

@@ -5,7 +5,37 @@ from .base_kpi_calculator import BaseKPICalculator
 
 class KPICalculatorResult(HasTraits):
     originator = Instance(BaseKPICalculator)
+    value_names = List(String)
     value_types = List(String)
     values = Array(shape=(None, ))
     accuracy = ArrayOrNone(shape=(None, ))
     quality = ArrayOrNone(shape=(None, ))
+
+    def __str__(self):
+        return """
+        DataSourceResults
+
+        originator:
+        {}
+
+        value_names:
+        {}
+
+        value_types:
+        {}
+
+        values:
+        {}
+
+        Accuracy:
+        {}
+
+        Quality:
+        {}
+        """.format(
+                self.originator,
+                self.value_names,
+                self.value_types,
+                self.values,
+                self.accuracy,
+                self.quality)

force_bdss/mco/base_mco_model.py

-from traits.api import ABCHasStrictTraits, Instance
+from traits.api import ABCHasStrictTraits, Instance, List
 
+from .parameters.base_mco_parameter import BaseMCOParameter
 from .i_multi_criteria_optimizer_bundle import IMultiCriteriaOptimizerBundle
 
 
@@ -17,6 +18,9 @@ class BaseMCOModel(ABCHasStrictTraits):
                       visible=False,
                       transient=True)
 
+    # A list of the parameters for the MCO
+    parameters = List(BaseMCOParameter)
+
     def __init__(self, bundle, *args, **kwargs):
         self.bundle = bundle
         super(BaseMCOModel, self).__init__(*args, **kwargs)

force_bdss/mco/parameters/base_mco_parameter.py

+from traits.api import HasStrictTraits, String, Instance
+
+from force_bdss.mco.parameters.base_mco_parameter_factory import \
+    BaseMCOParameterFactory
+
+
+class BaseMCOParameter(HasStrictTraits):
+    """The base class of all MCO Parameter models.
+    Must be reimplemented by specific classes handling the specific parameter
+    that MCOs understand.
+    """
+
+    #: The generating factory. Used to retrieve the ID at serialization.
+    factory = Instance(BaseMCOParameterFactory, visible=False, transient=True)
+
+    #: A user defined name for the parameter
+    value_name = String()
+
+    #: A CUBA key describing the type of the parameter
+    value_type = String()
+
+    def __init__(self, factory, *args, **kwargs):
+        self.factory = factory
+        super(BaseMCOParameter, self).__init__(*args, **kwargs)

force_bdss/mco/parameters/base_mco_parameter_factory.py

+from traits.has_traits import HasStrictTraits
+from traits.trait_types import String, Type
+
+
+class BaseMCOParameterFactory(HasStrictTraits):
+    """Factory that produces the model instance of a given BASEMCOParameter
+    instance.
+
+    Must be reimplemented for the specific parameter."""
+
+    #: A unique string identifying the parameter
+    id = String()
+
+    #: A user friendly name (for the UI)
+    name = String("Undefined parameter")
+
+    #: A long description of the parameter
+    description = String("Undefined parameter")
+
+    # The model class to instantiate when create_model is called.
+    model_class = Type('BaseMCOParameter')
+
+    def create_model(self, data_values=None):
+        """Creates the instance of the model class and returns it.
+        """
+        if data_values is None:
+            data_values = {}
+
+        return self.model_class(factory=self, **data_values)