Merge pull request #29 from force-h2020/extract-io-layer

Extract io layer to writer/reader class

doc/source/api/force_bdss.io.rst

+force_bdss.io package
+=====================
+
+Subpackages
+-----------
+
+.. toctree::
+
+    force_bdss.io.tests
+
+Submodules
+----------
+
+force_bdss.io.workflow_reader module
+------------------------------------
+
+.. automodule:: force_bdss.io.workflow_reader
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+force_bdss.io.workflow_writer module
+------------------------------------
+
+.. automodule:: force_bdss.io.workflow_writer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: force_bdss.io
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/source/api/force_bdss.io.tests.rst

+force_bdss.io.tests package
+===========================
+
+Submodules
+----------
+
+force_bdss.io.tests.test_workflow_reader module
+-----------------------------------------------
+
+.. automodule:: force_bdss.io.tests.test_workflow_reader
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: force_bdss.io.tests
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/source/api/force_bdss.rst

@@ -9,6 +9,7 @@ Subpackages
     force_bdss.cli
     force_bdss.core_plugins
     force_bdss.data_sources
+    force_bdss.io
     force_bdss.kpi
     force_bdss.mco
     force_bdss.tests

doc/source/api/force_bdss.tests.fixtures.rst

+force_bdss.tests.fixtures package
+=================================
+
+Module contents
+---------------
+
+.. automodule:: force_bdss.tests.fixtures
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/source/api/force_bdss.tests.rst

 force_bdss.tests package
 ========================
 
+Subpackages
+-----------
+
+.. toctree::
+
+    force_bdss.tests.fixtures
+
 Submodules
 ----------
 

doc/source/api/force_bdss.workspecs.rst

@@ -4,30 +4,6 @@ force_bdss.workspecs package
 Submodules
 ----------
 
-force_bdss.workspecs.data_source module
----------------------------------------
-
-.. automodule:: force_bdss.workspecs.data_source
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-force_bdss.workspecs.kpi_calculator module
-------------------------------------------
-
-.. automodule:: force_bdss.workspecs.kpi_calculator
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-force_bdss.workspecs.multi_criteria_optimizer module
-----------------------------------------------------
-
-.. automodule:: force_bdss.workspecs.multi_criteria_optimizer
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 force_bdss.workspecs.workflow module
 ------------------------------------
 

force_bdss/base_core_driver.py

 from envisage.plugin import Plugin
 from traits.trait_types import Instance
 
-from force_bdss.bundle_registry_plugin import (
+from .bundle_registry_plugin import (
     BundleRegistryPlugin,
     BUNDLE_REGISTRY_PLUGIN_ID
 )
+from .io.workflow_reader import WorkflowReader
+from .workspecs.workflow import Workflow
 
 
 class BaseCoreDriver(Plugin):
@@ -14,5 +16,13 @@ class BaseCoreDriver(Plugin):
 
     bundle_registry = Instance(BundleRegistryPlugin)
 
+    #: Deserialized content of the workflow file.
+    workflow = Instance(Workflow)
+
     def _bundle_registry_default(self):
         return self.application.get_plugin(BUNDLE_REGISTRY_PLUGIN_ID)
+
+    def _workflow_default(self):
+        reader = WorkflowReader(self.bundle_registry)
+        with open(self.application.workflow_filepath) as f:
+            return reader.read(f)

force_bdss/base_extension_plugin.py

 from envisage.plugin import Plugin
 from traits.trait_types import List
 
-from force_bdss.data_sources.i_data_source_bundle import IDataSourceBundle
-from force_bdss.kpi.i_kpi_calculator_bundle import IKPICalculatorBundle
-from force_bdss.mco.i_multi_criteria_optimizer_bundle import \
+from .data_sources.i_data_source_bundle import IDataSourceBundle
+from .kpi.i_kpi_calculator_bundle import IKPICalculatorBundle
+from .mco.i_multi_criteria_optimizer_bundle import \
     IMultiCriteriaOptimizerBundle
 
 

force_bdss/bdss_application.py

-import json
-
 from stevedore import extension
 from stevedore.exception import NoMatches
+
 from envisage.api import Application
 from envisage.core_plugin import CorePlugin
+from traits.api import Unicode, Bool
 
-from force_bdss.bundle_registry_plugin import BundleRegistryPlugin
-from force_bdss.core_evaluation_driver import CoreEvaluationDriver
-from force_bdss.core_mco_driver import CoreMCODriver
-
-from traits.api import Unicode, Bool, Instance
-
-from force_bdss.workspecs.workflow import Workflow
+from .bundle_registry_plugin import BundleRegistryPlugin
+from .core_evaluation_driver import CoreEvaluationDriver
+from .core_mco_driver import CoreMCODriver
 
 
 class BDSSApplication(Application):
@@ -22,9 +18,6 @@ class BDSSApplication(Application):
     #: The path of the workflow file to open
     workflow_filepath = Unicode()
 
-    #: Deserialized content of the workflow file.
-    workflow = Instance(Workflow)
-
     #: This flags signals to the application not to execute and orchestrate
     #: the MCO, but instead to perform a single evaluation under the
     #: coordination of the MCO itself. See design notes for more details.
@@ -56,7 +49,3 @@ class BDSSApplication(Application):
             print("No extensions found")
 
         super(BDSSApplication, self).__init__(plugins=plugins)
-
-    def _workflow_default(self):
-        with open(self.workflow_filepath) as f:
-            return Workflow.from_json(json.load(f))

force_bdss/bundle_registry_plugin.py

@@ -2,11 +2,12 @@ from envisage.extension_point import ExtensionPoint
 from envisage.plugin import Plugin
 from traits.api import List
 
-from force_bdss.data_sources.i_data_source_bundle import (
+from .data_sources.i_data_source_bundle import (
     IDataSourceBundle)
-from force_bdss.kpi.i_kpi_calculator_bundle import IKPICalculatorBundle
-from force_bdss.mco.i_multi_criteria_optimizer_bundle import (
-    IMultiCriteriaOptimizerBundle)
+from .kpi.i_kpi_calculator_bundle import IKPICalculatorBundle
+from .mco.i_multi_criteria_optimizer_bundle import (
+    IMultiCriteriaOptimizerBundle
+)
 
 
 BUNDLE_REGISTRY_PLUGIN_ID = "force.bdss.plugins.bundle_registry"

force_bdss/cli/tests/fixtures/test_csv.json

-{
-  "version": "1",
-  "multi_criteria_optimizer": {
-    "id": "force.bdss.bundles.enthought.dakota",
-    "model_data": {
-      "value_types": ["DUMMY"]
-    }
-  },
-  "data_sources": [
-    {
-      "id": "force.bdss.bundles.enthought.csv_extractor",
-      "model_data": {
-        "filename": "foo.csv",
-        "row": 3,
-        "column": 5,
-        "cuba_type": "PRESSURE"
-      }
-    },
-    {
-      "id": "force.bdss.bundles.enthought.csv_extractor",
-      "model_data": {
-        "filename": "foo.csv",
-        "row": 3,
-        "column": 5,
-        "cuba_type": "PRESSURE"
-      }
-    }
-  ],
-  "kpi_calculators": [
-    {
-      "id": "force.bdss.bundles.enthought.kpi_adder",
-      "model_data": {
-        "cuba_type_in": "PRESSURE",
-        "cuba_type_out": "TOTAL_PRESSURE"
-      }
-    }
-  ]
-}

force_bdss/cli/tests/fixtures/test_csv_v2.json

-{
-  "version": "2"
-}

force_bdss/cli/tests/test_execution.py

@@ -3,6 +3,8 @@ import subprocess
 import os
 from contextlib import contextmanager
 
+from force_bdss.tests import fixtures
+
 
 @contextmanager
 def cd(dir):
@@ -22,17 +24,17 @@ def fixture_dir():
 
 class TestExecution(unittest.TestCase):
     def test_plain_invocation_mco(self):
-        with cd(fixture_dir()):
+        with cd(fixtures.dirpath()):
             out = subprocess.check_call(["force_bdss", "test_csv.json"])
             self.assertEqual(out, 0)
 
     def test_unsupported_file_input(self):
-        with cd(fixture_dir()):
+        with cd(fixtures.dirpath()):
             with self.assertRaises(subprocess.CalledProcessError):
                 subprocess.check_call(["force_bdss", "test_csv_v2.json"])
 
     def test_corrupted_file_input(self):
-        with cd(fixture_dir()):
+        with cd(fixtures.dirpath()):
             with self.assertRaises(subprocess.CalledProcessError):
                 subprocess.check_call(["force_bdss",
                                        "test_csv_corrupted.json"])

force_bdss/core_evaluation_driver.py

-from traits.has_traits import on_trait_change
+from __future__ import print_function
 
-from force_bdss.base_core_driver import BaseCoreDriver
+import sys
+from traits.api import on_trait_change
+
+from .base_core_driver import BaseCoreDriver
+from .io.workflow_reader import (
+    InvalidVersionException,
+    InvalidFileException
+)
 
 
 class CoreEvaluationDriver(BaseCoreDriver):
@@ -10,11 +17,14 @@ class CoreEvaluationDriver(BaseCoreDriver):
 
     @on_trait_change("application:started")
     def application_started(self):
-        workflow = self.application.workflow
-
-        mco_data = workflow.multi_criteria_optimizer
-        mco_bundle = self.bundle_registry.mco_bundle_by_id(mco_data.id)
-        mco_model = mco_bundle.create_model(mco_data.model_data)
+        try:
+            workflow = self.workflow
+        except (InvalidVersionException, InvalidFileException) as e:
+            print(str(e), file=sys.stderr)
+            sys.exit(1)
+
+        mco_model = workflow.multi_criteria_optimizer
+        mco_bundle = mco_model.bundle
         mco_communicator = mco_bundle.create_communicator(
             self.application,
             mco_model)
@@ -22,22 +32,17 @@ class CoreEvaluationDriver(BaseCoreDriver):
         parameters = mco_communicator.receive_from_mco()
 
         ds_results = []
-        for requested_ds in workflow.data_sources:
-            ds_bundle = self.bundle_registry.data_source_bundle_by_id(
-                requested_ds.id)
-            ds_model = ds_bundle.create_model(requested_ds.model_data)
-            data_source = ds_bundle.create_data_source(
-                self.application, ds_model)
+        for ds_model in workflow.data_sources:
+            ds_bundle = ds_model.bundle
+            data_source = ds_bundle.create_data_source(self.application,
+                                                       ds_model)
             ds_results.append(data_source.run(parameters))
 
         kpi_results = []
-        for requested_kpic in workflow.kpi_calculators:
-            kpic_bundle = self.bundle_registry.kpi_calculator_bundle_by_id(
-                requested_kpic.id)
-            ds_model = kpic_bundle.create_model(
-                requested_kpic.model_data)
-            kpi_calculator = kpic_bundle.create_data_source(
-                self.application, ds_model)
+        for kpic_model in workflow.kpi_calculators:
+            kpic_bundle = kpic_model.bundle
+            kpi_calculator = kpic_bundle.create_kpi_calculator(
+                self.application, kpic_model)
             kpi_results.append(kpi_calculator.run(ds_results))
 
         mco_communicator.send_to_mco(kpi_results)

force_bdss/core_mco_driver.py

@@ -4,9 +4,11 @@ import sys
 
 from traits.api import on_trait_change
 
-from force_bdss.base_core_driver import BaseCoreDriver
-from force_bdss.workspecs.workflow import (InvalidVersionException,
-                                           InvalidFileException)
+from .base_core_driver import BaseCoreDriver
+from .io.workflow_reader import (
+    InvalidVersionException,
+    InvalidFileException
+)
 
 
 class CoreMCODriver(BaseCoreDriver):
@@ -17,14 +19,12 @@ class CoreMCODriver(BaseCoreDriver):
     @on_trait_change("application:started")
     def application_started(self):
         try:
-            workflow = self.application.workflow
+            workflow = self.workflow
         except (InvalidVersionException, InvalidFileException) as e:
             print(str(e), file=sys.stderr)
             sys.exit(1)
 
-        mco_data = workflow.multi_criteria_optimizer
-        mco_bundle = self.bundle_registry.mco_bundle_by_id(mco_data.id)
-        mco_model = mco_bundle.create_model(mco_data.model_data)
+        mco_model = workflow.multi_criteria_optimizer
+        mco_bundle = mco_model.bundle
         mco = mco_bundle.create_optimizer(self.application, mco_model)
-
         mco.run()

force_bdss/core_plugins/csv_extractor/csv_extractor/csv_extractor_bundle.py

@@ -12,9 +12,9 @@ class CSVExtractorBundle(HasStrictTraits):
 
     def create_model(self, model_data=None):
         if model_data is None:
-            return CSVExtractorModel()
-        else:
-            return CSVExtractorModel.from_json(model_data)
+            model_data = {}
+
+        return CSVExtractorModel(self, **model_data)
 
     def create_data_source(self, application, model):
         return CSVExtractorDataSource(self, application, model)

force_bdss/core_plugins/csv_extractor/csv_extractor/csv_extractor_model.py

@@ -8,12 +8,3 @@ class CSVExtractorModel(BaseDataSourceModel):
     row = Int()
     column = Int()
     cuba_type = String()
-
-    @classmethod
-    def from_json(cls, json_data):
-        return cls(
-            filename=json_data["filename"],
-            row=json_data["row"],
-            column=json_data["column"],
-            cuba_type=json_data["cuba_type"]
-        )

force_bdss/core_plugins/dummy_kpi/kpi_adder/kpi_adder_bundle.py

@@ -13,9 +13,9 @@ class KPIAdderBundle(HasStrictTraits):
 
     def create_model(self, model_data=None):
         if model_data is None:
-            return KPIAdderModel()
-        else:
-            return KPIAdderModel.from_json(model_data)
+            model_data = {}
 
-    def create_data_source(self, application, model):
+        return KPIAdderModel(self, **model_data)
+
+    def create_kpi_calculator(self, application, model):
         return KPIAdderCalculator(self, application, model)

force_bdss/core_plugins/dummy_kpi/kpi_adder/kpi_adder_model.py

@@ -6,10 +6,3 @@ from force_bdss.api import BaseKPICalculatorModel
 class KPIAdderModel(BaseKPICalculatorModel):
     cuba_type_in = String()
     cuba_type_out = String()
-
-    @classmethod
-    def from_json(cls, json_data):
-        return cls(
-            cuba_type_in=json_data["cuba_type_in"],
-            cuba_type_out=json_data["cuba_type_out"]
-        )

force_bdss/core_plugins/dummy_mco/dakota/dakota_bundle.py

@@ -12,9 +12,8 @@ class DakotaBundle(HasStrictTraits):
 
     def create_model(self, model_data=None):
         if model_data is None:
-            return DakotaModel()
-        else:
-            return DakotaModel.from_json(model_data)
+            model_data = {}
+        return DakotaModel(self, **model_data)
 
     def create_optimizer(self, application, model):
         return DakotaOptimizer(self, application, model)